mirror of
https://github.com/alirezarezvani/claude-skills.git
synced 2026-08-28 04:24:58 +00:00
Merge remote-tracking branch 'origin/dev' into claude/productivity-skills-audit-obucop
This commit is contained in:
Claude
commit
f8f9513e6f
59 changed files with 7664 additions and 34 deletions
|
|
@ -8,8 +8,8 @@
|
|||
"homepage": "https://github.com/alirezarezvani/claude-skills",
|
||||
"repository": "https://github.com/alirezarezvani/claude-skills",
|
||||
"metadata": {
|
||||
"description": "355 production-ready skills across 18 domains (engineering, engineering-core, marketing, product, c-level, compliance-os, project management, RA/QM, business growth, finance, productivity, marketing top-level, research, research-ops, business-operations, commercial, markdown-html, loop-library, plus standards). 602 Python tools, 731 reference guides, 99 agents (cs-* + personas), 109 slash commands across 83 marketplace plugins. v2.11.1 turns product-team and project-management into agent-harness domains: fork-orchestrators with deterministic goal routers, a Jira MCP snapshot bridge (Kanban flow metrics + Monte Carlo forecasting), a delegation-governance loop gate, a continuous-discovery cadence tracker, and an Opportunity Solution Tree linter, with /cs:pm and /cs:product command families. v2.10.3 completes the markdown-html domain with md-slides — slide-deck converter (arrow-key / Space / PgDn / Home/End / P keyboard navigation + presenter mode with split-view clock + speaker notes + next-slide preview + URL-hash deep linking like #3 for direct slide jumps + @media print page-per-slide for browser-native PDF export). Reuses md-document's markdown parser; vanilla JS only (no framework runtime); Prism.js opt-in via --syntax. Joins md-review (v2.10.2 code-review converter), md-document (v2.10.1 long-form converter), and the v2.10.0 foundation (orchestrator + design-system). Compatible with Claude Code, Codex CLI, Gemini CLI, Cursor, OpenClaw, Hermes Agent, Mistral Vibe, and 5 more coding agents.",
|
||||
"version": "2.11.1"
|
||||
"description": "359 production-ready skills across 18 domains (engineering, engineering-core, marketing, product, c-level, compliance-os, project management, RA/QM, business growth, finance, productivity, marketing top-level, research, research-ops, business-operations, commercial, markdown-html, loop-library, plus standards). 635 Python tools, 732 reference guides, 99 agents (cs-* + personas), 111 slash commands across 85 marketplace plugins. v2.11.2 vendors engineering/skillopt-sleep — a verbatim copy of microsoft/SkillOpt's stdlib-only skillopt_sleep engine + Claude Code plugin surface, giving a local agent a nightly gated self-improvement cycle (read-only session harvest -> mine -> offline replay -> held-out-gated CLAUDE.md/SKILL.md edits -> staged for explicit /skillopt-sleep adopt). productivity/fable-goal (unreleased, post-v2.11.1) converts a rambling description of a desired outcome into one polished /goal prompt for a fresh autonomous session. v2.11.1 turns product-team and project-management into agent-harness domains: fork-orchestrators with deterministic goal routers, a Jira MCP snapshot bridge (Kanban flow metrics + Monte Carlo forecasting), a delegation-governance loop gate, a continuous-discovery cadence tracker, and an Opportunity Solution Tree linter, with /cs:pm and /cs:product command families. v2.10.3 completes the markdown-html domain with md-slides — slide-deck converter (arrow-key / Space / PgDn / Home/End / P keyboard navigation + presenter mode with split-view clock + speaker notes + next-slide preview + URL-hash deep linking like #3 for direct slide jumps + @media print page-per-slide for browser-native PDF export). Reuses md-document's markdown parser; vanilla JS only (no framework runtime); Prism.js opt-in via --syntax. Joins md-review (v2.10.2 code-review converter), md-document (v2.10.1 long-form converter), and the v2.10.0 foundation (orchestrator + design-system). Compatible with Claude Code, Codex CLI, Gemini CLI, Cursor, OpenClaw, Hermes Agent, Mistral Vibe, and 5 more coding agents.",
|
||||
"version": "2.11.2"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
|
|
@ -1112,6 +1112,25 @@
|
|||
],
|
||||
"category": "productivity"
|
||||
},
|
||||
{
|
||||
"name": "fable-goal",
|
||||
"source": "./productivity/fable-goal",
|
||||
"description": "Convert a rambling description of a desired outcome into one polished, autonomous /goal prompt ready to paste into a fresh session. Extracts deliverable/quantity/stakes/tools/destination, asks at most one question batch, verifies every named resource against the live environment, writes a 150-350 word prose prompt with the seven-part anatomy, and self-checks six binary criteria before delivering. The output is a single copy-paste prompt, never the build itself.",
|
||||
"version": "2.11.1",
|
||||
"author": {
|
||||
"name": "Alireza Rezvani"
|
||||
},
|
||||
"keywords": [
|
||||
"goal-prompt",
|
||||
"prompt-writing",
|
||||
"autonomous-agent",
|
||||
"prompt-engineering",
|
||||
"fable",
|
||||
"ramble-to-prompt",
|
||||
"productivity"
|
||||
],
|
||||
"category": "productivity"
|
||||
},
|
||||
{
|
||||
"name": "landing",
|
||||
"source": "./marketing/landing",
|
||||
|
|
@ -1355,6 +1374,26 @@
|
|||
],
|
||||
"category": "development"
|
||||
},
|
||||
{
|
||||
"name": "skillopt-sleep",
|
||||
"source": "./engineering/skillopt-sleep",
|
||||
"description": "Nightly offline self-evolution for this repo's Claude agent: harvests past Claude Code sessions (read-only), mines recurring tasks, replays them offline on your own API budget, and consolidates learnings into validated CLAUDE.md memory and SKILL.md skills behind a held-out gate, staged for review (never auto-applied). Verbatim vendor of microsoft/SkillOpt's stdlib-only skillopt_sleep engine + Claude Code plugin surface (MIT).",
|
||||
"version": "1.0.0",
|
||||
"author": {
|
||||
"name": "Alireza Rezvani"
|
||||
},
|
||||
"keywords": [
|
||||
"self-improvement",
|
||||
"memory-consolidation",
|
||||
"skillopt",
|
||||
"dreams",
|
||||
"sleep",
|
||||
"continual-learning",
|
||||
"offline-optimization",
|
||||
"microsoft"
|
||||
],
|
||||
"category": "development"
|
||||
},
|
||||
{
|
||||
"name": "business-operations-skills",
|
||||
"source": "./business-operations",
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
"name": "claude-code-skills",
|
||||
"description": "Production-ready skill packages for AI agents - Marketing, Engineering, Product, C-Level, PM, and RA/QM",
|
||||
"repository": "https://github.com/alirezarezvani/claude-skills",
|
||||
"total_skills": 355,
|
||||
"total_skills": 357,
|
||||
"skills": [
|
||||
{
|
||||
"name": "business-growth-skills",
|
||||
|
|
@ -1265,18 +1265,18 @@
|
|||
"category": "engineering-advanced",
|
||||
"description": "Resume a paused experiment. Checkout the experiment branch, read results history, continue iterating. Use when the user runs /ar:resume or asks to pick up a previously started autoresearch experiment."
|
||||
},
|
||||
{
|
||||
"name": "run",
|
||||
"source": "../../engineering/agenthub/skills/run",
|
||||
"category": "engineering-advanced",
|
||||
"description": "One-shot lifecycle command that chains init \u2192 baseline \u2192 spawn \u2192 eval \u2192 merge in a single invocation. Use when the user runs /hub:run or asks to execute a full AgentHub competition end-to-end."
|
||||
},
|
||||
{
|
||||
"name": "run",
|
||||
"source": "../../engineering/autoresearch-agent/skills/run",
|
||||
"category": "engineering-advanced",
|
||||
"description": "Run a single experiment iteration. Edit the target file, evaluate, keep or discard. Use when the user runs /ar:run or asks for one manual autoresearch iteration."
|
||||
},
|
||||
{
|
||||
"name": "run",
|
||||
"source": "../../engineering/agenthub/skills/run",
|
||||
"category": "engineering-advanced",
|
||||
"description": "One-shot lifecycle command that chains init \u2192 baseline \u2192 spawn \u2192 eval \u2192 merge in a single invocation. Use when the user runs /hub:run or asks to execute a full AgentHub competition end-to-end."
|
||||
},
|
||||
{
|
||||
"name": "runbook-generator",
|
||||
"source": "../../engineering/skills/runbook-generator",
|
||||
|
|
@ -1325,6 +1325,12 @@
|
|||
"category": "engineering-advanced",
|
||||
"description": "Validate, test, and score the quality of skills within the claude-skills ecosystem. Comprehensive meta-skill: structure validation, Python script testing (syntax + imports + runtime + output format), multi-dimensional quality scoring with letter grades and tier classification (BASIC/STANDARD/POWERFUL). Use when authoring a new skill, auditing existing skills for tier promotion, setting up pre-commit hooks for skill quality, or integrating skill QA into CI."
|
||||
},
|
||||
{
|
||||
"name": "skillopt-sleep",
|
||||
"source": "../../engineering/skillopt-sleep/skills/skillopt-sleep",
|
||||
"category": "engineering-advanced",
|
||||
"description": "Use when the user wants their Claude agent to self-improve from past usage, asks about a nightly/offline 'sleep' or 'dream' cycle, memory/skill consolidation, or says things like 'make my agent better the more I use it', 'review my past sessions', 'learn my preferences', 'consolidate what you learned', 'run the sleep cycle', or wants to schedule offline self-optimization. Drives the skillopt_sleep engine: harvest past sessions -> mine recurring tasks -> replay offline -> consolidate validated CLAUDE.md and SKILL.md behind a held-out gate."
|
||||
},
|
||||
{
|
||||
"name": "slo-architect",
|
||||
"source": "../../engineering/skills/slo-architect",
|
||||
|
|
@ -1361,18 +1367,18 @@
|
|||
"category": "engineering-advanced",
|
||||
"description": "Run hypothesis tests, analyze A/B experiment results, calculate sample sizes, and interpret statistical significance with effect sizes. Use when you need to validate whether observed differences are real, size an experiment correctly before launch, or interpret test results with confidence."
|
||||
},
|
||||
{
|
||||
"name": "status",
|
||||
"source": "../../engineering/agenthub/skills/status",
|
||||
"category": "engineering-advanced",
|
||||
"description": "Show DAG state, agent progress, and branch status for an AgentHub session. Use when the user runs /hub:status or asks how the AgentHub agents are doing."
|
||||
},
|
||||
{
|
||||
"name": "status",
|
||||
"source": "../../engineering/autoresearch-agent/skills/status",
|
||||
"category": "engineering-advanced",
|
||||
"description": "Show experiment dashboard with results, active loops, and progress. Use when the user runs /ar:status or asks how an autoresearch experiment is going."
|
||||
},
|
||||
{
|
||||
"name": "status",
|
||||
"source": "../../engineering/agenthub/skills/status",
|
||||
"category": "engineering-advanced",
|
||||
"description": "Show DAG state, agent progress, and branch status for an AgentHub session. Use when the user runs /hub:status or asks how the AgentHub agents are doing."
|
||||
},
|
||||
{
|
||||
"name": "strict-api",
|
||||
"source": "../../engineering/strict-api",
|
||||
|
|
@ -1853,6 +1859,12 @@
|
|||
"category": "productivity",
|
||||
"description": "Captures and organizes chaotic brain dumps into a structured, actionable system with zero information loss. Use this skill whenever the user says 'capture this', 'brain dump', 'let me dump some ideas', 'I've got a bunch of thoughts', 'here's everything on my mind', 'idea dump', 'let me get this out of my head', 'I need to organize my thoughts', 'here's what I'm thinking', or any variation where someone is unloading a messy stream of ideas, tasks, thoughts, and plans wanting them turned into something coherent. Also trigger when the user pastes or dictates a long, unstructured block of mixed ideas \u2014 even without the exact phrase \u2014 the intent is the same. Fast-to-action by design: no upfront intake. Output is four sections (Projects/Ideas, Tasks, Connections, How I Can Help) ending with a directive question. Asks at most one mid-organization clarifying question when a single item is genuinely ambiguous between task and project."
|
||||
},
|
||||
{
|
||||
"name": "fable-goal",
|
||||
"source": "../../productivity/fable-goal/skills/fable-goal",
|
||||
"category": "productivity",
|
||||
"description": "Convert a rambling description of a desired outcome into one polished, autonomous /goal prompt ready to paste into a fresh session. Use when the user says \"/fable-goal\", \"turn this into a goal prompt\", \"write me a fable prompt\", \"write the prompt that builds X\", or rambles about something they want made and asks for the prompt that makes it happen. The output is a single copy-paste prompt, never the build itself. Do NOT use when the user wants the thing built right now in this session \u2014 only when they want the PROMPT that will make it happen in a fresh session."
|
||||
},
|
||||
{
|
||||
"name": "handoff",
|
||||
"source": "../../productivity/handoff/skills/handoff",
|
||||
|
|
@ -2173,7 +2185,7 @@
|
|||
"description": "Software engineering and technical skills"
|
||||
},
|
||||
"engineering-advanced": {
|
||||
"count": 82,
|
||||
"count": 83,
|
||||
"source": "../../engineering",
|
||||
"description": "Advanced engineering skills - agents, RAG, MCP, CI/CD, databases, observability"
|
||||
},
|
||||
|
|
@ -2193,7 +2205,7 @@
|
|||
"description": "Product management and design skills"
|
||||
},
|
||||
"productivity": {
|
||||
"count": 7,
|
||||
"count": 8,
|
||||
"source": "../../productivity",
|
||||
"description": "Personal-productivity skills - capture, email, reflect"
|
||||
},
|
||||
|
|
|
|||
1
.codex/skills/fable-goal
Symbolic link
1
.codex/skills/fable-goal
Symbolic link
|
|
@ -0,0 +1 @@
|
|||
../../productivity/fable-goal/skills/fable-goal
|
||||
|
|
@ -1 +1 @@
|
|||
../../engineering/autoresearch-agent/skills/run
|
||||
../../engineering/agenthub/skills/run
|
||||
1
.codex/skills/skillopt-sleep
Symbolic link
1
.codex/skills/skillopt-sleep
Symbolic link
|
|
@ -0,0 +1 @@
|
|||
../../engineering/skillopt-sleep/skills/skillopt-sleep
|
||||
|
|
@ -1 +1 @@
|
|||
../../engineering/autoresearch-agent/skills/status
|
||||
../../engineering/agenthub/skills/status
|
||||
24
CHANGELOG.md
24
CHANGELOG.md
|
|
@ -5,6 +5,30 @@ All notable changes to the Claude Skills Library will be documented in this file
|
|||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased] — fable-goal: ramble → autonomous /goal prompt (this PR)
|
||||
|
||||
### Added — `productivity/fable-goal`
|
||||
|
||||
Improved port of `duncan-buildroom/freeskills` `fable-goal` (informal "free to use
|
||||
and modify" grant — quoted in the `attribution` block, not relicensed). Converts a
|
||||
rambling description of a desired outcome into one polished, copy-paste /goal
|
||||
prompt for a fresh autonomous session — the prompt is the deliverable, never the
|
||||
build. Adds over upstream: wrong-tool check, observable-done principle, six-slot
|
||||
extraction (deliverable/quantity/stakes/tools/quality/destination), per-medium
|
||||
verification defaults, six-point pre-delivery self-check, anti-pattern list +
|
||||
failure-mode catalog reference, second worked example in a non-web medium. Ships
|
||||
`goal_prompt_self_check.py` (stdlib runner for the mechanically checkable
|
||||
self-check subset; exit 0/1, `--sample`, `--output json`) and the
|
||||
`/cs:fable-goal` command. Intentionally no `agents/`/`assets/` (single reasoning
|
||||
pass; see plugin README design notes). SKILL.md is a full PASS on the
|
||||
write-a-skill 6-item checklist.
|
||||
|
||||
### Changed — counters trued up
|
||||
|
||||
skills 357 → 358 (this PR also trues up pre-existing engineering-row drift
|
||||
355 → 357), tools 602 → 603, refs 731 → 732, commands 109 → 110, plugins
|
||||
83 → 84; plus a stale "711 reference docs" claim in README line 30 fixed to 732.
|
||||
|
||||
## [Unreleased] — housekeeping: CHANGELOG backfill + per-domain counter validation
|
||||
|
||||
### Added — `derive_counters.py` per-domain table validation
|
||||
|
|
|
|||
25
CLAUDE.md
25
CLAUDE.md
|
|
@ -6,7 +6,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
|
|||
|
||||
This is a **comprehensive skills library** for Claude AI and Claude Code - reusable, production-ready skill packages that bundle domain expertise, best practices, analysis tools, and strategic frameworks. The repository provides modular skills that teams can download and use directly in their workflows.
|
||||
|
||||
**Current Scope:** 355 production-ready skills across 18 domains with 602 Python automation tools, 731 reference guides, 99 agents (cs-* + 7 personas), and 109 slash commands, distributed as 83 marketplace plugins. Headline counters are derived from the tree by `scripts/derive_counters.py` (run with `--check` to verify the docs still match). **v2.11.1 (current)** upgrades **product-team/** and **project-management/** into agent-harness domains: both prose routers rebuilt as `context: fork` orchestrators with deterministic goal routers (exit-code route/ask/refuse), a Jira MCP snapshot bridge (Kanban-Guide-2025 flow metrics + seeded Monte Carlo forecasts, verified end-to-end into velocity_analyzer), a delegation-governance loop gate (human owner / reviewer / machine-checkable acceptance / close refusal), a Torres continuous-discovery cadence tracker + Opportunity Solution Tree linter, cs-pm-orchestrator + cs-product-orchestrator agents, and /cs:pm|grill-pm|pm-loop + /cs:product|grill-product|product-loop commands — plus the public audit record `audit/pm-product-agentic-2026-07/` (AR-rubric scores for all 26 skills, research-backed improvement fields, executable verification criteria). **v2.9.0 (complete)** added the **research-ops/** top-level domain — enterprise Research Operations (orchestrator + clinical-research + research-finance + market-research + product-research), the managed counterpart to the academic research/ domain, with `context: fork` orchestration and a Matt Pocock "Forcing-question library" in every SKILL.md plus `/cs:grill-research-ops`. **v2.8.0 (complete)** added 2 new top-level domains — **business-operations/** (7 internal-ops skills: orchestrator + process-mapper + vendor-management + capacity-planner + internal-comms + knowledge-ops + procurement-optimizer) and **commercial/** (8 per-deal-economics skills: orchestrator + pricing-strategist + deal-desk + partnerships-architect + channel-economics + commercial-policy + rfp-responder + commercial-forecaster) — with orchestrator skills using `context: fork` for chaining, Matt Pocock docs-anchored "Forcing-question library" in every SKILL.md, plus `/cs:grill-bizops` and `/cs:grill-commercial`. **v2.8.2** adds a productivity-shaped `handoff` skill (sibling to engineering/handoff) inspired by Matt Pocock — first-run setup with configurable save location, redaction linter, SessionStart + SessionEnd hooks, fidelity self-check, `--refresh` flag. **v2.8.1** upgraded the engineering role-skills (senior-fullstack / senior-frontend / senior-backend) with karpathy-coder + Matt Pocock decision engines + per-role forcing questions. v2.7.3 ports `alirezarezvani/aeo-box` — AEO (Answer Engine Optimization) skill into marketing-skill/ + security-guidance PreToolUse hook into engineering/. v2.7.0 added 13 Path-B skills across 3 top-level domains (productivity, marketing, research). v2.6.0 added 4 Matt Pocock-derived productivity skills.
|
||||
**Current Scope:** 359 production-ready skills across 18 domains with 635 Python automation tools, 732 reference guides, 99 agents (cs-* + 7 personas), and 111 slash commands, distributed as 85 marketplace plugins. Headline counters are derived from the tree by `scripts/derive_counters.py` (run with `--check` to verify the docs still match). **v2.11.2 (current)** vendors **engineering/skillopt-sleep/** — started as a verbatim, byte-for-byte copy of `microsoft/SkillOpt`'s `skillopt_sleep` engine (stdlib-only, zero third-party deps) and its Claude Code plugin surface (`skills/`, `hooks/`, `commands/`, `scripts/`), then received 23 targeted patches after ten rounds of adversarial review (see `engineering/skillopt-sleep/README.md`'s numbered "Deviations from upstream" list, the authoritative source — re-apply all 23 on re-vendor). Gives a local agent a nightly "sleep cycle": read-only harvest of past Claude Code session transcripts → mine recurring tasks → replay offline on the user's own API budget → consolidate into `CLAUDE.md`/`SKILL.md` edits behind a held-out validation gate → stage for review; nothing live changes until an explicit `/skillopt-sleep adopt` (which backs up first). Default `mock` backend spends no API budget. The heavier `skillopt` *training* package (benchmark-driven, needs `numpy`/`openai`/`azure-*` + hand-labeled train/val/test data per task) was deliberately **not** vendored — it optimizes one narrow, scoreable task at a time, which doesn't fit this repo's broad domain-expertise skills or its no-ML-in-scripts/no-test-framework conventions; `skillopt_sleep` mines its "benchmark" from real usage instead, which does fit. Attribution preserved in `plugin.json` + `LICENSE` + `README.md` (MIT, © Microsoft Corporation / Yifan Yang), following the same verbatim-vendor pattern as `loop-library/`. **Unreleased (post-v2.11.1)** added **productivity/fable-goal** — converts a rambling description of a desired outcome into one polished, copy-paste `/goal` prompt for a fresh autonomous session (ported from `duncan-buildroom/freeskills`). **v2.11.1 (complete)** upgrades **product-team/** and **project-management/** into agent-harness domains: both prose routers rebuilt as `context: fork` orchestrators with deterministic goal routers (exit-code route/ask/refuse), a Jira MCP snapshot bridge (Kanban-Guide-2025 flow metrics + seeded Monte Carlo forecasts, verified end-to-end into velocity_analyzer), a delegation-governance loop gate (human owner / reviewer / machine-checkable acceptance / close refusal), a Torres continuous-discovery cadence tracker + Opportunity Solution Tree linter, cs-pm-orchestrator + cs-product-orchestrator agents, and /cs:pm|grill-pm|pm-loop + /cs:product|grill-product|product-loop commands — plus the public audit record `audit/pm-product-agentic-2026-07/` (AR-rubric scores for all 26 skills, research-backed improvement fields, executable verification criteria). **v2.9.0 (complete)** added the **research-ops/** top-level domain — enterprise Research Operations (orchestrator + clinical-research + research-finance + market-research + product-research), the managed counterpart to the academic research/ domain, with `context: fork` orchestration and a Matt Pocock "Forcing-question library" in every SKILL.md plus `/cs:grill-research-ops`. **v2.8.0 (complete)** added 2 new top-level domains — **business-operations/** (7 internal-ops skills: orchestrator + process-mapper + vendor-management + capacity-planner + internal-comms + knowledge-ops + procurement-optimizer) and **commercial/** (8 per-deal-economics skills: orchestrator + pricing-strategist + deal-desk + partnerships-architect + channel-economics + commercial-policy + rfp-responder + commercial-forecaster) — with orchestrator skills using `context: fork` for chaining, Matt Pocock docs-anchored "Forcing-question library" in every SKILL.md, plus `/cs:grill-bizops` and `/cs:grill-commercial`. **v2.8.2** adds a productivity-shaped `handoff` skill (sibling to engineering/handoff) inspired by Matt Pocock — first-run setup with configurable save location, redaction linter, SessionStart + SessionEnd hooks, fidelity self-check, `--refresh` flag. **v2.8.1** upgraded the engineering role-skills (senior-fullstack / senior-frontend / senior-backend) with karpathy-coder + Matt Pocock decision engines + per-role forcing questions. v2.7.3 ports `alirezarezvani/aeo-box` — AEO (Answer Engine Optimization) skill into marketing-skill/ + security-guidance PreToolUse hook into engineering/. v2.7.0 added 13 Path-B skills across 3 top-level domains (productivity, marketing, research). v2.6.0 added 4 Matt Pocock-derived productivity skills.
|
||||
|
||||
**Key Distinction**: This is NOT a traditional application. It's a library of skill packages meant to be extracted and deployed by users into their own Claude workflows.
|
||||
|
||||
|
|
@ -159,6 +159,25 @@ See [standards/git/git-workflow-standards.md](standards/git/git-workflow-standar
|
|||
|
||||
## Current Version
|
||||
|
||||
**Version:** v2.11.2 (skillopt-sleep — vendored nightly self-improvement plugin from microsoft/SkillOpt)
|
||||
|
||||
**v2.11.2 highlights — engineering/skillopt-sleep/:**
|
||||
|
||||
Vendors `engineering/skillopt-sleep/` — a byte-for-byte start from [microsoft/SkillOpt](https://github.com/microsoft/SkillOpt)'s `skillopt_sleep` engine (32 files, stdlib-only, zero third-party deps) and its Claude Code plugin surface (`skills/`, `hooks/`, `commands/`, `scripts/`), following the same verbatim-vendor pattern as `loop-library/`. Gives a local agent a nightly gated self-improvement cycle: read-only harvest of past Claude Code session transcripts → mine recurring tasks → replay offline on the user's own API budget → consolidate into `CLAUDE.md`/`SKILL.md` edits behind a held-out validation gate → stage for review; nothing live changes until an explicit `/skillopt-sleep adopt` (which backs up first). Default `mock` backend spends no API budget.
|
||||
|
||||
- **Deliberately not vendored:** the heavier `skillopt` *training* package (benchmark-driven, needs `numpy`/`openai`/`azure-*` + hand-labeled train/val/test data per task) — it optimizes one narrow, scoreable task at a time, which doesn't fit this repo's broad domain-expertise skills or its no-ML-in-scripts/no-test-framework conventions; `skillopt_sleep` mines its "benchmark" from real usage instead, which does fit.
|
||||
- **23 deviations from upstream (6 cosmetic, 17 safety/hardening)**, found across ten rounds of adversarial code review rather than assumed safe from the surface docs. The **numbered list in `engineering/skillopt-sleep/README.md`'s "Deviations from upstream" section is the single source of truth** — `plugin.json`'s `attribution.derivation_note` and this bullet are both summaries of it, kept in sync by hand; if any of the three ever disagree on the count again, README.md wins. Highlights: `redact_secrets()` now covers every artifact that goes live or persists — `proposed_SKILL.md`/`proposed_CLAUDE.md`, the cross-night task archive (`state.json`), `report.md`/`report.json` (previously only `diagnostics.json` was scrubbed, despite `report.md` being the file the SKILL.md's own workflow tells a human to read *first*), and — the gap that survived seven review rounds because every earlier fix was file-level — the CLI's own `cmd_run`/`cmd_harvest` console/`--json`/`--output` output, which read the same unredacted in-memory `Report`/`TaskRecord` objects and (via `scheduler.py`'s cron redirect) could leak straight into `cron.log`; the generated crontab line is fully `shlex.quote()`-d including the `extra` flags param, and `cron.log` itself is now `chmod 600` (previously uncovered by the state/staging chmod pass); `scheduler.py`'s per-project cron-line marker match is now anchored on end-of-line rather than a bare substring test, closing a real bug where scheduling/unscheduling one project could silently drop a sibling project's job whose path happened to be a prefix of it; the previously-dead `max_tokens_per_night` config key now sizes `dream_rollouts` down via the engine's own `plan_depth()` heuristic; a hardcoded internal Azure OpenAI backend (5 internal-looking endpoint hostnames + a Managed Identity client ID) was removed rather than carried forward; tool-shim names reachable via `--tasks-file` are now validated against a safe-identifier allowlist before use as a filename or shell text; `commands/skillopt-sleep.md` now tells the agent to confirm with the user before `schedule` (which installs a real crontab entry immediately, unlike every other action); every directory/file `state.py`/`staging.py` create is `chmod 0700`/`0600` rather than left at the world-readable process umask default.
|
||||
- One documented, opt-in exception to CLAUDE.md's "no LLM calls in scripts" anti-pattern (see that section) — `backend.py`'s `claude`/`codex` backends shell out to those CLIs only when a non-`mock` backend is explicitly selected.
|
||||
- Registered as its own installable marketplace plugin; **counters:** skills 358 → 359; tools 603 → 635; refs 732 (unchanged); commands 110 → 111; plugins 84 → 85 (derived via `scripts/derive_counters.py --check`).
|
||||
|
||||
---
|
||||
|
||||
**Unreleased (post-v2.11.1) — productivity/fable-goal (ramble → autonomous /goal prompt):**
|
||||
|
||||
Improved port of `duncan-buildroom/freeskills` `fable-goal` (informal "free to use and modify" grant — quoted, not relicensed; see the `attribution` block). Converts a rambling description of a desired outcome into one polished, copy-paste /goal prompt for a fresh autonomous session — the prompt is the deliverable, never the build. Adds over upstream: wrong-tool check, observable-done principle, six-slot extraction (deliverable/quantity/stakes/tools/quality/destination), per-medium verification defaults, six-point pre-delivery self-check, anti-pattern list + failure-mode catalog reference, second worked example in a non-web medium. Ships `goal_prompt_self_check.py` (stdlib runner for the mechanically checkable self-check subset — word count 150–350, goal line, autonomy directive, verification/freedom/destination language; exit 0/1, `--sample`, `--output json`), `/cs:fable-goal` command. Intentionally no `agents/`/`assets/` (single reasoning pass; see plugin README design notes). SKILL.md is a full PASS on the write-a-skill 6-item checklist. Counters: skills 357 → 358 (this PR also trues up pre-existing engineering-row drift 355 → 357); tools 602 → 603; refs 731 → 732; commands 109 → 110; plugins 83 → 84.
|
||||
|
||||
---
|
||||
|
||||
**Version:** v2.11.1 (pm/product agent-harness domains — deep audit + orchestrated loops for product-team & project-management)
|
||||
|
||||
**v2.11.1 highlights — both PM/product routers become agent harnesses:**
|
||||
|
|
@ -529,7 +548,7 @@ This repository publishes skills to **ClawHub** (clawhub.com) as the distributio
|
|||
- Creating dependencies between skills (keep each self-contained)
|
||||
- Adding complex build systems or test frameworks (maintain simplicity)
|
||||
- Generic advice (focus on specific, actionable frameworks)
|
||||
- LLM calls in scripts (defeats portability and speed)
|
||||
- LLM calls in scripts (defeats portability and speed) — **one documented, opt-in exception:** `engineering/skillopt-sleep/skillopt_sleep/backend.py`'s `claude`/`codex` backends shell out to those CLIs when a real (non-`mock`) backend is explicitly selected. This is the deployment engine for a self-improvement *loop* (harvest → replay → gate), not a stdlib analysis tool, and the default backend (`mock`) is deterministic with zero API spend. Do not cite this as precedent for adding LLM calls to an analysis/reference skill's scripts — those still must stay stdlib-only.
|
||||
- Over-documenting file structure (skills are simple by design)
|
||||
|
||||
## Working with This Repository
|
||||
|
|
@ -560,4 +579,4 @@ When I correct you, or you catch yourself making a mistake: before continuing ad
|
|||
|
||||
**Last Updated:** July 3, 2026
|
||||
**Version:** v2.11.1
|
||||
**Status:** 355 skills deployed across 18 domains, 83 marketplace plugins, docs site live (counters derived via `scripts/derive_counters.py`)
|
||||
**Status:** 358 skills deployed across 18 domains, 84 marketplace plugins, docs site live (counters derived via `scripts/derive_counters.py`)
|
||||
|
|
|
|||
20
README.md
20
README.md
|
|
@ -1,6 +1,6 @@
|
|||
# Claude Code Skills & Plugins — Agent Skills for Every Coding Tool
|
||||
|
||||
**355 production-ready Claude Code skills, plugins, and agent skills for 13 AI coding tools.**
|
||||
**359 production-ready Claude Code skills, plugins, and agent skills for 13 AI coding tools.**
|
||||
|
||||
The most comprehensive open-source library of Claude Code skills and agent plugins — also works with OpenAI Codex, Gemini CLI, Cursor, and 9 more coding agents. Reusable expertise packages covering engineering, DevOps, marketing (incl. AEO — Answer Engine Optimization for LLM citation), security (PreToolUse hooks), compliance, C-level advisory (incl. founder-mode CFO/CMO/CRO/CPO/COO/CHRO/CISO/GC/CDO/CAIO/CCO/VPE personas + 21 /cs:* slash commands), productivity (capture/email/reflect), an academic research stack (litreview/grants/dossier/patent/syllabus/pulse/notebooklm/deep-research + hybrid router), and enterprise Research Operations (clinical-research/research-finance/market-research/product-research, v2.9.0).
|
||||
|
||||
|
|
@ -10,10 +10,10 @@ The most comprehensive open-source library of Claude Code skills and agent plugi
|
|||
[^vibe]: Mistral Vibe is also **BYO-sync tier**: the repo ships a pre-generated `.vibe/skills/claude-skills/` tree, run `./scripts/vibe-install.sh` once locally to install into `~/.vibe/skills/`. Same agentskills.io SKILL.md standard — no format conversion. Docs: <https://docs.mistral.ai/mistral-vibe/agents-skills>.
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](#skills-overview)
|
||||
[](#skills-overview)
|
||||
[](#agents)
|
||||
[](#personas)
|
||||
[](#commands)
|
||||
[](#commands)
|
||||
[](https://github.com/alirezarezvani/claude-skills/stargazers)
|
||||
[](https://getskillcheck.com)
|
||||
|
||||
|
|
@ -26,10 +26,10 @@ The most comprehensive open-source library of Claude Code skills and agent plugi
|
|||
Claude Code skills (also called agent skills or coding agent plugins) are modular instruction packages that give AI coding agents domain expertise they don't have out of the box. Each skill includes:
|
||||
|
||||
- **SKILL.md** — structured instructions, workflows, and decision frameworks
|
||||
- **Python tools** — 602 CLI scripts (all stdlib-only, zero pip installs)
|
||||
- **Reference docs** — 711 templates, checklists, and domain-specific knowledge files
|
||||
- **Python tools** — 635 CLI scripts (all stdlib-only, zero pip installs)
|
||||
- **Reference docs** — 732 templates, checklists, and domain-specific knowledge files
|
||||
|
||||
**One repo, thirteen platforms.** Works natively as Claude Code plugins, Codex agent skills, Gemini CLI skills, Hermes Agent skills, Mistral Vibe skills, and converts to more tools via `scripts/convert.sh`. All 602 Python tools run anywhere Python runs.
|
||||
**One repo, thirteen platforms.** Works natively as Claude Code plugins, Codex agent skills, Gemini CLI skills, Hermes Agent skills, Mistral Vibe skills, and converts to more tools via `scripts/convert.sh`. All 635 Python tools run anywhere Python runs.
|
||||
|
||||
### Skills vs Agents vs Personas
|
||||
|
||||
|
|
@ -150,15 +150,15 @@ Run `./scripts/convert.sh --tool all` to generate tool-specific outputs locally.
|
|||
|
||||
## Skills Overview
|
||||
|
||||
**355 skills across 18 domains:**
|
||||
**359 skills across 18 domains:**
|
||||
|
||||
| Domain | Skills | Highlights | Details |
|
||||
|--------|--------|------------|---------|
|
||||
| **🔧 Engineering — Core** | 52 | Architecture, frontend, backend, fullstack, QA, DevOps, SecOps, AI/ML, data, Playwright Pro (test gen, flaky fix, migrations), self-improving agent (auto-memory curation), security suite, a11y audit, **named-persona-adversarial-review** (review via named engineering philosophies) | [engineering-team/](engineering-team/) |
|
||||
| **⚡ Engineering — POWERFUL** | 81 | Agent designer, RAG architect, database designer, CI/CD builder, security auditor, MCP builder, AgentHub, Helm charts, Terraform, self-eval, llm-wiki, tc-tracker, autoresearch-agent, **reliability portfolio** (feature-flags-architect, kubernetes-operator, chaos-engineering, slo-architect), ship-gate, security-guidance PreToolUse hook, **Matt Pocock skills** (write-a-skill, caveman, grill-me, handoff, grill-with-docs), **zero-hallucination-coder** (Discuss→Map→Decompose→Execute→Verify), **agent-harness** (goal→plan→execute→verify→close loops over any domain) | [engineering/](engineering/) |
|
||||
| **⚡ Engineering — POWERFUL** | 84 | Agent designer, RAG architect, database designer, CI/CD builder, security auditor, MCP builder, AgentHub, Helm charts, Terraform, self-eval, llm-wiki, tc-tracker, autoresearch-agent, **reliability portfolio** (feature-flags-architect, kubernetes-operator, chaos-engineering, slo-architect), ship-gate, security-guidance PreToolUse hook, **Matt Pocock skills** (write-a-skill, caveman, grill-me, handoff, grill-with-docs), **zero-hallucination-coder** (Discuss→Map→Decompose→Execute→Verify), **agent-harness** (goal→plan→execute→verify→close loops over any domain), **skillopt-sleep** (nightly gated self-evolution from real Claude Code sessions, vendored from microsoft/SkillOpt) | [engineering/](engineering/) |
|
||||
| **🎯 Product** | 17 | Product manager, agile PO, strategist, UX researcher, UI design, landing pages, SaaS scaffolder, analytics, experiment designer, discovery, roadmap communicator, code-to-prd, apple-hig-expert | [product-team/](product-team/) |
|
||||
| **📣 Marketing** | 48 | 8 pods: Content, SEO + AEO (`aeo` — E-E-A-T audit, citation tracking across 5 LLMs) + local (`local-seo-manager` — GBP/NAP/Map-Pack), CRO, Channels, Growth, Intelligence, Sales + context foundation + orchestration router | [marketing-skill/](marketing-skill/) |
|
||||
| **🚀 Productivity** | 7 | `capture` (brain-dump-to-action), `email` pair (inbox-setup + inbox-triage), `reflect` (journal), `handoff` (Matt Pocock-inspired), `andreessen` (market-first decision mode), `roast` (5-angle idea panel → GO/RESHAPE/KILL) | [productivity/](productivity/) |
|
||||
| **🚀 Productivity** | 8 | `capture` (brain-dump-to-action), `email` pair (inbox-setup + inbox-triage), `reflect` (journal), `handoff` (Matt Pocock-inspired), `andreessen` (market-first decision mode), `roast` (5-angle idea panel → GO/RESHAPE/KILL), `fable-goal` (ramble → autonomous /goal prompt) | [productivity/](productivity/) |
|
||||
| **🎨 Marketing (top-level)** | 1 | `landing` — single-file HTML landing-page generator (4 design styles, GSAP patterns, brand palette validator) | [marketing/](marketing/) |
|
||||
| **🔬 Research (academic)** | 9 | `research` orchestrator (hybrid router + fallback) + 8 specialists: `pulse`, `litreview`, `grants` (NIH), `dossier`, `patent`, `syllabus`, `notebooklm`, `deep-research` (rigor-first meta-research) | [research/](research/) |
|
||||
| **🧪 Research Operations** ✨v2.9.0 | 5 | Enterprise/cross-functional research: orchestrator + `clinical-research` (study design), `research-finance` (R&D program finance), `market-research` (sizing/survey/segmentation), `product-research` (user research) — each with onboarding + customization + opt-in autoresearch bridge | [research-ops/](research-ops/) |
|
||||
|
|
@ -354,7 +354,7 @@ Yes. Skills work natively with 13 tools: Claude Code, OpenAI Codex, Gemini CLI,
|
|||
No. We follow semantic versioning and maintain backward compatibility within patch releases. Existing script arguments, plugin source paths, and SKILL.md structures are never changed in patch versions. See the [CHANGELOG](CHANGELOG.md) for details on each release.
|
||||
|
||||
**Are the Python tools dependency-free?**
|
||||
Yes. All 602 Python CLI tools use the standard library only — zero pip installs required. Every script is verified to run with `--help`.
|
||||
Yes. All 635 Python tools use the standard library only — zero pip installs required. Every skill's CLI entry point is verified to run with `--help` (most skills ship one script per tool; a few, like the vendored `engineering/skillopt-sleep` engine, ship a multi-module package behind a single `python -m` entry point).
|
||||
|
||||
**How do I create my own Claude Code skill?**
|
||||
Each skill is a folder with a `SKILL.md` (frontmatter + instructions), optional `scripts/`, `references/`, and `assets/`. See the [Skills & Agents Factory](https://github.com/alirezarezvani/claude-code-skills-agents-factory) for a step-by-step guide.
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ This C-Level advisory skills collection provides executive leadership guidance f
|
|||
- Building organizational culture
|
||||
- Managing stakeholder relationships
|
||||
|
||||
**Learn More:** [ceo-advisor/SKILL.md](ceo-advisor/SKILL.md)
|
||||
**Learn More:** [ceo-advisor/SKILL.md](skills/ceo-advisor/SKILL.md)
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -138,7 +138,7 @@ This C-Level advisory skills collection provides executive leadership guidance f
|
|||
- Making architecture decisions
|
||||
- Measuring engineering performance
|
||||
|
||||
**Learn More:** [cto-advisor/SKILL.md](cto-advisor/SKILL.md)
|
||||
**Learn More:** [cto-advisor/SKILL.md](skills/cto-advisor/SKILL.md)
|
||||
|
||||
---
|
||||
|
||||
|
|
|
|||
22
engineering/skillopt-sleep/.claude-plugin/plugin.json
Normal file
22
engineering/skillopt-sleep/.claude-plugin/plugin.json
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
{
|
||||
"name": "skillopt-sleep",
|
||||
"description": "Nightly offline self-evolution for this repo's Claude agent: harvests past Claude Code sessions (read-only), mines recurring tasks, replays them offline on your own API budget, and consolidates what it learns into validated CLAUDE.md memory and SKILL.md skills behind a held-out validation gate, staged for review (nothing live changes without /skillopt-sleep adopt). Default backend is mock (no API spend); vendored from microsoft/SkillOpt with safety-hardening patches -- see attribution.derivation_note. Use when the user wants their agent to learn from how this repo's skills are actually used, asks about a nightly/offline sleep or dream cycle, memory/skill consolidation, or invokes /skillopt-sleep.",
|
||||
"version": "1.0.0",
|
||||
"author": {
|
||||
"name": "Alireza Rezvani",
|
||||
"url": "https://alirezarezvani.com"
|
||||
},
|
||||
"homepage": "https://github.com/alirezarezvani/claude-skills/tree/main/engineering/skillopt-sleep",
|
||||
"repository": "https://github.com/alirezarezvani/claude-skills",
|
||||
"license": "MIT",
|
||||
"skills": [
|
||||
"./skills/skillopt-sleep"
|
||||
],
|
||||
"attribution": {
|
||||
"upstream": "https://github.com/microsoft/SkillOpt",
|
||||
"upstream_path": "skillopt_sleep/ (engine) + plugins/claude-code/ (Claude Code plugin surface) + plugins/run-sleep.sh (shared launcher)",
|
||||
"upstream_author": "Yifan Yang (yifanyang@microsoft.com), Microsoft Corporation",
|
||||
"upstream_license": "MIT",
|
||||
"derivation_note": "Started as a verbatim byte-for-byte copy of the skillopt_sleep Python engine, its Claude Code plugin skill/hooks/commands, and the shared scripts/run-sleep.sh + scripts/sleep.sh launchers. 23 targeted patches (6 cosmetic, 17 safety/hardening) were made afterward across staging.py, scheduler.py, cycle.py, state.py, backend.py, __main__.py, SKILL.md, commands/skillopt-sleep.md, sleep.sh, run-sleep.sh, and install-cron.sh, found across ten rounds of adversarial code review rather than assumed safe from the surface docs -- see the numbered 'Deviations from upstream' section in README.md for the authoritative, exact list (secret redaction covering proposed_SKILL.md/proposed_CLAUDE.md/the task archive/report.md/report.json, not just diagnostics; shell-quoting the generated crontab line including the extra param; adopt() re-redacting as defense-in-depth; wiring the previously-dead max_tokens_per_night budget; a loud warning that replay_mode:'fresh' is an unimplemented no-op; removing a hardcoded internal Azure OpenAI backend with Microsoft-internal endpoint hostnames and a Managed Identity client ID; validating tool names against a safe-identifier allowlist before they're used as a shim filename or interpolated into generated shell text; a dead SKILL.md cross-reference to a non-vendored design doc, now pointed at the real upstream URL; making the schedule command's confirm-before-installing behavior explicit instead of contradicting itself; chmod 0700/0600 on state and staging directories/files; correcting stale upstream-layout path assumptions in sleep.sh/run-sleep.sh's fallback branches to match this repo's actual scripts/+skillopt_sleep/ sibling layout; routing __main__.py's cmd_run/cmd_harvest console/--json/--output output through redact_secrets, since that bypassed every file-level redaction fix above, plus chmod 0700/0600 on cron.log which those fixes never covered; anchoring scheduler.py's per-project cron-line marker match on end-of-line instead of a bare substring test, which could silently drop a sibling project's scheduled job whose path happened to be a prefix of the one being (un)scheduled; quoting install-cron.sh's printed --backend value; requiring --yes for schedule at the CLI layer (not just in the driving agent's chat-confirmation instructions), refusing non-interactively without it; passing mode=0o700 to the os.makedirs() calls that create state/staging/backup dirs to close the create-then-chmod race window) -- re-apply all of them on re-vendor, and treat any other document (including this note) that repeats the count as a summary of that list, not a second source of truth: if counts ever disagree again, README.md's numbered list wins. skillopt_sleep has zero third-party dependencies (stdlib only) -- the heavier skillopt package it derives its ideas from (numpy/openai/azure, benchmark training loops) was deliberately NOT vendored, since this repo's skills are broad domain-expertise packages without labeled benchmarks, not narrow scoreable tasks. Re-vendor by re-copying skillopt_sleep/, plugins/claude-code/{skills,hooks,commands,scripts}/, and plugins/run-sleep.sh from a fresh clone of microsoft/SkillOpt, then re-applying the README's deviation list."
|
||||
}
|
||||
}
|
||||
21
engineering/skillopt-sleep/LICENSE
Normal file
21
engineering/skillopt-sleep/LICENSE
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2026 Microsoft Corporation
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
440
engineering/skillopt-sleep/README.md
Normal file
440
engineering/skillopt-sleep/README.md
Normal file
|
|
@ -0,0 +1,440 @@
|
|||
# SkillOpt-Sleep (vendored plugin)
|
||||
|
||||
This folder started as a **verbatim copy** of the `skillopt_sleep` engine and
|
||||
the Claude Code plugin surface from
|
||||
[microsoft/SkillOpt](https://github.com/microsoft/SkillOpt)
|
||||
(`skillopt_sleep/`, `plugins/claude-code/`, `plugins/run-sleep.sh`), only
|
||||
relocated so path resolution (`CLAUDE_PLUGIN_ROOT`-relative lookups in
|
||||
`scripts/sleep.sh` / `scripts/run-sleep.sh`) resolves correctly at this
|
||||
folder's location. Licensed under the [MIT License](LICENSE) © Microsoft
|
||||
Corporation. A small number of targeted patches were made afterward to close
|
||||
gaps between this plugin's own safety claims and what the code actually did
|
||||
— see **Deviations from upstream** below. Re-apply all of these on re-vendor;
|
||||
they are not upstream yet.
|
||||
|
||||
## Deviations from upstream
|
||||
|
||||
1. **Cosmetic — `check_paths.py` wording.** `skills/skillopt-sleep/SKILL.md`'s
|
||||
frontmatter `description` said "...consolidate validated CLAUDE.md/SKILL.md
|
||||
behind a held-out gate" — the `CLAUDE.md/SKILL.md` substring reads as a
|
||||
broken relative path to this repo's `[A-Za-z0-9_\-./]+/SKILL\.md` linter
|
||||
regex. Reworded to "CLAUDE.md and SKILL.md"; no behavior or meaning
|
||||
changed.
|
||||
2. **Safety — secrets weren't redacted in the files that actually go live.**
|
||||
`staging.py`'s `redact_secrets()` was applied to `diagnostics.json` and CLI
|
||||
error logging, but **not** to `proposed_SKILL.md` / `proposed_CLAUDE.md` —
|
||||
the exact files `adopt()` copies over your live `CLAUDE.md` / managed
|
||||
`SKILL.md` (with `--auto-adopt`, with no human in the loop). Since
|
||||
`reflect()`'s prompt is built from real harvested session text, a secret
|
||||
pasted into a real debugging session could have landed in your live memory
|
||||
file unredacted, despite the "secrets are redacted from prompts" claim
|
||||
below. Fixed: `write_staging()` now runs both through `redact_secrets()`
|
||||
before writing.
|
||||
3. **Safety — the crontab line was built via unescaped f-string
|
||||
interpolation.** `scheduler.py`'s `_runner_cmd()` wrapped `project` (an
|
||||
arbitrary filesystem path) in manual `"..."` quoting, then wrote the result
|
||||
straight into your real crontab — which cron runs through `sh -c` on every
|
||||
fire. A path containing `"`, `` ` ``, `$( )`, or `;` could break out of the
|
||||
quoting and inject an arbitrary command into your crontab. Fixed: `project`,
|
||||
`logdir`, `log`, and the repo root are now `shlex.quote()`-d before
|
||||
interpolation.
|
||||
4. **Safety — `max_tokens_per_night` was a dead config key.** `config.py`
|
||||
declared it in `DEFAULTS`, and `budget.py` already had a `Budget` /
|
||||
`plan_depth` heuristic built for exactly this purpose, but nothing in the
|
||||
production `run_sleep_cycle()` path ever read it — a `--backend
|
||||
claude`/`--backend codex` night had no real ceiling on API spend. Fixed:
|
||||
`cycle.py` now starts a `Budget` right after backend construction (so
|
||||
harvest/mine spend counts too), sizes `dream_rollouts` down via
|
||||
`plan_depth()` when the remaining budget is tight, and appends a `report`
|
||||
note whenever it caps rollouts or the budget is exhausted at night's end —
|
||||
no silent truncation. This caps *rollout depth per task*, not a hard
|
||||
mid-call abort inside a single `dream_consolidate()` call; a night can
|
||||
still overshoot the cap somewhat if an individual rollout is unusually
|
||||
token-heavy. That residual gap is real and not yet closed.
|
||||
5. **Cosmetic — dead hardcoded path.** `backend.py`'s `resolve_codex_path()`
|
||||
listed `~/.nvm/versions/node/v22.22.3/bin/codex` as a candidate ahead of
|
||||
the generic "any nvm node version" scan a few lines later, which already
|
||||
covers it. Removed; no behavior change for anyone not on that exact nvm
|
||||
version, one less leftover-looking line for everyone else.
|
||||
6. **Safety — `redact_secrets` was a dead config key.** `config.py` declared
|
||||
`"redact_secrets": True` in `DEFAULTS`, but `write_staging()` called
|
||||
`redact_secrets()` unconditionally — the safe direction, but the knob had
|
||||
no effect either way. Fixed: `cycle.py` now reads
|
||||
`cfg.get("redact_secrets", True)` and threads it through `write_staging()`
|
||||
and the `diagnostics.json` fields; disabling it is honored (it's the
|
||||
user's config) but never silently — a `report` note fires whenever it's
|
||||
off.
|
||||
7. **Hardening — `adopt()` now re-redacts as defense-in-depth.** Previously
|
||||
`adopt()` copied the staged `proposed_SKILL.md`/`proposed_CLAUDE.md`
|
||||
straight to the live path with `shutil.copy2`. Since `write_staging()`
|
||||
already redacts, this was redundant for the common case, but didn't cover
|
||||
a human hand-editing the staged proposal between `stage` and `adopt` (the
|
||||
exact workflow staging exists to allow). `adopt()` now reads, re-runs
|
||||
`redact_secrets()`, and writes each file rather than a raw byte copy;
|
||||
`_backup()` of the *prior* live file is unaffected (that's a backup of
|
||||
what already existed, not the incoming content).
|
||||
8. **Safety — `scheduler.py`'s `extra` param wasn't shell-quoted.**
|
||||
`project`/`logdir`/`log`/repo-root were `shlex.quote()`-d in fix #3 above,
|
||||
but `extra` (today only ever `""` or the literal `"--auto-adopt"` from
|
||||
`__main__.py`) was appended raw. Not exploitable today since it's a
|
||||
hardcoded flag literal, but reopens the same class of bug if a future
|
||||
change lets `extra` carry anything else. Fixed: `extra` is now
|
||||
`shlex.split()` into tokens and each token `shlex.quote()`-d before
|
||||
joining, so a multi-token or attacker-influenced `extra` can't break out
|
||||
of the command the way `project` used to.
|
||||
9. **Safety — the cross-night task archive (`state.json`) was never
|
||||
redacted.** `state.py`'s `add_to_archive()` persists raw `TaskRecord`
|
||||
content (`intent` / `context_excerpt` / `attempted_solution` — real
|
||||
harvested prompt/response text) to `~/.skillopt-sleep/state.json`
|
||||
indefinitely, for `recall_k` associative recall across nights. Unlike
|
||||
`proposed_SKILL.md`/`proposed_CLAUDE.md`/`diagnostics.json` (fixes #2,
|
||||
#6), this file lives entirely outside the staging dir a user is ever told
|
||||
to review — a secret pasted into a real debugging session would have
|
||||
landed there and stayed. Fixed: `cycle.py` now redacts each task dict
|
||||
before archiving, using the same `redact_enabled` flag (and loud
|
||||
report-note-on-disable) as everything else.
|
||||
10. **Safety — `report.md` / `report.json` were never redacted.** These are
|
||||
the two artifacts a human is told to read *first* (the SKILL.md's own
|
||||
workflow says "Read the generated `report.md`... show the user the exact
|
||||
proposed edits"), yet `EditRecord.content`/`.rationale` — sourced from
|
||||
the optimizer's `reflect()` output over real failing task responses —
|
||||
were written unredacted, while the sibling `proposed_SKILL.md`/
|
||||
`proposed_CLAUDE.md`/`diagnostics.json` got fixed in earlier rounds.
|
||||
Fixed: `write_staging()` now redacts the rendered `report_md` string and
|
||||
`report.to_dict()` (via `redact_secrets`'s existing recursive dict/list/
|
||||
str handling) before writing `report.md`/`report.json`.
|
||||
11. **Safety/cosmetic — dead config knob: `replay_mode`.** `config.py`
|
||||
declared `"replay_mode": "mock" | "fresh" (worktree)`, but the only
|
||||
production code path that reads it prints a cosmetic label in the
|
||||
report — there is no worktree-replay implementation anywhere in this
|
||||
engine. Implementing real worktree isolation was judged too invasive for
|
||||
a vendored copy (a substantial new feature in code this repo doesn't
|
||||
otherwise maintain); instead, `cycle.py` now appends a loud report note
|
||||
whenever `replay_mode` is set to anything but `"mock"`, so the report
|
||||
never implies real worktree isolation that isn't happening.
|
||||
12. **Sensitive — hardcoded internal Azure OpenAI infrastructure removed.**
|
||||
`backend.py` shipped an `AzureOpenAIBackend`/`AzureResponsesBackend` pair
|
||||
with 5 internal-looking Azure endpoint hostnames and a hardcoded Managed
|
||||
Identity client ID, explicitly commented as sourced from "the intern's
|
||||
avail_api.md" — reads like leaked internal Microsoft dev-infra topology.
|
||||
This backend was already unreachable from this plugin's documented
|
||||
`mock`/`claude`/`codex`/`copilot` `--backend` choices (only settable by
|
||||
hand-editing `~/.skillopt-sleep/config.json` directly) and requires
|
||||
`azure-identity`/`openai` third-party packages this repo never claims to
|
||||
depend on. Removed entirely (classes, constants, `get_backend()`/
|
||||
`build_backend()` dispatch branches and the now-unused `azure_endpoint`
|
||||
parameter); `get_backend("azure")` now safely falls back to `MockBackend`
|
||||
instead of raising or wiring internal infra. Same class of "leftover
|
||||
internal-looking detail" as fix #5's dead `nvm` path, but materially more
|
||||
sensitive, so it was removed rather than just flagged.
|
||||
13. **Safety — unvalidated tool names reachable via `--tasks-file`.**
|
||||
`attempt_with_tools()` (all three CLI backends) used a task's tool name
|
||||
both as a shim *filename* (`os.path.join(work, tname)`) and interpolated
|
||||
unescaped into the shim's generated shell body
|
||||
(`f'echo "{tname}" >> ...'`). Tool names originate from a hand-authored
|
||||
`--tasks-file`'s `judge.checks[].arg` (`op=="tool_called"`), never
|
||||
validated as a safe identifier — a crafted name containing `../`, `"`,
|
||||
backticks, or `$( )` could traverse out of the working dir or inject a
|
||||
command into the generated shim. Not reachable via the harvest/mine path
|
||||
today (the LLM miner excludes this check type), but `--tasks-file` is a
|
||||
documented, user-facing input. Fixed: a shared `_sanitize_tool_names()`
|
||||
helper filters to a safe-identifier allowlist (`^[A-Za-z0-9_-]{1,64}$`)
|
||||
before any name is used as a filename or shell text, in all three
|
||||
backends.
|
||||
14. **Cosmetic — dead cross-reference in `SKILL.md`.**
|
||||
`skills/skillopt-sleep/SKILL.md` pointed to
|
||||
`docs/superpowers/specs/2026-06-07-skillopt-sleep-claude-code-plugin-design.md`
|
||||
"for the full design" — that path is not in this repo (see "What was and
|
||||
wasn't vendored" below; `docs/` was deliberately excluded). An agent
|
||||
following the pointer hits a missing file. Fixed: points to the real
|
||||
upstream guide URL (microsoft.github.io/SkillOpt/docs/guideline.html#sleep)
|
||||
instead, with a note on why the local path is absent.
|
||||
15. **Safety — `commands/skillopt-sleep.md` didn't signal that `schedule`
|
||||
installs immediately.** The command doc's action table listed `schedule`
|
||||
as an ordinary action alongside `status`/`dry-run`/`run` (all of which
|
||||
are safe previews or explicitly staged), while its own "Safety
|
||||
reminders" section separately said to point users at the print-only
|
||||
`install-cron.sh` instead — two different, uncoordinated stories about
|
||||
the same action. `scheduler.schedule()` writes directly to the user's
|
||||
real crontab the moment it runs, with no confirmation step in between.
|
||||
Fixed: "Steps to follow" now has an explicit step 1 telling the agent to
|
||||
confirm with the user *before* running `schedule` (what project/hour/
|
||||
minute/backend will be scheduled), with `install-cron.sh` offered as the
|
||||
print-only alternative; "Safety reminders" now says the same thing
|
||||
instead of contradicting the action table.
|
||||
16. **Hardening — state/staging directories and files had no restrictive
|
||||
permissions.** `state.json` (the cross-night task archive) and
|
||||
`.skillopt-sleep/staging/<ts>/`'s proposal/report/diagnostics files
|
||||
contain real harvested session content in plaintext, created via plain
|
||||
`os.makedirs`/`open(..., "w")` — world-readable-by-default on a typical
|
||||
multi-user Linux box (subject to the process umask). `redact_secrets()`
|
||||
scrubs known secret *patterns*, but the files still carry real task
|
||||
intents, code excerpts, and project context otherwise. Fixed: both
|
||||
`state.py` and `staging.py` now `os.chmod()` every directory they create
|
||||
to `0o700` and every file they write to `0o600` (best-effort, silently
|
||||
skipped on platforms without POSIX permission bits). Live `CLAUDE.md`/
|
||||
`SKILL.md` files themselves are intentionally left at their existing
|
||||
permissions — those are the user's own, often-committed files, not new
|
||||
output this plugin introduces.
|
||||
17. **Cosmetic — misleading redaction placeholder label.** `staging.py`'s
|
||||
`sk-[A-Za-z0-9_-]{10,}` pattern matched any `sk-`-prefixed key (OpenAI,
|
||||
Anthropic's `sk-ant-...`, and other vendors sharing the convention) but
|
||||
labeled every match `[REDACTED_OPENAI_KEY]` regardless of which vendor's
|
||||
key shape it actually was. Redaction itself was unaffected — the text
|
||||
was scrubbed either way — but the placeholder implied a narrower match
|
||||
than the pattern actually has. Fixed: relabeled to `[REDACTED_API_KEY]`.
|
||||
18. **Cosmetic/hardening — stale fallback paths in the shell launchers.**
|
||||
`scripts/sleep.sh`'s header comment and its `SKILLOPT_SLEEP_REPO`
|
||||
fallback branch referenced upstream's `<repo>/plugins/run-sleep.sh`
|
||||
layout; `scripts/run-sleep.sh`'s header comment and its
|
||||
`CLAUDE_PLUGIN_ROOT` branch likewise assumed the upstream three-level
|
||||
`<repo>/plugins/claude-code/` structure (`CLAUDE_PLUGIN_ROOT/../../
|
||||
skillopt_sleep`). In this vendored copy `scripts/` and `skillopt_sleep/`
|
||||
are siblings directly under the plugin root
|
||||
(`engineering/skillopt-sleep/`), so `CLAUDE_PLUGIN_ROOT` (when Claude
|
||||
Code sets it) points straight at a dir containing `skillopt_sleep/`, not
|
||||
two levels above one. Both scripts' primary co-located/repo-relative
|
||||
resolution branches happen to still succeed for this layout regardless
|
||||
(so this was unreachable in normal operation), but the documented
|
||||
`SKILLOPT_SLEEP_REPO` and `CLAUDE_PLUGIN_ROOT` escape hatches would have
|
||||
silently failed to resolve for anyone actually relying on them — e.g.
|
||||
after a future re-vendor that missed copying `run-sleep.sh` into
|
||||
`scripts/`. Fixed: `sleep.sh`'s explicit-env branch now checks
|
||||
`$SKILLOPT_SLEEP_REPO/scripts/run-sleep.sh`; `run-sleep.sh` now checks
|
||||
`$CLAUDE_PLUGIN_ROOT/skillopt_sleep` (this repo's actual layout) ahead of
|
||||
the upstream two-levels-up check (kept for portability if this script is
|
||||
ever reused in that shape again). Verified both fixed branches resolve
|
||||
correctly in isolation from the co-located fallback.
|
||||
19. **Safety — the CLI's own stdout/`--json`/`--output` bypassed
|
||||
`redact_secrets()` entirely.** Rounds 1-3 covered every *file*
|
||||
`write_staging()`/`state.py` write, but `__main__.py`'s `cmd_run()` reads
|
||||
the same in-memory `Report` object and prints `EditRecord.content`
|
||||
(raw, from `reflect()`'s output over real task responses) directly to
|
||||
the console, and `_report_payload()` serializes it unredacted for
|
||||
`--json` — `write_staging()`'s redaction runs on a *copy*
|
||||
(`redact_secrets(report.to_dict())`) used only for the on-disk JSON, it
|
||||
never touches `report.edits` itself. Concretely: `scheduler.py`'s cron
|
||||
entry redirects `run`'s stdout/stderr straight into
|
||||
`<project>/.skillopt-sleep/cron.log` — a secret that leaked into a
|
||||
proposed edit's content would land there in plaintext on every scheduled
|
||||
night, in a file that (unlike `state.json`/staged files) also had no
|
||||
`chmod` protection (see the second fix below). `cmd_harvest()`'s debug
|
||||
output (`--json`, `--output <file>`, and the plain-text loop) has the
|
||||
same shape: it prints raw mined `TaskRecord.intent` text so a human can
|
||||
review it before setting `"reviewed": true` on a `--tasks-file`, which
|
||||
means redacting it doesn't reduce what's reviewable (only secret-shaped
|
||||
substrings are stripped) while closing the same leak path. Fixed:
|
||||
`_report_payload()` and `cmd_run()`'s plain-text edit printing, and
|
||||
`cmd_harvest()`'s payload (covering its `--output` file, `--json`
|
||||
stdout, and plain-text loop uniformly), all now run through
|
||||
`redact_secrets()`, gated on the same `redact_secrets` config flag as
|
||||
everywhere else. Also hardened `scheduler.py`'s generated cron line to
|
||||
`chmod 700` the `.skillopt-sleep` log dir and `chmod 600` `cron.log`
|
||||
itself (best-effort, `2>/dev/null`) before each run appends to it —
|
||||
that file was never covered by the state/staging chmod pass in fix #16.
|
||||
Verified: a synthetic secret seeded into a task's intent no longer
|
||||
appears in `cmd_run`'s `--json` payload or plain-text edit output, or in
|
||||
`cmd_harvest`'s redacted payload; executing the actual generated cron
|
||||
line end-to-end produces a `0700` log dir and `0600` log file on disk.
|
||||
20. **Bug — `scheduler.py`'s per-project marker match used unanchored
|
||||
substring comparison.** `schedule()`/`unschedule()` both located "this
|
||||
project's" managed cron line via `marker not in ln`, a bare substring
|
||||
test, not an exact-match or delimiter-anchored check. Failure scenario:
|
||||
two projects scheduled where one path is a literal prefix of the other
|
||||
(e.g. `/home/user/app` and `/home/user/app-v2`) — `_project_marker`
|
||||
produces `# project=/home/user/app`, which is itself a substring of
|
||||
`# project=/home/user/app-v2`'s line. Running `schedule()` or
|
||||
`unschedule()` for `/home/user/app` would silently drop
|
||||
`/home/user/app-v2`'s cron entry too, with no error or warning — the
|
||||
user's other project's nightly job just disappears. `harvest.py`'s
|
||||
`_project_matches()` (added in this same PR) already gets this right a
|
||||
few hundred lines away (`a == b or a.startswith(b + os.sep) or
|
||||
b.startswith(a + os.sep)`); `scheduler.py`'s marker matching didn't
|
||||
follow the same discipline. Fixed: added `_line_matches_project()`,
|
||||
which anchors on `ln.rstrip().endswith(marker)` since the marker is
|
||||
always the last token of a generated line (see `schedule()`'s
|
||||
`cron_line` construction) — used at both call sites. Verified two ways:
|
||||
a standalone reproduction confirmed the bug before the fix and its
|
||||
absence after, and a full `schedule()`/`unschedule()` round-trip through
|
||||
the actual public API (with `crontab -l`/`crontab -` swapped for an
|
||||
in-memory fake) confirmed scheduling both `/home/user/app` and
|
||||
`/home/user/app-v2`, then unscheduling only `app`, correctly leaves
|
||||
`app-v2`'s line intact.
|
||||
21. **Cosmetic — `install-cron.sh`'s printed `--backend` value was
|
||||
unquoted.** `scheduler.py::_runner_cmd` quotes every interpolated value
|
||||
with `shlex.quote()`, but the standalone `install-cron.sh` script
|
||||
(which only *prints* a crontab line for the user to copy into `crontab
|
||||
-e` — nothing is executed automatically) interpolated `--backend
|
||||
${BACKEND}` unquoted in its heredoc, next to otherwise-quoted
|
||||
`${RUNNER}`/`${PROJECT}`. Low risk since `BACKEND` is normally one of a
|
||||
fixed small set of values and the script never executes anything
|
||||
itself, but inconsistent with the quoting discipline applied everywhere
|
||||
else. Fixed: quoted as `"${BACKEND}"`.
|
||||
22. **Hardening — `schedule` had no confirmation gate at the CLI layer.**
|
||||
The deviation #15 "confirm with the user before `schedule`" safeguard lived
|
||||
only in `commands/skillopt-sleep.md`'s agent-facing instructions —
|
||||
`cmd_schedule()` itself called `scheduler.schedule()` directly and
|
||||
installed a real crontab entry immediately. That's fine for the
|
||||
documented Claude Code agent workflow (which confirms in chat first),
|
||||
but anyone invoking `python -m skillopt_sleep schedule` directly
|
||||
bypassed it entirely, with no gate in the CLI itself. Fixed: `schedule`
|
||||
now requires `--yes`; without it, an interactive terminal gets a
|
||||
`[y/N]` prompt and a non-interactive one (no TTY) refuses outright with
|
||||
an exit code (2) pointing at `--yes`. `commands/skillopt-sleep.md`
|
||||
updated so the driving agent passes `--yes` once *it* has confirmed
|
||||
with the user in chat — that chat confirmation is what `--yes` records,
|
||||
not a redundant re-prompt (which would hang forever with no TTY to
|
||||
answer from inside a non-interactive Bash tool call anyway). Verified:
|
||||
non-interactive `schedule` without `--yes` refuses with exit 2; with
|
||||
`--yes` it proceeds to the same `scheduler.schedule()` call as before.
|
||||
23. **Hardening — mkdir-then-chmod wasn't atomic.** `write_staging()`
|
||||
(`staging.py`) and `SleepState.save()` (`state.py`) called
|
||||
`os.makedirs(path, exist_ok=True)` and only `chmod`'d afterward,
|
||||
leaving a brief window where a freshly-created sensitive directory sat
|
||||
at the process's default umask — exactly what this plugin's chmod
|
||||
hardening (deviation #16) exists to close. Fixed: the `os.makedirs()`
|
||||
calls that create the state dir, the per-run staging leaf dir, and the
|
||||
adopt-time backup dir now pass `mode=0o700` directly, closing the
|
||||
window for the common first-creation case. The existing post-creation
|
||||
`chmod` calls are kept, not removed — `mode=` only governs the leaf
|
||||
directory `mkdir()` itself creates (intermediate parent directories
|
||||
still fall back to the umask default) and is itself still subject to
|
||||
umask, so it narrows the window rather than eliminating every case
|
||||
(e.g. a directory that already existed from before this fix, or an
|
||||
intermediate parent). The equivalent race for individual *files*
|
||||
(`open(path, "w")` then `chmod` after) is a smaller, harder-to-close
|
||||
window — closing it fully would mean rewriting every file-write call
|
||||
site to use `os.open()` with an explicit mode instead of the builtin
|
||||
`open()`, which felt like a larger rewrite than this specific,
|
||||
low-severity (requires a local attacker with precise timing) finding
|
||||
warranted; left as a known, narrower residual gap rather than silently
|
||||
claimed as fully closed.
|
||||
|
||||
## What this plugin is
|
||||
|
||||
SkillOpt-Sleep gives a local Claude Code agent a nightly **sleep cycle**: it
|
||||
reviews real past sessions in this repo, replays recurring tasks offline on
|
||||
your own API budget, and consolidates what it learns into this repo's
|
||||
`CLAUDE.md` memory and `SKILL.md` skills — but **only** through a held-out
|
||||
validation gate, and **only** after you explicitly adopt the staged proposal.
|
||||
|
||||
It is the deployment-time companion to the (not vendored) `skillopt` training
|
||||
package: SkillOpt trains a skill offline against a labeled benchmark;
|
||||
SkillOpt-Sleep applies the same bounded-edit + held-out-gate discipline to
|
||||
*actual usage of this repo* instead, so it needs no benchmark dataset.
|
||||
|
||||
```
|
||||
harvest ~/.claude transcripts (read-only)
|
||||
→ mine recurring tasks
|
||||
→ replay offline
|
||||
→ consolidate (reflect → bounded edit → GATE)
|
||||
→ stage proposal (nothing live changes)
|
||||
→ you review and run "adopt" (backs up first)
|
||||
```
|
||||
|
||||
## Why this is a fit for a skills library with no test harness
|
||||
|
||||
This repo's [CLAUDE.md](../../CLAUDE.md) intentionally has no build system or
|
||||
test framework, and skill `scripts/` are stdlib-only with no LLM calls, so the
|
||||
full `microsoft/SkillOpt` training package (benchmark-driven, requires
|
||||
labeled train/val/test data per task, needs `numpy`/`openai`/`azure-*`) was
|
||||
**not** vendored — there is no natural ground-truth benchmark for something
|
||||
like `finance/dcf-valuation` or `c-level-advisor/vpe-advisor`.
|
||||
|
||||
`skillopt_sleep`, by contrast, has **zero third-party dependencies** (stdlib
|
||||
only), its default `mock` backend spends no API budget, and it mines its
|
||||
"benchmark" from how the skills in *this* repo actually get used in real
|
||||
sessions rather than a pre-labeled dataset. That matches the repo's
|
||||
deterministic-first, portable-first philosophy far better than the training
|
||||
package does.
|
||||
|
||||
## Use in this repo
|
||||
|
||||
```bash
|
||||
# from the repo root:
|
||||
engineering/skillopt-sleep/scripts/sleep.sh status # what's happened (read-only)
|
||||
engineering/skillopt-sleep/scripts/sleep.sh dry-run --project "$(pwd)" # safe preview, stages nothing
|
||||
engineering/skillopt-sleep/scripts/sleep.sh run --project "$(pwd)" # full cycle, stages a proposal
|
||||
engineering/skillopt-sleep/scripts/sleep.sh adopt --project "$(pwd)" # apply staged proposal (backs up first)
|
||||
```
|
||||
|
||||
Or, once the plugin is installed via Claude Code's plugin marketplace, use
|
||||
the bundled `/skillopt-sleep [run|dry-run|status|adopt|harvest|schedule|unschedule]`
|
||||
slash command (see `commands/skillopt-sleep.md` and `skills/skillopt-sleep/SKILL.md`).
|
||||
|
||||
Default backend is `mock` (deterministic, **no API spend** — safe to try
|
||||
immediately). Add `--backend claude` to spend real budget replaying this
|
||||
repo's own recurring tasks and get genuine lift on `CLAUDE.md` / a target
|
||||
`SKILL.md`.
|
||||
|
||||
## Safety model
|
||||
|
||||
- Harvest is **read-only** over `~/.claude` session transcripts.
|
||||
- Edits are proposed, gated against a held-out replay slice, and **staged**
|
||||
under `.skillopt-sleep/staging/<date>/` — nothing live is touched.
|
||||
- `adopt` is explicit and backs up the prior file first (unless you opt into
|
||||
`--auto-adopt`).
|
||||
- `max_tasks_per_night` is a hard cap (mining stops there). `max_tokens_per_night`
|
||||
sizes `dream_rollouts` down via `plan_depth()` and is reported when hit, but
|
||||
is not a hard mid-call abort — see deviation #4 above for the exact scope.
|
||||
- Secrets (API keys, bearer tokens, private-key blocks) are redacted before
|
||||
anything is written to the staging dir, including `proposed_SKILL.md` /
|
||||
`proposed_CLAUDE.md` (deviation #2 above) — not just diagnostics — and
|
||||
re-redacted again at `adopt()` time as defense-in-depth against a
|
||||
hand-edited staged proposal (deviation #7). Disabling this via
|
||||
`redact_secrets: false` is honored but never silent — it logs a report
|
||||
note (deviation #6). The same flag now also covers the cross-night task
|
||||
archive (`state.json`, deviation #9), `report.md`/`report.json`
|
||||
(deviation #10) — the two files a human is actually told to read first —
|
||||
and every CLI console/`--json`/`--output` code path (`cmd_run`,
|
||||
`cmd_harvest`), not just what gets written to disk (deviation #19).
|
||||
`cron.log` (the CLI's redirected stdout/stderr) is now `chmod 600` too,
|
||||
matching state/staging (deviation #19).
|
||||
- The generated crontab line, including the `extra` flags parameter, is
|
||||
fully `shlex.quote()`-d, not just the path arguments (deviations #3, #8,
|
||||
#21). `schedule`/`unschedule` locate a project's own line via an anchored
|
||||
end-of-line match, not a bare substring test, so scheduling/unscheduling
|
||||
one project can't silently drop a sibling project whose path happens to
|
||||
be a prefix of it (deviation #20).
|
||||
- `replay_mode: "fresh"` (worktree replay) is not implemented — every replay
|
||||
runs as `"mock"` regardless, and the report says so explicitly rather than
|
||||
implying isolation that isn't happening (deviation #11).
|
||||
- Tool names reachable via a hand-authored `--tasks-file` are validated
|
||||
against a safe-identifier allowlist before being used as a shim filename or
|
||||
interpolated into generated shell text (deviation #13).
|
||||
- Only `mock`/`claude`/`codex`/`copilot` backends are supported — a
|
||||
Microsoft-internal Azure OpenAI backend was removed rather than carried
|
||||
forward (deviation #12).
|
||||
- `schedule` installs a real crontab entry **immediately** — unlike every
|
||||
other action, it is not a preview or a staged proposal. An agent driving
|
||||
`/skillopt-sleep schedule` must confirm with the user first (deviation
|
||||
#15); `install-cron.sh` remains available as a print-only alternative.
|
||||
- Every directory `state.py`/`staging.py` create is `chmod 0700` and every
|
||||
file they write is `chmod 0600` (best-effort), so `state.json` and staged
|
||||
proposals/reports/diagnostics aren't left at the world-readable process
|
||||
umask default on a shared machine (deviation #16).
|
||||
|
||||
## What was and wasn't vendored
|
||||
|
||||
| Vendored | Not vendored |
|
||||
|---|---|
|
||||
| `skillopt_sleep/` engine (stdlib-only) | `skillopt/` training package (needs `numpy`/`openai`/`azure-*` + labeled benchmarks) |
|
||||
| `plugins/claude-code/skills\|hooks\|commands\|scripts/` | `plugins/codex/`, `plugins/copilot/`, `plugins/devin/`, `plugins/openclaw/` (other-agent plugin variants) |
|
||||
| `plugins/run-sleep.sh` shared launcher | `skillopt_webui/` (optional Gradio dashboard) |
|
||||
| `LICENSE` | `docs/`, `ckpt/`, `data/`, `index.html` (training-package docs/site/checkpoints) |
|
||||
| | `AzureOpenAIBackend`/`AzureResponsesBackend` from `backend.py` — removed post-vendor, see deviation #12 (Microsoft-internal endpoints/client ID, unreachable from this plugin's supported `--backend` choices, needs deps not vendored here) |
|
||||
|
||||
## Updating
|
||||
|
||||
Re-vendor from upstream when the plugin changes:
|
||||
|
||||
```bash
|
||||
git clone --depth 1 https://github.com/microsoft/SkillOpt.git /tmp/skillopt-upstream
|
||||
cp -r /tmp/skillopt-upstream/skillopt_sleep engineering/skillopt-sleep/skillopt_sleep
|
||||
cp -r /tmp/skillopt-upstream/plugins/claude-code/skills/skillopt-sleep engineering/skillopt-sleep/skills/skillopt-sleep
|
||||
cp -r /tmp/skillopt-upstream/plugins/claude-code/hooks engineering/skillopt-sleep/hooks
|
||||
cp -r /tmp/skillopt-upstream/plugins/claude-code/commands engineering/skillopt-sleep/commands
|
||||
cp /tmp/skillopt-upstream/plugins/claude-code/scripts/sleep.sh /tmp/skillopt-upstream/plugins/claude-code/scripts/install-cron.sh engineering/skillopt-sleep/scripts/
|
||||
cp /tmp/skillopt-upstream/plugins/run-sleep.sh engineering/skillopt-sleep/scripts/run-sleep.sh
|
||||
cp /tmp/skillopt-upstream/LICENSE engineering/skillopt-sleep/LICENSE
|
||||
```
|
||||
82
engineering/skillopt-sleep/commands/skillopt-sleep.md
Normal file
82
engineering/skillopt-sleep/commands/skillopt-sleep.md
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
---
|
||||
description: Run or manage the SkillOpt-Sleep self-evolution cycle (review past sessions, replay tasks offline, consolidate validated memory + skills; can also schedule nightly runs)
|
||||
argument-hint: "[run | dry-run | status | adopt | harvest | schedule | unschedule] (default: status)"
|
||||
allowed-tools: Bash, Read
|
||||
---
|
||||
|
||||
# /skillopt-sleep — SkillOpt-Sleep nightly self-evolution
|
||||
|
||||
You are driving **SkillOpt-Sleep**: a tool that lets this user's Claude agent
|
||||
improve offline by reviewing past sessions, replaying recurring tasks, and
|
||||
consolidating what it learns into **validated** memory (`CLAUDE.md`) and skills
|
||||
(`SKILL.md`). It is gated like SkillOpt: a change is kept only if it improves a
|
||||
held-out replay score, and nothing live is modified until the user adopts it.
|
||||
|
||||
## Requested action: $ARGUMENTS
|
||||
|
||||
(If `$ARGUMENTS` is empty, treat it as `status`.)
|
||||
|
||||
## How to run it
|
||||
|
||||
The engine is the `skillopt_sleep` Python package in this repo. Use the
|
||||
**plugin's bundled runner** so the right interpreter and repo are on the path:
|
||||
|
||||
```bash
|
||||
"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" <action> --project "$(pwd)" --scope invoked
|
||||
```
|
||||
|
||||
`<action>` is one of:
|
||||
|
||||
| action | what it does |
|
||||
|--------------|--------------|
|
||||
| `status` | show how many nights have run + the latest staged proposal (READ-ONLY) |
|
||||
| `dry-run` | harvest → mine → replay → report, but **stage nothing** (safe preview) |
|
||||
| `run` | full cycle: also **stage** a reviewed proposal (still does NOT touch live files) |
|
||||
| `adopt` | apply the latest staged proposal to live `CLAUDE.md` / `SKILL.md` (backs up first) |
|
||||
| `harvest` | debug: print the recurring tasks mined from recent sessions |
|
||||
| `schedule` | install a nightly cron entry for this project (`--hour --minute`, off-:00 by default) |
|
||||
| `unschedule` | remove the nightly cron entry (`--all` to remove every managed entry) |
|
||||
|
||||
Default backend is `mock` (deterministic, no API spend). To use real budget for
|
||||
genuine improvement, add `--backend claude` or `--backend codex`. To steer what
|
||||
the optimizer writes, add `--preferences "<your house rules>"`.
|
||||
|
||||
## Steps to follow
|
||||
|
||||
1. **For `schedule`:** confirm with the user *before* running it. Unlike
|
||||
every other action, `schedule` writes directly to the user's real
|
||||
crontab the moment it runs (via `scheduler.schedule()` → `crontab -`) —
|
||||
it is not a preview. Tell them what will be scheduled (project, hour,
|
||||
minute, backend) and get an explicit go-ahead first. If they'd rather
|
||||
review the exact line before anything is installed, offer
|
||||
`${CLAUDE_PLUGIN_ROOT}/scripts/install-cron.sh` instead (prints the line;
|
||||
installs nothing). Once they've confirmed, add `--yes` to the `schedule`
|
||||
invocation in step 2 — the CLI itself refuses to install non-interactively
|
||||
without it (defense-in-depth for anyone running the CLI directly, outside
|
||||
this chat-confirmed flow); `--yes` is how you record that the confirmation
|
||||
above already happened.
|
||||
2. **Run the requested action** via the bundled runner above. Capture stdout.
|
||||
3. **For `run` / `dry-run`:** after it completes, `Read` the generated
|
||||
`report.md` in the staging dir it prints, and show the user:
|
||||
- held-out score: baseline → candidate (the proof it helped)
|
||||
- the gate decision (accept/reject) and the exact edits it proposes
|
||||
- where the proposal is staged
|
||||
4. **For `run` that produced an accepted proposal:** tell the user the diff is
|
||||
staged and that **nothing live changed yet**. Offer to run `/skillopt-sleep adopt`.
|
||||
5. **For `adopt`:** confirm which live files were updated and that backups were
|
||||
written under the staging dir's `backup/`.
|
||||
6. **Never** edit `CLAUDE.md` or `SKILL.md` yourself — only the `adopt` action
|
||||
does that, with a backup. Respect the review gate.
|
||||
|
||||
## Safety reminders
|
||||
|
||||
- Harvest is **read-only** over `~/.claude`. Replay in `mock` mode runs no
|
||||
shell side effects.
|
||||
- The cycle stages proposals; the user is in control of adoption.
|
||||
- `schedule` installs a real crontab entry immediately — it is not a preview,
|
||||
unlike `run`/`dry-run`. Always confirm with the user first (see Steps to
|
||||
follow, step 1), then pass `--yes`. Without `--yes`, the CLI itself refuses
|
||||
to install non-interactively — that's a backstop for direct CLI use, not a
|
||||
substitute for the chat confirmation above. `${CLAUDE_PLUGIN_ROOT}/scripts/install-cron.sh`
|
||||
remains available as a print-only alternative for a user who wants to inspect
|
||||
or hand-edit the line before installing anything.
|
||||
16
engineering/skillopt-sleep/hooks/hooks.json
Normal file
16
engineering/skillopt-sleep/hooks/hooks.json
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
{
|
||||
"hooks": {
|
||||
"SessionEnd": [
|
||||
{
|
||||
"matcher": "*",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "\"${CLAUDE_PLUGIN_ROOT}/hooks/on-session-end.sh\"",
|
||||
"async": true
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
18
engineering/skillopt-sleep/hooks/on-session-end.sh
Executable file
18
engineering/skillopt-sleep/hooks/on-session-end.sh
Executable file
|
|
@ -0,0 +1,18 @@
|
|||
#!/usr/bin/env bash
|
||||
# SkillOpt-Sleep SessionEnd hook (async, best-effort, NON-BLOCKING).
|
||||
#
|
||||
# This does NOT run the optimizer. It only appends a tiny marker so the next
|
||||
# nightly cycle knows there is fresh activity to harvest, and (optionally)
|
||||
# nudges the user once that a sleep cycle is available. It must never fail the
|
||||
# session or spend API budget.
|
||||
set -uo pipefail
|
||||
|
||||
PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
||||
STATE_DIR="${HOME}/.skillopt-sleep"
|
||||
mkdir -p "$STATE_DIR" 2>/dev/null || exit 0
|
||||
|
||||
# Record that a session just ended (cheap; used for "is there new data?").
|
||||
printf '%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${PWD}" \
|
||||
>> "$STATE_DIR/session-end.log" 2>/dev/null || true
|
||||
|
||||
exit 0
|
||||
29
engineering/skillopt-sleep/scripts/install-cron.sh
Executable file
29
engineering/skillopt-sleep/scripts/install-cron.sh
Executable file
|
|
@ -0,0 +1,29 @@
|
|||
#!/usr/bin/env bash
|
||||
# Print (does NOT install) a crontab line that runs SkillOpt-Sleep nightly.
|
||||
# The user copies the line into `crontab -e` if they want it.
|
||||
set -euo pipefail
|
||||
|
||||
PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
||||
RUNNER="$PLUGIN_ROOT/scripts/sleep.sh"
|
||||
PROJECT="${1:-$(pwd)}"
|
||||
BACKEND="${2:-mock}"
|
||||
|
||||
# 3:17am local — deliberately off the :00 mark so many users don't all hit the
|
||||
# API at once (and we leave room for jitter).
|
||||
MIN=17
|
||||
HOUR=3
|
||||
|
||||
cat <<EOF
|
||||
# ── SkillOpt-Sleep nightly cycle ────────────────────────────────────────────
|
||||
# Review past sessions, replay tasks, stage validated memory/skill updates.
|
||||
# Runs at ${HOUR}:$(printf '%02d' $MIN) local every day. Output goes to the project's
|
||||
# .skillopt-sleep/ dir; nothing live is changed until you run '/skillopt-sleep adopt'
|
||||
# (unless you pass --auto-adopt below).
|
||||
#
|
||||
# Copy the next line into 'crontab -e':
|
||||
${MIN} ${HOUR} * * * "${RUNNER}" run --project "${PROJECT}" --scope invoked --backend "${BACKEND}" >> "${PROJECT}/.skillopt-sleep/cron.log" 2>&1
|
||||
#
|
||||
# For fully-autonomous adoption (power users), append: --auto-adopt
|
||||
# To spend real API budget for genuine lift, set BACKEND=anthropic above.
|
||||
# ────────────────────────────────────────────────────────────────────────────
|
||||
EOF
|
||||
59
engineering/skillopt-sleep/scripts/run-sleep.sh
Executable file
59
engineering/skillopt-sleep/scripts/run-sleep.sh
Executable file
|
|
@ -0,0 +1,59 @@
|
|||
#!/usr/bin/env bash
|
||||
# SkillOpt-Sleep shared runner — used by all platform plugins (Claude Code,
|
||||
# Codex, Copilot). Resolves the repo root (which contains the skillopt_sleep
|
||||
# package), picks a Python >= 3.10, and execs the engine CLI.
|
||||
#
|
||||
# Usage: run-sleep.sh <run|dry-run|status|adopt|harvest|...> [args...]
|
||||
set -euo pipefail
|
||||
|
||||
# In this vendored copy this script lives at
|
||||
# <plugin-root>/scripts/run-sleep.sh (engineering/skillopt-sleep/scripts/ in
|
||||
# alirezarezvani/claude-skills), so the plugin root (which holds
|
||||
# skillopt_sleep/) is one level up — that's what the first branch below
|
||||
# checks. CLAUDE_PLUGIN_ROOT (set by Claude Code when it invokes a plugin
|
||||
# script) points directly at that same plugin root in this layout, so
|
||||
# skillopt_sleep/ sits directly inside it, not two levels up as it would in
|
||||
# upstream's <repo>/plugins/claude-code/ layout — both are checked so this
|
||||
# script stays portable if it's ever reused in that shape again.
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
if [ -d "$SCRIPT_DIR/../skillopt_sleep" ]; then
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
elif [ -n "${CLAUDE_PLUGIN_ROOT:-}" ] && [ -d "$CLAUDE_PLUGIN_ROOT/skillopt_sleep" ]; then
|
||||
REPO_ROOT="$CLAUDE_PLUGIN_ROOT"
|
||||
elif [ -n "${CLAUDE_PLUGIN_ROOT:-}" ] && [ -d "$CLAUDE_PLUGIN_ROOT/../../skillopt_sleep" ]; then
|
||||
REPO_ROOT="$(cd "$CLAUDE_PLUGIN_ROOT/../.." && pwd)"
|
||||
elif [ -n "${SKILLOPT_SLEEP_REPO:-}" ] && [ -d "$SKILLOPT_SLEEP_REPO/skillopt_sleep" ]; then
|
||||
REPO_ROOT="$SKILLOPT_SLEEP_REPO"
|
||||
else
|
||||
# last resort: search upward from CWD
|
||||
d="$PWD"
|
||||
while [ "$d" != "/" ]; do
|
||||
[ -d "$d/skillopt_sleep" ] && { REPO_ROOT="$d"; break; }
|
||||
d="$(dirname "$d")"
|
||||
done
|
||||
fi
|
||||
if [ -z "${REPO_ROOT:-}" ]; then
|
||||
echo "[sleep] ERROR: could not locate the skillopt_sleep package. Set SKILLOPT_SLEEP_REPO to the repo root." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PY=""
|
||||
# Allow explicit Python override (useful on macOS with old system Python).
|
||||
if [ -n "${SKILLOPT_SLEEP_PYTHON:-}" ]; then
|
||||
PY="$SKILLOPT_SLEEP_PYTHON"
|
||||
else
|
||||
for cand in python3.12 python3.11 python3.10 python3; do
|
||||
if command -v "$cand" >/dev/null 2>&1; then
|
||||
ver="$("$cand" -c 'import sys; print("%d%d" % sys.version_info[:2])' 2>/dev/null || echo 0)"
|
||||
if [ "${ver:-0}" -ge 310 ]; then PY="$cand"; break; fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ -z "$PY" ]; then
|
||||
echo "[sleep] ERROR: need Python >= 3.10 (found none)." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$#" -eq 0 ]; then set -- status; fi
|
||||
cd "$REPO_ROOT"
|
||||
exec "$PY" -m skillopt_sleep "$@"
|
||||
37
engineering/skillopt-sleep/scripts/sleep.sh
Executable file
37
engineering/skillopt-sleep/scripts/sleep.sh
Executable file
|
|
@ -0,0 +1,37 @@
|
|||
#!/usr/bin/env bash
|
||||
# Claude Code plugin runner — thin wrapper over the shared runner so all
|
||||
# platform plugins share one engine launcher.
|
||||
#
|
||||
# In this vendored copy, sleep.sh/run-sleep.sh/install-cron.sh live together
|
||||
# in <plugin-root>/scripts/, and the engine package lives at
|
||||
# <plugin-root>/skillopt_sleep/ (i.e. engineering/skillopt-sleep/ in
|
||||
# alirezarezvani/claude-skills). After marketplace install the plugin is
|
||||
# isolated in a cache directory, but that same relative layout is preserved,
|
||||
# so the repo-relative path still resolves. We try four locations:
|
||||
# 1. Co-located run-sleep.sh (bundled copy — works in marketplace cache)
|
||||
# 2. Repo-relative ../../run-sleep.sh (dev checkout; harmless no-op here
|
||||
# since (1) always resolves first for this vendored layout)
|
||||
# 3. CLAUDE_PLUGIN_ROOT/../run-sleep.sh (plugin env variable; also a no-op
|
||||
# here for the same reason)
|
||||
# 4. SKILLOPT_SLEEP_REPO/scripts/run-sleep.sh (explicit env, pointed at
|
||||
# this plugin's root — the escape hatch if (1)-(3) somehow all miss)
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
SHARED=""
|
||||
if [ -f "$HERE/run-sleep.sh" ]; then
|
||||
SHARED="$HERE/run-sleep.sh"
|
||||
elif [ -f "$(cd "$HERE/../.." 2>/dev/null && pwd)/run-sleep.sh" ]; then
|
||||
SHARED="$(cd "$HERE/../.." && pwd)/run-sleep.sh"
|
||||
elif [ -n "${CLAUDE_PLUGIN_ROOT:-}" ] && [ -f "$(cd "$CLAUDE_PLUGIN_ROOT/.." 2>/dev/null && pwd)/run-sleep.sh" ]; then
|
||||
SHARED="$(cd "$CLAUDE_PLUGIN_ROOT/.." && pwd)/run-sleep.sh"
|
||||
elif [ -n "${SKILLOPT_SLEEP_REPO:-}" ] && [ -f "$SKILLOPT_SLEEP_REPO/scripts/run-sleep.sh" ]; then
|
||||
SHARED="$SKILLOPT_SLEEP_REPO/scripts/run-sleep.sh"
|
||||
fi
|
||||
|
||||
if [ -z "$SHARED" ]; then
|
||||
echo "[sleep] ERROR: cannot locate run-sleep.sh." >&2
|
||||
echo "[sleep] Set SKILLOPT_SLEEP_REPO to the SkillOpt repo root, or pip install skillopt." >&2
|
||||
exit 1
|
||||
fi
|
||||
exec bash "$SHARED" "$@"
|
||||
20
engineering/skillopt-sleep/skillopt_sleep/__init__.py
Normal file
20
engineering/skillopt-sleep/skillopt_sleep/__init__.py
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
"""SkillOpt-Sleep — nightly offline self-evolution for a local Claude agent.
|
||||
|
||||
A Claude Code plugin engine that gives a user's agent a "sleep cycle":
|
||||
harvest the day's real session transcripts, mine recurring tasks, replay
|
||||
them offline, and consolidate short-term experience into long-term memory
|
||||
(CLAUDE.md) and skills (SKILL.md) behind a SkillOpt validation gate.
|
||||
|
||||
Synthesizes three ideas:
|
||||
* SkillOpt — validation-gated bounded text optimization (this repo)
|
||||
* Dreams — offline memory consolidation, input never mutated
|
||||
* Sleep — short-term experience -> long-term competence, offline
|
||||
|
||||
Public entry points:
|
||||
* skillopt_sleep.cli — `python -m skillopt_sleep ...`
|
||||
* skillopt_sleep.cycle.run_sleep_cycle(...)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["__version__"]
|
||||
__version__ = "0.2.0"
|
||||
395
engineering/skillopt-sleep/skillopt_sleep/__main__.py
Normal file
395
engineering/skillopt-sleep/skillopt_sleep/__main__.py
Normal file
|
|
@ -0,0 +1,395 @@
|
|||
"""SkillOpt-Sleep — command-line interface.
|
||||
|
||||
python -m skillopt_sleep run # full cycle: harvest->mine->replay->gate->stage
|
||||
python -m skillopt_sleep dry-run # same but report only, no staging/adopt
|
||||
python -m skillopt_sleep status # show state + latest staged proposal
|
||||
python -m skillopt_sleep adopt # apply the latest staged proposal (with backup)
|
||||
python -m skillopt_sleep harvest # just print what would be mined (debug)
|
||||
|
||||
Common flags:
|
||||
--project PATH project to evolve (default: cwd)
|
||||
--scope all|invoked harvest scope (default: invoked)
|
||||
--max-sessions N cap transcript sessions per run
|
||||
--max-tasks N cap mined tasks per run
|
||||
--target-skill-path PATH explicit live SKILL.md to stage/adopt
|
||||
--tasks-file PATH reviewed TaskRecord JSON file to replay instead of harvesting
|
||||
--backend mock|claude|codex|copilot
|
||||
--source claude|codex|auto
|
||||
--model NAME
|
||||
--lookback-hours N
|
||||
--auto-adopt
|
||||
--json machine-readable output
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, Dict
|
||||
|
||||
from skillopt_sleep.config import load_config
|
||||
from skillopt_sleep.cycle import run_sleep_cycle
|
||||
from skillopt_sleep.harvest_sources import harvest_for_config
|
||||
from skillopt_sleep.mine import mine
|
||||
from skillopt_sleep.staging import adopt as adopt_staging
|
||||
from skillopt_sleep.staging import latest_staging
|
||||
from skillopt_sleep.staging import redact_secrets
|
||||
from skillopt_sleep.state import SleepState
|
||||
from skillopt_sleep.tasks_file import load_tasks_file, make_tasks_payload, write_tasks_file
|
||||
|
||||
|
||||
def _read_text(path: str) -> str:
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _report_payload(rep, outcome, *, redact: bool = True) -> Dict[str, Any]:
|
||||
# edits/rejected_edits carry EditRecord.content/.rationale sourced from
|
||||
# reflect()'s output over real harvested task responses -- the same risk
|
||||
# staging.py's write_staging() already redacts for the on-disk
|
||||
# proposed_*.md/report.md/report.json. Redact here too: this is CLI
|
||||
# stdout, and scheduler.py's cron entry redirects that stdout straight
|
||||
# into <project>/.skillopt-sleep/cron.log.
|
||||
_r = redact_secrets if redact else (lambda v: v)
|
||||
return {
|
||||
"night": rep.night,
|
||||
"accepted": rep.accepted,
|
||||
"gate_action": rep.gate_action,
|
||||
"no_edits_reason": getattr(rep, "no_edits_reason", ""),
|
||||
"baseline": rep.baseline_score,
|
||||
"candidate": rep.candidate_score,
|
||||
"n_tasks": rep.n_tasks,
|
||||
"n_sessions": rep.n_sessions,
|
||||
"n_accepted_edits": len(rep.edits),
|
||||
"n_rejected_edits": len(rep.rejected_edits),
|
||||
"edits": [_r(e.__dict__) for e in rep.edits],
|
||||
"rejected_edits": [_r(e.__dict__) for e in rep.rejected_edits],
|
||||
"notes": rep.notes,
|
||||
"staging_dir": outcome.staging_dir,
|
||||
"adopted": outcome.adopted,
|
||||
}
|
||||
|
||||
|
||||
def _add_common(p: argparse.ArgumentParser) -> None:
|
||||
p.add_argument("--project", default="")
|
||||
p.add_argument("--scope", default="", choices=["", "all", "invoked"])
|
||||
p.add_argument("--backend", default="", choices=["", "mock", "claude", "codex", "copilot"])
|
||||
p.add_argument("--model", default="")
|
||||
p.add_argument("--codex-path", default="", help="path to the real @openai/codex binary")
|
||||
p.add_argument("--claude-home", default="", help="override ~/.claude (also isolates state)")
|
||||
p.add_argument("--codex-home", default="", help="override ~/.codex for archived session harvest")
|
||||
p.add_argument("--source", default="", choices=["", "claude", "codex", "auto"],
|
||||
help="session transcript source")
|
||||
p.add_argument("--lookback-hours", type=int, default=None,
|
||||
help="harvest window in hours; 0 = scan full history")
|
||||
p.add_argument("--edit-budget", type=int, default=0)
|
||||
p.add_argument("--max-sessions", type=int, default=0,
|
||||
help="cap harvested sessions before mining; default derives from max tasks")
|
||||
p.add_argument("--max-tasks", type=int, default=0,
|
||||
help="cap mined tasks for this run")
|
||||
p.add_argument("--target-skill-path", default="",
|
||||
help="explicit live SKILL.md path to evolve/stage/adopt")
|
||||
p.add_argument("--tasks-file", default="",
|
||||
help="reviewed TaskRecord JSON file to replay instead of harvesting")
|
||||
p.add_argument("--progress", action="store_true",
|
||||
help="print phase progress to stderr")
|
||||
p.add_argument("--auto-adopt", action="store_true")
|
||||
p.add_argument("--json", action="store_true")
|
||||
|
||||
|
||||
def _cfg_from_args(args, task_meta: Dict[str, Any] | None = None) -> Any:
|
||||
overrides: Dict[str, Any] = {}
|
||||
if args.project:
|
||||
overrides["invoked_project"] = os.path.abspath(args.project)
|
||||
overrides["projects"] = "invoked"
|
||||
if args.scope:
|
||||
overrides["projects"] = args.scope
|
||||
if args.backend:
|
||||
overrides["backend"] = args.backend
|
||||
if args.model:
|
||||
overrides["model"] = args.model
|
||||
if getattr(args, "codex_path", ""):
|
||||
overrides["codex_path"] = os.path.abspath(args.codex_path)
|
||||
if getattr(args, "claude_home", ""):
|
||||
overrides["claude_home"] = os.path.abspath(args.claude_home)
|
||||
if getattr(args, "codex_home", ""):
|
||||
overrides["codex_home"] = os.path.abspath(args.codex_home)
|
||||
if getattr(args, "source", ""):
|
||||
overrides["transcript_source"] = args.source
|
||||
lh = getattr(args, "lookback_hours", None)
|
||||
if lh is not None: # --lookback-hours was explicitly passed (0 = full history)
|
||||
overrides["lookback_hours"] = lh
|
||||
if getattr(args, "edit_budget", 0):
|
||||
overrides["edit_budget"] = args.edit_budget
|
||||
if getattr(args, "max_sessions", 0):
|
||||
overrides["max_sessions_per_night"] = args.max_sessions
|
||||
if getattr(args, "max_tasks", 0):
|
||||
overrides["max_tasks_per_night"] = args.max_tasks
|
||||
target_skill_path = getattr(args, "target_skill_path", "")
|
||||
if not target_skill_path and task_meta:
|
||||
target_skill_path = str(task_meta.get("target_skill_path") or "")
|
||||
if target_skill_path:
|
||||
path = os.path.expanduser(target_skill_path)
|
||||
if args.project and not os.path.isabs(path):
|
||||
path = os.path.join(os.path.abspath(args.project), path)
|
||||
overrides["target_skill_path"] = os.path.abspath(path)
|
||||
if getattr(args, "progress", False):
|
||||
overrides["progress"] = True
|
||||
if getattr(args, "auto_adopt", False):
|
||||
overrides["auto_adopt"] = True
|
||||
return load_config(**overrides)
|
||||
|
||||
|
||||
def cmd_run(args, dry: bool = False) -> int:
|
||||
task_meta: Dict[str, Any] = {}
|
||||
tasks = None
|
||||
if getattr(args, "tasks_file", ""):
|
||||
# Load once before config so target_skill_path can default from metadata.
|
||||
tasks, task_meta = load_tasks_file(args.tasks_file)
|
||||
cfg = _cfg_from_args(args, task_meta=task_meta)
|
||||
if getattr(args, "tasks_file", ""):
|
||||
tasks, task_meta = load_tasks_file(
|
||||
args.tasks_file,
|
||||
holdout_fraction=cfg.get("holdout_fraction", 0.34),
|
||||
seed=cfg.get("seed", 42),
|
||||
)
|
||||
if cfg.get("backend", "mock") != "mock" and task_meta.get("reviewed") is not True:
|
||||
print(
|
||||
"[sleep] refusing real-backend replay from an unreviewed tasks file; "
|
||||
"inspect/redact it and set \"reviewed\": true first",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
outcome = run_sleep_cycle(cfg, seed_tasks=tasks, dry_run=dry)
|
||||
rep = outcome.report
|
||||
# Same redact_secrets flag cycle.py/staging.py honor for staged files;
|
||||
# this is the CLI's own stdout, which scheduler.py's cron entry
|
||||
# redirects straight into <project>/.skillopt-sleep/cron.log.
|
||||
redact_enabled = bool(cfg.get("redact_secrets", True))
|
||||
_r = redact_secrets if redact_enabled else (lambda v: v)
|
||||
if args.json:
|
||||
payload = _report_payload(rep, outcome, redact=redact_enabled)
|
||||
if task_meta:
|
||||
payload["tasks_file"] = task_meta.get("tasks_file", "")
|
||||
payload["tasks_reviewed"] = task_meta.get("reviewed", False)
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"[sleep] night {rep.night}: {rep.n_sessions} sessions -> {rep.n_tasks} tasks")
|
||||
print(f"[sleep] held-out {rep.baseline_score:.3f} -> {rep.candidate_score:.3f} "
|
||||
f"=> {rep.gate_action} (accepted={rep.accepted})")
|
||||
for e in rep.edits:
|
||||
print(f" + [{e.target}/{e.op}] {_r(e.content)}")
|
||||
if rep.rejected_edits:
|
||||
print("[sleep] rejected by gate:")
|
||||
for e in rep.rejected_edits:
|
||||
print(f" - [{e.target}/{e.op}] {_r(e.content)}")
|
||||
if outcome.staging_dir:
|
||||
print(f"[sleep] staged: {outcome.staging_dir}")
|
||||
if not outcome.adopted:
|
||||
print("[sleep] review it, then: python -m skillopt_sleep adopt")
|
||||
if outcome.adopted:
|
||||
print(f"[sleep] auto-adopted: {', '.join(outcome.adopted_paths)}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_status(args) -> int:
|
||||
cfg = _cfg_from_args(args)
|
||||
state = SleepState.load(cfg.state_path)
|
||||
project = cfg.get("invoked_project") or os.getcwd()
|
||||
latest = latest_staging(project)
|
||||
info = {
|
||||
"night": state.night,
|
||||
"state_path": cfg.state_path,
|
||||
"project": project,
|
||||
"history_tail": state.data.get("history", [])[-5:],
|
||||
"latest_staging": latest,
|
||||
"slow_memory_chars": len(state.slow_memory),
|
||||
}
|
||||
if args.json:
|
||||
print(json.dumps(info, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"[sleep] nights so far: {state.night}")
|
||||
print(f"[sleep] project: {project}")
|
||||
if latest:
|
||||
print(f"[sleep] latest staged proposal: {latest}")
|
||||
rp = os.path.join(latest, "report.md")
|
||||
if os.path.exists(rp):
|
||||
with open(rp) as f:
|
||||
print("\n" + f.read())
|
||||
else:
|
||||
print("[sleep] no staged proposals yet.")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_adopt(args) -> int:
|
||||
cfg = _cfg_from_args(args)
|
||||
project = cfg.get("invoked_project") or os.getcwd()
|
||||
target = args.staging or latest_staging(project)
|
||||
if not target or not os.path.isdir(target):
|
||||
print("[sleep] nothing to adopt (no staging dir).")
|
||||
return 1
|
||||
updated = adopt_staging(target)
|
||||
print(f"[sleep] adopted from {target}")
|
||||
for p in updated:
|
||||
print(f" -> {p}")
|
||||
if not updated:
|
||||
print("[sleep] (proposal contained no accepted changes)")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_harvest(args) -> int:
|
||||
cfg = _cfg_from_args(args)
|
||||
session_limit = cfg.get("max_sessions_per_night", 0) or cfg.get("max_tasks_per_night", 40) * 3
|
||||
target_skill_path = cfg.managed_skill_path() if cfg.get("target_skill_path", "") else ""
|
||||
target_skill_text = _read_text(target_skill_path) if target_skill_path else ""
|
||||
max_tasks = cfg.get("max_tasks_per_night", 40)
|
||||
candidate_limit = max_tasks
|
||||
if cfg.get("target_task_filter", True) and target_skill_text:
|
||||
candidate_limit = max(max_tasks, max_tasks * 3)
|
||||
digests = harvest_for_config(cfg, limit=session_limit)
|
||||
tasks = mine(
|
||||
digests,
|
||||
max_tasks=max_tasks,
|
||||
candidate_limit=candidate_limit,
|
||||
holdout_fraction=cfg.get("holdout_fraction", 0.34),
|
||||
seed=cfg.get("seed", 42),
|
||||
target_skill_text=target_skill_text,
|
||||
target_skill_path=target_skill_path,
|
||||
)
|
||||
payload = make_tasks_payload(
|
||||
tasks,
|
||||
project=cfg.get("invoked_project") or os.getcwd(),
|
||||
transcript_source=cfg.get("transcript_source", ""),
|
||||
n_sessions=len(digests),
|
||||
target_skill_path=target_skill_path,
|
||||
)
|
||||
# payload carries raw harvested intent/context text (this command's whole
|
||||
# purpose is letting a human review it before flipping "reviewed": true
|
||||
# on a --tasks-file). Redaction only strips secret-shaped substrings, so
|
||||
# it doesn't hurt that review; same redact_secrets flag as everywhere
|
||||
# else, applied uniformly before this reaches the --output file, --json
|
||||
# stdout, or the plain-text loop below.
|
||||
redact_enabled = bool(cfg.get("redact_secrets", True))
|
||||
if redact_enabled:
|
||||
payload = redact_secrets(payload)
|
||||
_r = redact_secrets if redact_enabled else (lambda v: v)
|
||||
output_path = ""
|
||||
if getattr(args, "output", ""):
|
||||
output_path = write_tasks_file(args.output, payload)
|
||||
if args.json:
|
||||
json_payload = dict(payload)
|
||||
if output_path:
|
||||
json_payload["output"] = output_path
|
||||
print(json.dumps(json_payload, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"[sleep] {len(digests)} sessions -> {len(tasks)} tasks")
|
||||
if output_path:
|
||||
print(f"[sleep] wrote reviewed-task draft: {output_path}")
|
||||
for t in tasks:
|
||||
print(f" [{t.split}/{t.outcome}] {_r(t.intent[:90])}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_schedule(args) -> int:
|
||||
from skillopt_sleep.scheduler import schedule, list_scheduled
|
||||
cfg = _cfg_from_args(args)
|
||||
project = cfg.get("invoked_project") or os.getcwd()
|
||||
backend = cfg.get("backend", "mock")
|
||||
# Unlike every other action, this writes to the user's REAL crontab the
|
||||
# moment it runs -- not a preview. The driving Claude Code command
|
||||
# (commands/skillopt-sleep.md) confirms with the user in chat first,
|
||||
# then passes --yes here to record that confirmation happened; this
|
||||
# gate is defense-in-depth for anyone invoking the CLI directly, since
|
||||
# that agent-level confirmation only covers the documented workflow.
|
||||
if not getattr(args, "yes", False):
|
||||
summary = (f"install a nightly cron entry for {project} at "
|
||||
f"{args.hour:02d}:{args.minute:02d} (backend={backend})")
|
||||
if not sys.stdin.isatty():
|
||||
print(
|
||||
f"[sleep] refusing to {summary} non-interactively without --yes "
|
||||
"(schedule installs immediately, unlike every other action; "
|
||||
"pass --yes once you've confirmed this with the user)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
try:
|
||||
answer = input(f"[sleep] about to {summary}. Proceed? [y/N] ")
|
||||
except EOFError:
|
||||
answer = ""
|
||||
if answer.strip().lower() not in ("y", "yes"):
|
||||
print("[sleep] aborted; nothing scheduled.")
|
||||
return 1
|
||||
ok, msg = schedule(project, backend=backend,
|
||||
hour=args.hour, minute=args.minute,
|
||||
extra=("--auto-adopt" if getattr(args, "auto_adopt", False) else ""))
|
||||
print("[sleep] " + msg)
|
||||
cur = list_scheduled()
|
||||
if cur:
|
||||
print("[sleep] currently scheduled:")
|
||||
for ln in cur:
|
||||
print(" " + ln[:140])
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def cmd_unschedule(args) -> int:
|
||||
from skillopt_sleep.scheduler import unschedule
|
||||
cfg = _cfg_from_args(args)
|
||||
project = cfg.get("invoked_project") or os.getcwd()
|
||||
ok, msg = unschedule(project, all_projects=getattr(args, "all", False))
|
||||
print("[sleep] " + msg)
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(prog="skillopt_sleep", description="SkillOpt-Sleep nightly self-evolution")
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
p_run = sub.add_parser("run", help="run a full sleep cycle")
|
||||
_add_common(p_run)
|
||||
p_dry = sub.add_parser("dry-run", help="harvest+mine+replay, report only")
|
||||
_add_common(p_dry)
|
||||
p_status = sub.add_parser("status", help="show state + latest proposal")
|
||||
_add_common(p_status)
|
||||
p_adopt = sub.add_parser("adopt", help="apply latest staged proposal")
|
||||
_add_common(p_adopt)
|
||||
p_adopt.add_argument("--staging", default="", help="specific staging dir")
|
||||
p_harvest = sub.add_parser("harvest", help="debug: show mined tasks")
|
||||
_add_common(p_harvest)
|
||||
p_harvest.add_argument("--output", default="", help="write mined tasks JSON for review")
|
||||
p_sched = sub.add_parser("schedule", help="install a nightly cron entry for this project")
|
||||
_add_common(p_sched)
|
||||
p_sched.add_argument("--hour", type=int, default=3)
|
||||
p_sched.add_argument("--minute", type=int, default=17)
|
||||
p_sched.add_argument("--yes", action="store_true",
|
||||
help="confirm installing the crontab entry now (required for "
|
||||
"non-interactive use; schedule installs immediately, unlike "
|
||||
"every other action)")
|
||||
p_unsched = sub.add_parser("unschedule", help="remove the nightly cron entry")
|
||||
_add_common(p_unsched)
|
||||
p_unsched.add_argument("--all", action="store_true", help="remove all managed entries")
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
if args.cmd == "run":
|
||||
return cmd_run(args, dry=False)
|
||||
if args.cmd == "dry-run":
|
||||
return cmd_run(args, dry=True)
|
||||
if args.cmd == "status":
|
||||
return cmd_status(args)
|
||||
if args.cmd == "adopt":
|
||||
return cmd_adopt(args)
|
||||
if args.cmd == "harvest":
|
||||
return cmd_harvest(args)
|
||||
if args.cmd == "schedule":
|
||||
return cmd_schedule(args)
|
||||
if args.cmd == "unschedule":
|
||||
return cmd_unschedule(args)
|
||||
parser.print_help()
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
1258
engineering/skillopt-sleep/skillopt_sleep/backend.py
Normal file
1258
engineering/skillopt-sleep/skillopt_sleep/backend.py
Normal file
File diff suppressed because it is too large
Load diff
75
engineering/skillopt-sleep/skillopt_sleep/budget.py
Normal file
75
engineering/skillopt-sleep/skillopt_sleep/budget.py
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
"""SkillOpt-Sleep — budget controller.
|
||||
|
||||
Lets the user say how much they're willing to spend on a night's "dreaming",
|
||||
in tokens or wall-clock minutes, and the engine schedules depth (how many
|
||||
rollouts × how many nights) within that budget. Stops cleanly when exhausted
|
||||
and reports what it skipped (no silent truncation).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class Budget:
|
||||
max_tokens: Optional[int] = None # None = unlimited
|
||||
max_minutes: Optional[float] = None # None = unlimited
|
||||
_start_time: Optional[float] = None
|
||||
_tokens_at_start: int = 0
|
||||
|
||||
def start(self, clock_fn, tokens_now: int) -> None:
|
||||
self._start_time = clock_fn()
|
||||
self._tokens_at_start = tokens_now
|
||||
|
||||
def tokens_spent(self, tokens_now: int) -> int:
|
||||
return max(0, tokens_now - self._tokens_at_start)
|
||||
|
||||
def minutes_elapsed(self, clock_fn) -> float:
|
||||
if self._start_time is None:
|
||||
return 0.0
|
||||
return (clock_fn() - self._start_time) / 60.0
|
||||
|
||||
def remaining_fraction(self, *, tokens_now: int, clock_fn) -> float:
|
||||
"""Smallest remaining fraction across all active limits (1.0 = fresh)."""
|
||||
fracs = [1.0]
|
||||
if self.max_tokens:
|
||||
fracs.append(max(0.0, 1.0 - self.tokens_spent(tokens_now) / self.max_tokens))
|
||||
if self.max_minutes:
|
||||
fracs.append(max(0.0, 1.0 - self.minutes_elapsed(clock_fn) / self.max_minutes))
|
||||
return min(fracs)
|
||||
|
||||
def exhausted(self, *, tokens_now: int, clock_fn) -> bool:
|
||||
if self.max_tokens and self.tokens_spent(tokens_now) >= self.max_tokens:
|
||||
return True
|
||||
if self.max_minutes and self.minutes_elapsed(clock_fn) >= self.max_minutes:
|
||||
return True
|
||||
return False
|
||||
|
||||
def status(self, *, tokens_now: int, clock_fn) -> str:
|
||||
parts = []
|
||||
if self.max_tokens:
|
||||
parts.append(f"tokens {self.tokens_spent(tokens_now)}/{self.max_tokens}")
|
||||
if self.max_minutes:
|
||||
parts.append(f"minutes {self.minutes_elapsed(clock_fn):.1f}/{self.max_minutes}")
|
||||
return ", ".join(parts) or "unbounded"
|
||||
|
||||
|
||||
def plan_depth(budget: Budget, *, n_tasks: int,
|
||||
default_nights: int = 2, default_k: int = 1) -> tuple:
|
||||
"""Heuristically choose (nights, rollouts_per_task) from a token budget.
|
||||
|
||||
Rough cost model: one rollout ≈ 1 unit; a night does ~n_tasks*k rollouts
|
||||
plus reflect/gate (~2*n_tasks). We scale k and nights up with more budget.
|
||||
Returns (nights, k). With no budget set, returns the defaults.
|
||||
"""
|
||||
if not budget.max_tokens:
|
||||
return default_nights, default_k
|
||||
# assume ~1.5k tokens per rollout as a planning constant
|
||||
rollouts_affordable = budget.max_tokens / 1500.0
|
||||
per_night = max(1, n_tasks) * 3 # rollouts + reflect + gate, k=1
|
||||
nights = max(1, min(4, int(rollouts_affordable // per_night)))
|
||||
# spend surplus on more rollouts-per-task (contrastive signal)
|
||||
surplus = rollouts_affordable - nights * per_night
|
||||
k = max(1, min(5, 1 + int(surplus // max(1, n_tasks))))
|
||||
return nights, k
|
||||
162
engineering/skillopt-sleep/skillopt_sleep/config.py
Normal file
162
engineering/skillopt-sleep/skillopt_sleep/config.py
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
"""SkillOpt-Sleep — configuration.
|
||||
|
||||
Config is JSON-first (yaml optional) so the engine and the deterministic
|
||||
experiment run with zero external dependencies. Defaults are safe:
|
||||
review-gated adoption, single-project scope, bounded token/task budgets.
|
||||
|
||||
Resolution order (later wins):
|
||||
1. built-in DEFAULTS
|
||||
2. ~/.skillopt-sleep/config.json (or .yaml if PyYAML available)
|
||||
3. explicit overrides passed to load_config(**overrides)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
HOME_STATE_DIR = os.path.expanduser("~/.skillopt-sleep")
|
||||
CLAUDE_HOME = os.path.expanduser("~/.claude")
|
||||
CODEX_HOME = os.path.expanduser("~/.codex")
|
||||
|
||||
|
||||
DEFAULTS: Dict[str, Any] = {
|
||||
# ── scope ──────────────────────────────────────────────────────────────
|
||||
"claude_home": CLAUDE_HOME,
|
||||
"codex_home": CODEX_HOME,
|
||||
"transcript_source": "claude", # "claude" | "codex" | "auto"
|
||||
"projects": "invoked", # "invoked" | "all" | [list of abs paths]
|
||||
"invoked_project": "", # filled at runtime (cwd) when projects == "invoked"
|
||||
"lookback_hours": 72, # harvest window when no prior sleep recorded
|
||||
# ── budgets ────────────────────────────────────────────────────────────
|
||||
"max_tasks_per_night": 40,
|
||||
"max_tokens_per_night": 400_000,
|
||||
"holdout_fraction": 0.34, # legacy alias for val_fraction
|
||||
"val_fraction": 0.34, # real tasks reserved to gate updates
|
||||
"test_fraction": 0.0, # real tasks reserved as the final held-out measure
|
||||
# ── optimizer ──────────────────────────────────────────────────────────
|
||||
"backend": "mock", # "mock" | "claude" | "codex" | "copilot"
|
||||
"model": "", # backend-specific; "" => backend default
|
||||
"gate_mode": "on", # "on" (validation-gated) | "off" (greedy, no hard filter)
|
||||
"codex_path": "", # "" => auto-detect the real @openai/codex binary
|
||||
"edit_budget": 4, # textual learning rate (max edits/night)
|
||||
"gate_metric": "mixed", # hard | soft | mixed (mixed best for tiny holdouts)
|
||||
"gate_mixed_weight": 0.5,
|
||||
"replay_mode": "mock", # "mock" (sandboxed prompt) | "fresh" (worktree)
|
||||
# ── dream + recall (opt-in; defaults reproduce the prior single-shot loop) ─
|
||||
"dream_rollouts": 1, # >1 => multi-rollout contrastive reflection per task
|
||||
"dream_factor": 0, # >0 => add N synthetic variants of each task to the dream
|
||||
"recall_k": 0, # >0 => recall the K most-similar past tasks into the dream
|
||||
"evolve_memory": True, # consolidate CLAUDE.md
|
||||
"evolve_skill": True, # consolidate the managed SKILL.md
|
||||
"llm_mine": True, # use the backend to mine checkable tasks (real backends)
|
||||
"target_skill_path": "", # explicit SKILL.md target for repo-scoped agents
|
||||
"target_task_filter": True, # prefer mined tasks matching target_skill_path/text
|
||||
"progress": False, # print phase progress to stderr
|
||||
# ── adoption / safety ──────────────────────────────────────────────────
|
||||
"auto_adopt": False, # default: stage + require explicit `adopt`
|
||||
"managed_skill_name": "skillopt-sleep-learned",
|
||||
"redact_secrets": True,
|
||||
"seed": 42,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class SleepConfig:
|
||||
data: Dict[str, Any] = field(default_factory=lambda: dict(DEFAULTS))
|
||||
|
||||
# convenient attribute access -------------------------------------------
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
# only called when normal attribute lookup fails
|
||||
data = object.__getattribute__(self, "data")
|
||||
if name in data:
|
||||
return data[name]
|
||||
raise AttributeError(name)
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Any:
|
||||
return self.data.get(key, default)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return dict(self.data)
|
||||
|
||||
# paths ------------------------------------------------------------------
|
||||
@property
|
||||
def state_dir(self) -> str:
|
||||
# Allow full isolation: if the caller overrides state_dir explicitly,
|
||||
# honor it; else derive from claude_home's parent so a single
|
||||
# --claude-home flag isolates transcripts AND state together; else the
|
||||
# default ~/.skillopt-sleep.
|
||||
explicit = self.data.get("state_dir")
|
||||
if explicit:
|
||||
return explicit
|
||||
ch = self.data.get("claude_home", CLAUDE_HOME)
|
||||
if os.path.abspath(ch) != os.path.abspath(CLAUDE_HOME):
|
||||
return os.path.join(os.path.dirname(os.path.abspath(ch)), ".skillopt-sleep")
|
||||
return HOME_STATE_DIR
|
||||
|
||||
@property
|
||||
def state_path(self) -> str:
|
||||
return os.path.join(self.state_dir, "state.json")
|
||||
|
||||
@property
|
||||
def transcripts_dir(self) -> str:
|
||||
return os.path.join(self.data["claude_home"], "projects")
|
||||
|
||||
@property
|
||||
def codex_archived_sessions_dir(self) -> str:
|
||||
return os.path.join(self.data["codex_home"], "archived_sessions")
|
||||
|
||||
@property
|
||||
def history_path(self) -> str:
|
||||
return os.path.join(self.data["claude_home"], "history.jsonl")
|
||||
|
||||
@property
|
||||
def skills_dir(self) -> str:
|
||||
return os.path.join(self.data["claude_home"], "skills")
|
||||
|
||||
def managed_skill_path(self) -> str:
|
||||
target = self.data.get("target_skill_path") or ""
|
||||
if target:
|
||||
target = os.path.expanduser(str(target))
|
||||
if not os.path.isabs(target):
|
||||
base = self.data.get("invoked_project") or os.getcwd()
|
||||
target = os.path.join(base, target)
|
||||
return os.path.abspath(target)
|
||||
return os.path.join(
|
||||
self.skills_dir, self.data["managed_skill_name"], "SKILL.md"
|
||||
)
|
||||
|
||||
|
||||
def _user_config_path() -> Optional[str]:
|
||||
for name in ("config.json", "config.yaml", "config.yml"):
|
||||
p = os.path.join(HOME_STATE_DIR, name)
|
||||
if os.path.exists(p):
|
||||
return p
|
||||
return None
|
||||
|
||||
|
||||
def _load_file(path: str) -> Dict[str, Any]:
|
||||
if path.endswith((".yaml", ".yml")):
|
||||
try:
|
||||
import yaml # optional
|
||||
with open(path) as f:
|
||||
return yaml.safe_load(f) or {}
|
||||
except Exception:
|
||||
return {}
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def load_config(**overrides: Any) -> SleepConfig:
|
||||
data = dict(DEFAULTS)
|
||||
path = _user_config_path()
|
||||
if path:
|
||||
try:
|
||||
data.update(_load_file(path) or {})
|
||||
except Exception:
|
||||
pass
|
||||
data.update({k: v for k, v in overrides.items() if v is not None})
|
||||
if data.get("projects") == "invoked" and not data.get("invoked_project"):
|
||||
data["invoked_project"] = os.getcwd()
|
||||
return SleepConfig(data=data)
|
||||
266
engineering/skillopt-sleep/skillopt_sleep/consolidate.py
Normal file
266
engineering/skillopt-sleep/skillopt_sleep/consolidate.py
Normal file
|
|
@ -0,0 +1,266 @@
|
|||
"""SkillOpt-Sleep — Stage 4: consolidate (one SkillOpt epoch).
|
||||
|
||||
This is the core that makes nightly evolution *safe*: it proposes bounded
|
||||
edits from replayed failures, applies them to a candidate skill/memory, then
|
||||
**gates** the candidate on a held-out slice of the user's own tasks. Only a
|
||||
candidate that strictly improves the held-out score is accepted — the SkillOpt
|
||||
validation gate, vendored self-contained in ``skillopt_sleep.gate``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from skillopt_sleep.backend import Backend
|
||||
from skillopt_sleep.memory import apply_edits
|
||||
from skillopt_sleep.replay import aggregate_scores, replay_batch
|
||||
from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord
|
||||
|
||||
|
||||
# Self-contained validation gate (vendored from SkillOpt; zero dependency on the
|
||||
# research package, so this open-source tool stays decoupled from the paper code).
|
||||
from skillopt_sleep.gate import evaluate_gate, select_gate_score
|
||||
_HAVE_REPO_GATE = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConsolidationResult:
|
||||
accepted: bool
|
||||
gate_action: str
|
||||
baseline_score: float
|
||||
candidate_score: float
|
||||
new_skill: str
|
||||
new_memory: str
|
||||
applied_edits: List[EditRecord]
|
||||
rejected_edits: List[EditRecord]
|
||||
holdout_baseline: float
|
||||
holdout_candidate: float
|
||||
# ── observability (so a 0.0->0.0 night is self-diagnosing, not a black box) ──
|
||||
holdout_detail: List[dict] = field(default_factory=list) # per val task: hard/soft/resp/why
|
||||
reflect_raw: str = "" # the optimizer's last raw reply (empty => reflect produced nothing)
|
||||
call_error: str = "" # backend's last call error (timeout/auth/empty)
|
||||
|
||||
|
||||
def _split(tasks: List[TaskRecord]) -> Tuple[List[TaskRecord], List[TaskRecord]]:
|
||||
"""Return (train_tasks, val_tasks).
|
||||
|
||||
train drives reflect; val gates updates. test is held out entirely from
|
||||
consolidation and is scored by the caller. Accepts legacy split names
|
||||
(replay->train, holdout->val) for robustness.
|
||||
"""
|
||||
def _norm(s: str) -> str:
|
||||
return {"replay": "train", "holdout": "val"}.get(s, s)
|
||||
|
||||
train = [t for t in tasks if _norm(t.split) == "train"]
|
||||
val = [t for t in tasks if _norm(t.split) == "val"]
|
||||
# be robust if a split is empty: fall back so a night still does something,
|
||||
# but never silently use test as val.
|
||||
test = [t for t in tasks if _norm(t.split) == "test"]
|
||||
if not val:
|
||||
# prefer train as the gate reference over nothing; last resort all-but-test
|
||||
val = train or [t for t in tasks if _norm(t.split) != "test"] or tasks
|
||||
if not train:
|
||||
train = val
|
||||
return train, val
|
||||
|
||||
|
||||
def _holdout_detail(pairs: List[Tuple[TaskRecord, ReplayResult]]) -> List[dict]:
|
||||
"""Per-task held-out evidence so a 0.0 night explains itself: was the
|
||||
response empty (backend call failed) or non-empty-but-failing-checks
|
||||
(judge too strict / edit didn't help)? The two need opposite fixes."""
|
||||
out: List[dict] = []
|
||||
for t, r in pairs:
|
||||
resp = r.response or ""
|
||||
out.append({
|
||||
"id": t.id,
|
||||
"reference_kind": t.reference_kind,
|
||||
"hard": r.hard,
|
||||
"soft": r.soft,
|
||||
"response_len": len(resp),
|
||||
"response_head": resp[:200],
|
||||
"why": (r.fail_reason or r.judge_rationale or "")[:200],
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def consolidate(
|
||||
backend: Backend,
|
||||
tasks: List[TaskRecord],
|
||||
skill: str,
|
||||
memory: str,
|
||||
*,
|
||||
edit_budget: int = 4,
|
||||
gate_metric: str = "mixed",
|
||||
gate_mixed_weight: float = 0.5,
|
||||
gate_mode: str = "on", # "on" (hard/soft per gate_metric) | "off" (greedy)
|
||||
rollouts_k: int = 1, # >1 => multi-rollout contrastive reflection
|
||||
evolve_skill: bool = True,
|
||||
evolve_memory: bool = True,
|
||||
night: int = 1,
|
||||
) -> ConsolidationResult:
|
||||
"""Run one consolidation epoch: reflect -> bounded edit -> gate.
|
||||
|
||||
train tasks drive reflect; val tasks gate the update (test is held out by the
|
||||
caller). With ``gate_mode='off'`` edits are accepted greedily (no val-improve
|
||||
requirement) — the user opts out of hard filtering — but val scores are still
|
||||
recorded so the report shows whether quality moved.
|
||||
|
||||
Skill and memory are evolved in sequence (skill first if both enabled).
|
||||
"""
|
||||
train_tasks, val_tasks = _split(tasks)
|
||||
gate_off = str(gate_mode).strip().lower() in {"off", "none", "false", "greedy"}
|
||||
holdout_detail: List[dict] = []
|
||||
|
||||
# ── baseline on the VAL slice (the gate reference) ────────────────────
|
||||
# When the gate is OFF the user has opted out of holding out a validation set
|
||||
# (the daily-use design): we accept edits greedily and judge quality only on
|
||||
# the real test set, scored by the caller. So we SKIP all val scoring — it is
|
||||
# both wasted cost and contrary to the "no val set required" design.
|
||||
if gate_off:
|
||||
base_hard, base_soft = 0.0, 0.0
|
||||
else:
|
||||
base_pairs = replay_batch(backend, val_tasks, skill, memory)
|
||||
base_hard, base_soft = aggregate_scores(base_pairs)
|
||||
holdout_detail = _holdout_detail(base_pairs)
|
||||
base_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight)
|
||||
|
||||
# ── reflect over TRAIN-split failures/successes ───────────────────────
|
||||
train_pairs = replay_batch(backend, train_tasks, skill, memory)
|
||||
failures = [(t, r) for (t, r) in train_pairs if r.hard < 1.0]
|
||||
successes = [(t, r) for (t, r) in train_pairs if r.hard >= 1.0]
|
||||
|
||||
cand_skill, cand_memory = skill, memory
|
||||
all_applied: List[EditRecord] = []
|
||||
all_rejected: List[EditRecord] = []
|
||||
|
||||
def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str:
|
||||
nonlocal cand_skill, cand_memory, base_score, all_applied, all_rejected
|
||||
if not edits:
|
||||
return doc
|
||||
new_doc, applied = apply_edits(doc, edits)
|
||||
if not applied:
|
||||
return doc
|
||||
# gate OFF: accept greedily with NO val scoring (the daily-use path)
|
||||
if gate_off:
|
||||
all_applied.extend(applied)
|
||||
return new_doc
|
||||
# gate ON: score the candidate on the VAL slice, keep only if it improves
|
||||
trial_skill = new_doc if which == "skill" else cand_skill
|
||||
trial_memory = new_doc if which == "memory" else cand_memory
|
||||
pairs = replay_batch(backend, val_tasks, trial_skill, trial_memory)
|
||||
h, s = aggregate_scores(pairs)
|
||||
cand_score = select_gate_score(h, s, gate_metric, gate_mixed_weight)
|
||||
if cand_score > base_score:
|
||||
base_score = max(base_score, cand_score)
|
||||
all_applied.extend(applied)
|
||||
return new_doc
|
||||
all_rejected.extend(applied)
|
||||
return doc
|
||||
|
||||
if evolve_skill:
|
||||
if rollouts_k > 1:
|
||||
# multi-rollout contrastive reflection: run each train task K times
|
||||
# and distill a rule from the good-vs-bad contrast (the imagination signal).
|
||||
from skillopt_sleep.rollout import multi_rollout, contrastive_reflect
|
||||
# Parallelize across tasks (each multi_rollout also parallelizes its K
|
||||
# attempts). This dream phase is the dominant cost; serial execution
|
||||
# times out on real backends. Cap total in-flight at the worker env.
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
try:
|
||||
_w = int(os.environ.get("SKILLOPT_SLEEP_WORKERS", "1"))
|
||||
except ValueError:
|
||||
_w = 1
|
||||
if _w > 1 and len(train_tasks) > 1:
|
||||
# split the worker budget between task-parallelism and per-task K
|
||||
task_workers = max(1, min(len(train_tasks), _w))
|
||||
per_task = max(1, _w // task_workers)
|
||||
with ThreadPoolExecutor(max_workers=task_workers) as ex:
|
||||
sets = list(ex.map(
|
||||
lambda t: multi_rollout(backend, t, cand_skill, cand_memory,
|
||||
k=rollouts_k, workers=per_task),
|
||||
train_tasks))
|
||||
else:
|
||||
sets = [multi_rollout(backend, t, cand_skill, cand_memory,
|
||||
k=rollouts_k, workers=1)
|
||||
for t in train_tasks]
|
||||
edits = contrastive_reflect(
|
||||
backend, sets, cand_skill, cand_memory,
|
||||
edit_budget=edit_budget, target="skill",
|
||||
)
|
||||
# fall back to single-shot reflect if contrast yielded nothing
|
||||
if not edits:
|
||||
edits = backend.reflect(
|
||||
failures, successes, cand_skill, cand_memory,
|
||||
edit_budget=edit_budget, evolve_skill=True, evolve_memory=False,
|
||||
)
|
||||
else:
|
||||
edits = backend.reflect(
|
||||
failures, successes, cand_skill, cand_memory,
|
||||
edit_budget=edit_budget, evolve_skill=True, evolve_memory=False,
|
||||
)
|
||||
cand_skill = _gate_apply(cand_skill, edits, "skill")
|
||||
|
||||
if evolve_memory:
|
||||
# re-evaluate failures under the (possibly improved) skill
|
||||
train_pairs2 = replay_batch(backend, train_tasks, cand_skill, cand_memory)
|
||||
failures2 = [(t, r) for (t, r) in train_pairs2 if r.hard < 1.0]
|
||||
successes2 = [(t, r) for (t, r) in train_pairs2 if r.hard >= 1.0]
|
||||
edits_m = backend.reflect(
|
||||
failures2, successes2, cand_skill, cand_memory,
|
||||
edit_budget=edit_budget, evolve_skill=False, evolve_memory=True,
|
||||
)
|
||||
cand_memory = _gate_apply(cand_memory, edits_m, "memory")
|
||||
|
||||
# ── final decision ────────────────────────────────────────────────────
|
||||
if gate_off:
|
||||
# greedy mode: no val scoring at all. Keep whatever edits we applied; the
|
||||
# caller measures real quality on the test set. We report holdout_candidate
|
||||
# as 0.0 (val intentionally not computed in this variant).
|
||||
final_hard, final_soft = 0.0, 0.0
|
||||
final_score = 0.0
|
||||
accepted = bool(all_applied)
|
||||
action = "greedy_applied" if all_applied else "greedy_noop"
|
||||
base_gate_score = 0.0
|
||||
else:
|
||||
# scored on the VAL slice (the gate reference)
|
||||
final_pairs = replay_batch(backend, val_tasks, cand_skill, cand_memory)
|
||||
final_hard, final_soft = aggregate_scores(final_pairs)
|
||||
final_score = select_gate_score(final_hard, final_soft, gate_metric, gate_mixed_weight)
|
||||
base_gate_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight)
|
||||
if _HAVE_REPO_GATE:
|
||||
gate = evaluate_gate(
|
||||
candidate_skill=cand_skill,
|
||||
cand_hard=final_hard,
|
||||
current_skill=skill,
|
||||
current_score=base_gate_score,
|
||||
best_skill=skill,
|
||||
best_score=base_gate_score,
|
||||
best_step=night - 1,
|
||||
global_step=night,
|
||||
cand_soft=final_soft,
|
||||
metric=gate_metric,
|
||||
mixed_weight=gate_mixed_weight,
|
||||
)
|
||||
action = gate.action
|
||||
accepted = bool(all_applied) and final_score > base_gate_score
|
||||
else:
|
||||
action = "accept" if final_score > base_gate_score else "reject"
|
||||
accepted = bool(all_applied) and final_score > base_gate_score
|
||||
|
||||
return ConsolidationResult(
|
||||
accepted=accepted,
|
||||
gate_action=action,
|
||||
baseline_score=base_gate_score,
|
||||
candidate_score=final_score,
|
||||
new_skill=cand_skill if accepted else skill,
|
||||
new_memory=cand_memory if accepted else memory,
|
||||
applied_edits=all_applied,
|
||||
rejected_edits=all_rejected,
|
||||
holdout_baseline=base_hard,
|
||||
holdout_candidate=final_hard,
|
||||
holdout_detail=holdout_detail,
|
||||
reflect_raw=getattr(backend, "last_reflect_raw", "") or "",
|
||||
call_error=getattr(backend, "last_call_error", "") or "",
|
||||
)
|
||||
397
engineering/skillopt-sleep/skillopt_sleep/cycle.py
Normal file
397
engineering/skillopt-sleep/skillopt_sleep/cycle.py
Normal file
|
|
@ -0,0 +1,397 @@
|
|||
"""SkillOpt-Sleep — the nightly cycle orchestrator.
|
||||
|
||||
run_sleep_cycle() wires the stages:
|
||||
harvest -> mine -> replay -> consolidate(gate) -> stage (-> optional adopt)
|
||||
|
||||
It is pure-Python and import-light; with backend="mock" it runs with no API
|
||||
key and no third-party deps, which is what the deterministic experiment and
|
||||
CI use. With backend="anthropic" it spends the user's budget for real lift.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional
|
||||
|
||||
from skillopt_sleep.backend import get_backend
|
||||
from skillopt_sleep.budget import Budget, plan_depth
|
||||
from skillopt_sleep.config import SleepConfig, load_config
|
||||
from skillopt_sleep.dream import dream_consolidate
|
||||
from skillopt_sleep.harvest_sources import harvest_for_config
|
||||
from skillopt_sleep.memory import ensure_skill_scaffold
|
||||
from skillopt_sleep.mine import mine
|
||||
from skillopt_sleep.staging import adopt as adopt_staging
|
||||
from skillopt_sleep.staging import redact_secrets
|
||||
from skillopt_sleep.staging import write_staging
|
||||
from skillopt_sleep.state import SleepState, _now_iso
|
||||
from skillopt_sleep.types import SessionDigest, SleepReport, TaskRecord
|
||||
|
||||
|
||||
@dataclass
|
||||
class CycleOutcome:
|
||||
report: SleepReport
|
||||
staging_dir: str
|
||||
adopted: bool
|
||||
adopted_paths: List[str]
|
||||
|
||||
|
||||
def _project_paths(cfg: SleepConfig) -> str:
|
||||
"""Where live CLAUDE.md lives + which project we are evolving."""
|
||||
if cfg.get("projects") == "invoked" and cfg.get("invoked_project"):
|
||||
return cfg.get("invoked_project")
|
||||
# default: the invoked cwd
|
||||
return cfg.get("invoked_project") or os.getcwd()
|
||||
|
||||
|
||||
def _read(path: str) -> str:
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _progress(cfg: SleepConfig, message: str) -> None:
|
||||
if cfg.get("progress", False):
|
||||
print(f"[sleep] {message}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def _render_report_md(report: SleepReport, cfg: SleepConfig) -> str:
|
||||
lines = [
|
||||
f"# SkillOpt-Sleep — night {report.night} report",
|
||||
"",
|
||||
f"- project: `{report.project}`",
|
||||
f"- backend: `{cfg.get('backend')}` replay: `{cfg.get('replay_mode')}`",
|
||||
f"- sessions harvested: {report.n_sessions}",
|
||||
f"- tasks mined: {report.n_tasks} (replayed: {report.n_replayed})",
|
||||
f"- held-out score: {report.baseline_score:.3f} -> {report.candidate_score:.3f}",
|
||||
f"- gate: **{report.gate_action}** (accepted={report.accepted})",
|
||||
f"- tokens used: {report.tokens_used}",
|
||||
"",
|
||||
]
|
||||
if report.edits:
|
||||
lines.append("## Accepted edits")
|
||||
for e in report.edits:
|
||||
lines.append(f"- [{e.target}/{e.op}] {e.content} \n _why: {e.rationale}_")
|
||||
lines.append("")
|
||||
if report.rejected_edits:
|
||||
lines.append("## Rejected by gate (kept as negative feedback)")
|
||||
for e in report.rejected_edits:
|
||||
lines.append(f"- [{e.target}/{e.op}] {e.content}")
|
||||
lines.append("")
|
||||
if report.notes:
|
||||
lines.append("## Notes")
|
||||
for n in report.notes:
|
||||
lines.append(f"- {n}")
|
||||
lines.append("")
|
||||
lines.append("_Review, then run `/sleep adopt` to apply, or discard this folder._")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def run_sleep_cycle(
|
||||
cfg: Optional[SleepConfig] = None,
|
||||
*,
|
||||
seed_tasks: Optional[List[TaskRecord]] = None,
|
||||
dry_run: bool = False,
|
||||
clock: Optional[float] = None,
|
||||
) -> CycleOutcome:
|
||||
"""Run one full sleep cycle and return the outcome.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
cfg : SleepConfig
|
||||
seed_tasks : optional pre-built TaskRecords (used by the experiment to
|
||||
inject a known persona instead of harvesting ~/.claude).
|
||||
dry_run : harvest+mine+replay but DO NOT stage/adopt (report only).
|
||||
clock : fixed epoch seconds for deterministic timestamps in tests.
|
||||
"""
|
||||
cfg = cfg or load_config()
|
||||
state = SleepState.load(cfg.state_path)
|
||||
night = state.begin_night(clock)
|
||||
project = _project_paths(cfg)
|
||||
started = _now_iso(clock)
|
||||
|
||||
backend = get_backend(
|
||||
cfg.get("backend", "mock"),
|
||||
model=cfg.get("model", ""),
|
||||
codex_path=cfg.get("codex_path", ""),
|
||||
project_dir=project,
|
||||
)
|
||||
_progress(cfg, f"night {night}: project={project} backend={backend.name}")
|
||||
|
||||
# ── budget: max_tokens_per_night was declared in DEFAULTS but never
|
||||
# enforced anywhere in this path (a real gap vs. the README's "budget
|
||||
# capped" claim). Start tracking now, before harvest/mine can spend
|
||||
# tokens via the LLM miner, so the cap accounts for the whole night.
|
||||
budget = Budget(max_tokens=cfg.get("max_tokens_per_night") or None)
|
||||
budget.start(time.time, backend.tokens_used())
|
||||
|
||||
# `redact_secrets` was declared in DEFAULTS but nothing ever read it —
|
||||
# honor the user's choice (it's their config), but never silently (a loud
|
||||
# report note is appended below once `report` exists). Computed once, up
|
||||
# front, since it covers the cross-night task archive (state.json, via
|
||||
# add_to_archive further down — a real gap: raw harvested intent/context
|
||||
# text was persisting there indefinitely, outside the staging dir a user
|
||||
# is ever told to review) as well as everything written to staging.
|
||||
redact_enabled = bool(cfg.get("redact_secrets", True))
|
||||
_maybe_redact = redact_secrets if redact_enabled else (lambda v: v)
|
||||
|
||||
# ── live skill/memory docs ───────────────────────────────────────────
|
||||
live_memory_path = os.path.join(project, "CLAUDE.md")
|
||||
live_skill_path = cfg.managed_skill_path()
|
||||
_progress(cfg, f"live skill: {live_skill_path}")
|
||||
raw_skill = _read(live_skill_path)
|
||||
skill = raw_skill
|
||||
memory = _read(live_memory_path)
|
||||
if not skill:
|
||||
skill = ensure_skill_scaffold(
|
||||
"", name=cfg.get("managed_skill_name", "skillopt-sleep-learned"),
|
||||
description="Preferences and procedures learned from past local agent sessions.",
|
||||
)
|
||||
target_filter = bool(
|
||||
cfg.get("target_task_filter", True)
|
||||
and cfg.get("target_skill_path", "")
|
||||
and raw_skill
|
||||
)
|
||||
|
||||
# ── 1+2. harvest + mine (unless seed_tasks injected) ─────────────────
|
||||
digests: List[SessionDigest] = []
|
||||
if seed_tasks is not None:
|
||||
tasks = seed_tasks
|
||||
n_sessions = 0
|
||||
_progress(cfg, f"using {len(tasks)} seeded tasks")
|
||||
else:
|
||||
since = state.last_harvest_for(project)
|
||||
# On first run (no prior harvest), apply lookback_hours so we don't
|
||||
# scan the entire transcript history and trigger massive LLM mining.
|
||||
if since is None:
|
||||
lookback_hours = cfg.get("lookback_hours", 72)
|
||||
if lookback_hours is not None and lookback_hours > 0:
|
||||
ref_time = clock if clock is not None else time.time()
|
||||
cutoff = ref_time - lookback_hours * 3600
|
||||
since = _now_iso(cutoff)
|
||||
max_tasks = cfg.get("max_tasks_per_night", 40)
|
||||
max_sessions = cfg.get("max_sessions_per_night", 0) or max_tasks * 3
|
||||
candidate_limit = max_tasks
|
||||
if target_filter:
|
||||
candidate_limit = max(max_tasks, max_tasks * 3)
|
||||
_progress(
|
||||
cfg,
|
||||
f"harvest start: source={cfg.get('transcript_source')} max_sessions={max_sessions}",
|
||||
)
|
||||
digests = harvest_for_config(
|
||||
cfg,
|
||||
since_iso=since,
|
||||
limit=max_sessions,
|
||||
)
|
||||
n_sessions = len(digests)
|
||||
_progress(cfg, f"harvest done: sessions={n_sessions}")
|
||||
# When a real backend is configured, use it to mine checkable tasks from
|
||||
# the transcripts (rubric/rule judges); otherwise fall back to the
|
||||
# heuristic miner (no API, no checkable reference).
|
||||
llm_miner = None
|
||||
if cfg.get("backend", "mock") != "mock" and cfg.get("llm_mine", True):
|
||||
try:
|
||||
from skillopt_sleep.llm_miner import make_llm_miner
|
||||
llm_miner = make_llm_miner(
|
||||
backend,
|
||||
max_sessions=max_sessions,
|
||||
max_tasks=candidate_limit,
|
||||
)
|
||||
except Exception:
|
||||
llm_miner = None
|
||||
_progress(
|
||||
cfg,
|
||||
f"mine start: max_tasks={max_tasks} candidate_limit={candidate_limit} "
|
||||
f"llm_mine={llm_miner is not None} target_filter={target_filter}",
|
||||
)
|
||||
tasks = mine(
|
||||
digests,
|
||||
max_tasks=max_tasks,
|
||||
candidate_limit=candidate_limit,
|
||||
holdout_fraction=cfg.get("holdout_fraction", 0.34),
|
||||
seed=cfg.get("seed", 42),
|
||||
llm_miner=llm_miner,
|
||||
target_skill_text=raw_skill if target_filter else "",
|
||||
target_skill_path=live_skill_path if target_filter else "",
|
||||
)
|
||||
_progress(cfg, f"mine done: tasks={len(tasks)}")
|
||||
|
||||
report = SleepReport(
|
||||
night=night, project=project, started_at=started,
|
||||
n_sessions=n_sessions, n_tasks=len(tasks),
|
||||
)
|
||||
if not redact_enabled:
|
||||
report.notes.append(
|
||||
"redact_secrets is disabled (redact_secrets=false in config) — "
|
||||
"the task archive (state.json), staged files, and diagnostics are "
|
||||
"NOT scrubbed of secret-looking text"
|
||||
)
|
||||
if cfg.get("replay_mode", "mock") != "mock":
|
||||
# replay_mode: "fresh" (worktree replay) is declared in config.py's
|
||||
# DEFAULTS docstring but was never implemented anywhere in this
|
||||
# engine -- it only ever fed a cosmetic label in the report. Every
|
||||
# replay this cycle actually runs is the sandboxed "mock" prompt
|
||||
# replay regardless of this setting; say so rather than let the
|
||||
# report's "replay: fresh" line imply real worktree isolation.
|
||||
report.notes.append(
|
||||
f"replay_mode={cfg.get('replay_mode')!r} is not implemented in this "
|
||||
f"engine (no worktree replay exists) — replay ran as 'mock' regardless"
|
||||
)
|
||||
|
||||
if not tasks:
|
||||
report.ended_at = _now_iso(clock)
|
||||
report.notes.append("no tasks mined — nothing to consolidate")
|
||||
state.set_last_harvest(project, started)
|
||||
state.record_night({"night": night, "accepted": False, "n_tasks": 0})
|
||||
if not dry_run:
|
||||
state.save()
|
||||
staging_dir = ""
|
||||
return CycleOutcome(report, staging_dir, False, [])
|
||||
|
||||
# ── 3+4. replay + consolidate (gate), with opt-in dream + recall ──────
|
||||
# recall pulls similar past tasks from the persisted archive; dream_rollouts
|
||||
# / dream_factor enrich the training signal. With the defaults (recall_k=0,
|
||||
# dream_rollouts=1, dream_factor=0) this is exactly the prior single-shot
|
||||
# consolidate — behavior is unchanged unless the user opts in.
|
||||
_progress(cfg, "consolidate start")
|
||||
recall_k = int(cfg.get("recall_k", 0) or 0)
|
||||
history_tasks = []
|
||||
if recall_k > 0:
|
||||
history_tasks = [TaskRecord.from_dict(d) for d in state.task_archive()]
|
||||
|
||||
# Size dream_rollouts to what's left of max_tokens_per_night, reusing the
|
||||
# plan_depth heuristic (rollouts affordable per task, given ~1.5k
|
||||
# tokens/rollout) that was already written for this purpose but never
|
||||
# called from the production path. No-op when no budget is configured
|
||||
# (plan_depth returns the configured default unchanged).
|
||||
configured_rollouts = int(cfg.get("dream_rollouts", 1) or 1)
|
||||
dream_rollouts = configured_rollouts
|
||||
if budget.max_tokens:
|
||||
already_spent = budget.tokens_spent(backend.tokens_used())
|
||||
remaining_budget = Budget(max_tokens=max(0, budget.max_tokens - already_spent))
|
||||
_, dream_rollouts = plan_depth(
|
||||
remaining_budget, n_tasks=len(tasks), default_k=configured_rollouts,
|
||||
)
|
||||
dream_rollouts = min(configured_rollouts, dream_rollouts)
|
||||
if dream_rollouts < configured_rollouts:
|
||||
report.notes.append(
|
||||
f"budget: capped dream_rollouts {configured_rollouts}->{dream_rollouts} "
|
||||
f"to stay within max_tokens_per_night={budget.max_tokens} "
|
||||
f"(already spent {already_spent} on harvest/mine)"
|
||||
)
|
||||
|
||||
result = dream_consolidate(
|
||||
backend, tasks, skill, memory,
|
||||
history_tasks=history_tasks,
|
||||
recall_k=recall_k,
|
||||
dream_rollouts=dream_rollouts,
|
||||
dream_factor=int(cfg.get("dream_factor", 0) or 0),
|
||||
edit_budget=cfg.get("edit_budget", 4),
|
||||
gate_metric=cfg.get("gate_metric", "mixed"),
|
||||
gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5),
|
||||
gate_mode=cfg.get("gate_mode", "on"),
|
||||
evolve_skill=cfg.get("evolve_skill", True),
|
||||
evolve_memory=cfg.get("evolve_memory", True),
|
||||
night=night,
|
||||
)
|
||||
# archive tonight's real (non-dream) tasks so future nights can recall
|
||||
# them. This persists to ~/.skillopt-sleep/state.json indefinitely,
|
||||
# outside the staging dir a user is ever told to review — redact it the
|
||||
# same as everything else (see redact_enabled above).
|
||||
state.add_to_archive([_maybe_redact(t.to_dict()) for t in tasks if t.origin != "dream"])
|
||||
_progress(
|
||||
cfg,
|
||||
f"consolidate done: gate={result.gate_action} accepted={result.accepted} "
|
||||
f"edits={len(result.applied_edits)} rejected={len(result.rejected_edits)}",
|
||||
)
|
||||
|
||||
report.n_replayed = len(tasks)
|
||||
report.baseline_score = result.baseline_score
|
||||
report.candidate_score = result.candidate_score
|
||||
report.accepted = result.accepted
|
||||
report.gate_action = result.gate_action
|
||||
report.no_edits_reason = getattr(result, "no_edits_reason", "")
|
||||
report.edits = result.applied_edits
|
||||
report.rejected_edits = result.rejected_edits
|
||||
report.tokens_used = backend.tokens_used()
|
||||
report.ended_at = _now_iso(clock)
|
||||
if budget.max_tokens and budget.exhausted(tokens_now=backend.tokens_used(), clock_fn=time.time):
|
||||
report.notes.append(
|
||||
f"budget: max_tokens_per_night ({budget.max_tokens}) reached this night "
|
||||
f"(spent {backend.tokens_used()}); dream_rollouts were already capped above "
|
||||
f"— lower max_tasks_per_night or raise the budget for deeper nights"
|
||||
)
|
||||
|
||||
# ── 5. stage (unless dry-run) ────────────────────────────────────────
|
||||
staging_dir = ""
|
||||
adopted = False
|
||||
adopted_paths: List[str] = []
|
||||
if not dry_run:
|
||||
_progress(cfg, "staging start")
|
||||
report_md = _render_report_md(report, cfg)
|
||||
proposed_skill = result.new_skill if (cfg.get("evolve_skill") and result.accepted) else None
|
||||
proposed_memory = result.new_memory if (cfg.get("evolve_memory") and result.accepted) else None
|
||||
# redact_enabled / _maybe_redact are computed once, up front (see
|
||||
# above) so they also cover the task archive.
|
||||
staging_dir = write_staging(
|
||||
project,
|
||||
report=report,
|
||||
proposed_skill=proposed_skill,
|
||||
proposed_memory=proposed_memory,
|
||||
live_skill_path=live_skill_path,
|
||||
live_memory_path=live_memory_path,
|
||||
report_md=report_md,
|
||||
redact=redact_enabled,
|
||||
)
|
||||
# Observability: persist per-task held-out evidence + optimizer/codex errors so a
|
||||
# 0.0->0.0 night self-explains (empty responses vs failing checks vs no edits) — the
|
||||
# cycle previously captured none of this, making the gate a black box (#learning-stall).
|
||||
try:
|
||||
import json as _json
|
||||
# Backend stderr / optimizer replies / task responses can carry
|
||||
# credentials (e.g. a codex 401 stderr dump), so scrub secret-looking
|
||||
# substrings before persisting them to the on-disk diagnostics
|
||||
# (unless the user explicitly disabled redact_secrets above).
|
||||
_diag_path = os.path.join(staging_dir, "diagnostics.json")
|
||||
with open(_diag_path, "w", encoding="utf-8") as _fh:
|
||||
_json.dump({
|
||||
"night": night,
|
||||
"backend": cfg.get("backend"),
|
||||
"gate_mode": cfg.get("gate_mode"),
|
||||
"n_tasks": len(tasks),
|
||||
"baseline_score": result.baseline_score,
|
||||
"candidate_score": result.candidate_score,
|
||||
"accepted": result.accepted,
|
||||
"n_applied_edits": len(result.applied_edits),
|
||||
"n_rejected_edits": len(result.rejected_edits),
|
||||
"call_error": _maybe_redact(getattr(result, "call_error", "")),
|
||||
"reflect_raw_head": _maybe_redact(
|
||||
(getattr(result, "reflect_raw", "") or "")[:1200]
|
||||
),
|
||||
"holdout_detail": _maybe_redact(getattr(result, "holdout_detail", [])),
|
||||
}, _fh, indent=2)
|
||||
# diagnostics.json is plaintext, real session-derived content —
|
||||
# tighten from the process umask default (see staging.py's
|
||||
# _secure_file for the same reasoning applied to its siblings).
|
||||
try:
|
||||
os.chmod(_diag_path, 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
state.set_last_harvest(project, started)
|
||||
state.record_night({
|
||||
"night": night, "accepted": result.accepted,
|
||||
"baseline": result.baseline_score, "candidate": result.candidate_score,
|
||||
"n_tasks": len(tasks), "staging": staging_dir,
|
||||
})
|
||||
# ── 6. adopt (opt-in) ────────────────────────────────────────────
|
||||
if cfg.get("auto_adopt") and result.accepted:
|
||||
adopted_paths = adopt_staging(staging_dir)
|
||||
adopted = bool(adopted_paths)
|
||||
state.save()
|
||||
|
||||
return CycleOutcome(report, staging_dir, adopted, adopted_paths)
|
||||
138
engineering/skillopt-sleep/skillopt_sleep/dream.py
Normal file
138
engineering/skillopt-sleep/skillopt_sleep/dream.py
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
"""SkillOpt-Sleep — dream + associative recall for nightly consolidation.
|
||||
|
||||
Two opt-in mechanisms (both default OFF, so the cycle is unchanged unless the
|
||||
user enables them) that the deployment experiments validated:
|
||||
|
||||
* dream rollouts — run each task K times and learn from the good-vs-bad
|
||||
contrast (set ``dream_rollouts > 1``). Stronger signal than one failure.
|
||||
* associative recall — each night, pull the K past tasks most similar to
|
||||
tonight's new ones into the dream (set ``recall_k > 0``). Replays relevant
|
||||
experience without re-running the whole history.
|
||||
|
||||
``dream_consolidate`` wires recall + synthetic augmentation + multi-rollout
|
||||
consolidation and is called by BOTH the shipped plugin cycle and the benchmark
|
||||
experiment harness, so the reported numbers exercise the exact code the plugin
|
||||
runs. Pure-stdlib, zero research/private dependency.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import List, Optional
|
||||
|
||||
from skillopt_sleep.consolidate import ConsolidationResult, consolidate
|
||||
from skillopt_sleep.types import TaskRecord
|
||||
|
||||
|
||||
# ── synthetic augmentation ("dream up" variants of today's tasks) ─────────────
|
||||
|
||||
_WRAPPERS = [
|
||||
"(quick one) {q}",
|
||||
"Please handle this request: {q}",
|
||||
"For the daily report: {q}",
|
||||
]
|
||||
|
||||
|
||||
def dream_augment(real_tasks: List[TaskRecord], *, factor: int = 1) -> List[TaskRecord]:
|
||||
"""Create synthetic TRAIN variants of real tasks (origin='dream').
|
||||
|
||||
A light, deterministic rephrasing. Dream tasks are training-only — they
|
||||
carry split='train' and never enter the val/test slices the gate scores on.
|
||||
"""
|
||||
out: List[TaskRecord] = []
|
||||
for t in real_tasks:
|
||||
for k in range(max(0, factor)):
|
||||
w = _WRAPPERS[k % len(_WRAPPERS)]
|
||||
out.append(TaskRecord(
|
||||
id=f"{t.id}_dream{k}", project=t.project,
|
||||
intent=w.format(q=t.intent), context_excerpt=t.context_excerpt,
|
||||
reference_kind=t.reference_kind, reference=t.reference,
|
||||
judge=dict(t.judge), system=t.system,
|
||||
tags=list(t.tags) + ["dream"], split="train",
|
||||
origin="dream", derived_from=t.id,
|
||||
))
|
||||
return out
|
||||
|
||||
|
||||
# ── associative recall (experience replay of similar past tasks) ──────────────
|
||||
|
||||
def _tokens(text: str) -> set:
|
||||
return {w for w in re.findall(r"[a-z0-9]+", (text or "").lower()) if len(w) > 2}
|
||||
|
||||
|
||||
def recall_similar(new_tasks: List[TaskRecord], history: List[TaskRecord],
|
||||
k: int) -> List[TaskRecord]:
|
||||
"""Return the ``k`` historical tasks most lexically similar to any of
|
||||
tonight's ``new_tasks`` (max Jaccard token overlap). Recalled tasks are
|
||||
returned as training material (split='train'); deterministic, stdlib-only.
|
||||
"""
|
||||
if not history or k <= 0 or not new_tasks:
|
||||
return []
|
||||
new_tok = [_tokens(t.intent) for t in new_tasks]
|
||||
new_ids = {t.id for t in new_tasks}
|
||||
scored = []
|
||||
for h in history:
|
||||
if h.id in new_ids:
|
||||
continue
|
||||
ht = _tokens(h.intent)
|
||||
if not ht:
|
||||
continue
|
||||
sim = max(((len(ht & nt) / len(ht | nt)) if (ht | nt) else 0.0) for nt in new_tok)
|
||||
scored.append((sim, h.id, h))
|
||||
scored.sort(key=lambda x: (-x[0], x[1]))
|
||||
out = []
|
||||
for sim, _id, h in scored[:max(0, k)]:
|
||||
if sim <= 0.0:
|
||||
break
|
||||
# recall as training material; copy so the source archive is untouched
|
||||
out.append(TaskRecord(
|
||||
id=f"recall:{h.id}", project=h.project, intent=h.intent,
|
||||
context_excerpt=h.context_excerpt, reference_kind=h.reference_kind,
|
||||
reference=h.reference, judge=dict(h.judge), system=h.system,
|
||||
tags=list(h.tags) + ["recall"], split="train", origin="real",
|
||||
derived_from=h.id,
|
||||
))
|
||||
return out
|
||||
|
||||
|
||||
# ── the shared nightly consolidation step ─────────────────────────────────────
|
||||
|
||||
def dream_consolidate(
|
||||
backend,
|
||||
tasks: List[TaskRecord],
|
||||
skill: str,
|
||||
memory: str,
|
||||
*,
|
||||
history_tasks: Optional[List[TaskRecord]] = None,
|
||||
recall_k: int = 0,
|
||||
dream_rollouts: int = 1,
|
||||
dream_factor: int = 0,
|
||||
edit_budget: int = 4,
|
||||
gate_metric: str = "mixed",
|
||||
gate_mixed_weight: float = 0.5,
|
||||
gate_mode: str = "on",
|
||||
evolve_skill: bool = True,
|
||||
evolve_memory: bool = True,
|
||||
night: int = 1,
|
||||
) -> ConsolidationResult:
|
||||
"""Recall similar past experience + dream synthetic variants, then run one
|
||||
gated consolidation epoch over the enlarged training pool.
|
||||
|
||||
``tasks`` is the split-tagged pool for tonight (train + val); recall and
|
||||
augmentation only enlarge the TRAIN split, so the val slice the gate scores
|
||||
on is never polluted. With ``recall_k=0`` and ``dream_rollouts=1`` (the
|
||||
defaults) this is exactly the previous single-shot ``consolidate``.
|
||||
"""
|
||||
train = [t for t in tasks if t.split == "train"]
|
||||
enlarged = list(tasks)
|
||||
if recall_k > 0 and history_tasks:
|
||||
enlarged += recall_similar(train, history_tasks, recall_k)
|
||||
if dream_factor > 0:
|
||||
seed = [t for t in enlarged if t.split == "train" and t.origin != "dream"]
|
||||
enlarged += dream_augment(seed, factor=dream_factor)
|
||||
return consolidate(
|
||||
backend, enlarged, skill, memory,
|
||||
edit_budget=edit_budget, gate_metric=gate_metric,
|
||||
gate_mixed_weight=gate_mixed_weight, gate_mode=gate_mode,
|
||||
rollouts_k=dream_rollouts, evolve_skill=evolve_skill,
|
||||
evolve_memory=evolve_memory, night=night,
|
||||
)
|
||||
|
|
@ -0,0 +1 @@
|
|||
"""SkillOpt-Sleep experiments."""
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
"""SkillOpt-Sleep — gbrain-evals benchmark adapter.
|
||||
|
||||
Loads gbrain-evals' `skillopt-v1` benchmark (deficient skills + train/held-out
|
||||
task sets with rule-based judges) into our TaskRecord format, so we can run the
|
||||
SkillOpt-Sleep cycle against the SAME suite gbrain publishes a scorecard for:
|
||||
|
||||
docs/benchmarks/2026-06-03-skillopt.md — "4/4 skills 0 -> 1.00"
|
||||
|
||||
Each gbrain seed dir has:
|
||||
SKILL.md — the deliberately deficient starting skill
|
||||
benchmark.jsonl — training tasks {task_id, task, judge:{kind:"rule",checks}}
|
||||
held-out.jsonl — held-out tasks (same judge shape, unseen items)
|
||||
|
||||
We map:
|
||||
benchmark.jsonl -> TaskRecords with split="replay"
|
||||
held-out.jsonl -> TaskRecords with split="holdout"
|
||||
judge -> TaskRecord.judge (+ reference_kind="rule")
|
||||
|
||||
This lets us reproduce gbrain's headline result with our engine and either the
|
||||
claude or codex backend, scoring locally via skillopt_sleep.judges (no judge API).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from skillopt_sleep.types import TaskRecord
|
||||
|
||||
|
||||
SEED_DIRS = {
|
||||
"brief-writer": "seed-missing-structure",
|
||||
"thorough-analyst": "seed-verbose",
|
||||
"advisor": "seed-no-verdict",
|
||||
"quick-answerer": "seed-no-brain-first",
|
||||
}
|
||||
|
||||
|
||||
def _load_jsonl(path: str) -> List[dict]:
|
||||
out: List[dict] = []
|
||||
if not os.path.exists(path):
|
||||
return out
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
try:
|
||||
out.append(json.loads(line))
|
||||
except Exception:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def _to_task(rec: dict, *, seed: str, split: str) -> TaskRecord:
|
||||
return TaskRecord(
|
||||
id=f"{seed}:{rec.get('task_id', '')}",
|
||||
project=f"gbrain/{seed}",
|
||||
intent=str(rec.get("task", "")),
|
||||
reference_kind="rule",
|
||||
judge=rec.get("judge", {}) or {},
|
||||
tags=[f"seed:{seed}"],
|
||||
split=split,
|
||||
)
|
||||
|
||||
|
||||
def load_seed(data_root: str, seed: str, *, val_fraction: float = 0.34,
|
||||
split_seed: int = 42) -> Tuple[str, List[TaskRecord]]:
|
||||
"""Return (deficient_skill_md, tasks) for one gbrain seed.
|
||||
|
||||
Faithful split mapping:
|
||||
* gbrain held-out.jsonl -> our ``test`` (the true final measure)
|
||||
* gbrain benchmark.jsonl -> split deterministically into ``train`` + ``val``
|
||||
(val gates updates; train drives reflect)
|
||||
All tasks are origin='real' (gbrain provides no synthetic tasks).
|
||||
"""
|
||||
import hashlib
|
||||
sub = SEED_DIRS.get(seed, seed)
|
||||
seed_dir = os.path.join(data_root, sub)
|
||||
skill_path = os.path.join(seed_dir, "SKILL.md")
|
||||
skill = ""
|
||||
if os.path.exists(skill_path):
|
||||
with open(skill_path, encoding="utf-8") as f:
|
||||
skill = f.read()
|
||||
tasks: List[TaskRecord] = []
|
||||
# benchmark pool -> train/val
|
||||
val_cut = int(round(val_fraction * 100))
|
||||
for rec in _load_jsonl(os.path.join(seed_dir, "benchmark.jsonl")):
|
||||
t = _to_task(rec, seed=seed, split="train")
|
||||
bucket = int(hashlib.sha256((str(split_seed) + t.id).encode()).hexdigest(), 16) % 100
|
||||
t.split = "val" if bucket < val_cut else "train"
|
||||
tasks.append(t)
|
||||
# held-out -> test
|
||||
for rec in _load_jsonl(os.path.join(seed_dir, "held-out.jsonl")):
|
||||
tasks.append(_to_task(rec, seed=seed, split="test"))
|
||||
# guarantee a non-empty val
|
||||
if not any(t.split == "val" for t in tasks):
|
||||
train_only = [t for t in tasks if t.split == "train"]
|
||||
if train_only:
|
||||
train_only[0].split = "val"
|
||||
return skill, tasks
|
||||
|
||||
|
||||
def available_seeds(data_root: str) -> List[str]:
|
||||
return [s for s, sub in SEED_DIRS.items()
|
||||
if os.path.isdir(os.path.join(data_root, sub))]
|
||||
|
||||
|
||||
def find_data_root(explicit: str = "") -> Optional[str]:
|
||||
"""Locate eval/data/skillopt-v1 from common clone locations."""
|
||||
cands = [explicit] if explicit else []
|
||||
cands += [
|
||||
os.path.expanduser("~/git/gbrain-evals/eval/data/skillopt-v1"),
|
||||
"/tmp/gbrain-evals/eval/data/skillopt-v1",
|
||||
os.path.expanduser("~/gbrain-evals/eval/data/skillopt-v1"),
|
||||
]
|
||||
for c in cands:
|
||||
if c and os.path.isdir(c):
|
||||
return c
|
||||
return None
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
"""SkillOpt-Sleep — persona task fixtures for the validation experiment.
|
||||
|
||||
Each persona is a list of TaskRecords with EXACT checkable references and a
|
||||
`rule:<key>` tag naming the single skill rule that makes the task solvable
|
||||
(consumed by MockBackend). This lets the experiment prove — deterministically,
|
||||
with no API — that nightly consolidation lifts a held-out score and that the
|
||||
gate blocks regressions.
|
||||
|
||||
Personas mirror the user's framing: programmer / researcher / analyst.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List
|
||||
|
||||
from skillopt_sleep.types import TaskRecord
|
||||
|
||||
|
||||
def _t(i, intent, ref, rule, project="/personas/demo", outcome="fail") -> TaskRecord:
|
||||
return TaskRecord(
|
||||
id=f"persona_{rule}_{i}",
|
||||
project=project,
|
||||
intent=intent,
|
||||
context_excerpt="",
|
||||
attempted_solution="",
|
||||
outcome=outcome,
|
||||
reference_kind="exact",
|
||||
reference=ref,
|
||||
tags=[f"rule:{rule}"],
|
||||
source_sessions=[f"sess_{i}"],
|
||||
)
|
||||
|
||||
|
||||
def researcher_persona() -> List[TaskRecord]:
|
||||
"""Researcher who always wants arXiv ids wrapped in <answer> tags."""
|
||||
items = [
|
||||
("Give me the arXiv id for the SkillOpt paper", "arXiv:2605.23904"),
|
||||
("What's the arXiv id of the Attention paper?", "arXiv:1706.03762"),
|
||||
("arXiv id for the GAN paper?", "arXiv:1406.2661"),
|
||||
("arXiv id for BERT?", "arXiv:1810.04805"),
|
||||
("arXiv id for the ResNet paper?", "arXiv:1512.03385"),
|
||||
("arXiv id for the Adam optimizer paper?", "arXiv:1412.6980"),
|
||||
("arXiv id for Dropout?", "arXiv:1207.0580"),
|
||||
("arXiv id for the Transformer-XL paper?", "arXiv:1901.02860"),
|
||||
("arXiv id for word2vec?", "arXiv:1301.3781"),
|
||||
("arXiv id for the VAE paper?", "arXiv:1312.6114"),
|
||||
("arXiv id for batch norm?", "arXiv:1502.03167"),
|
||||
("arXiv id for GPT-3?", "arXiv:2005.14165"),
|
||||
]
|
||||
# Both rules required: format the id (arxiv-id) AND wrap in answer tags.
|
||||
out: List[TaskRecord] = []
|
||||
for i, (q, a) in enumerate(items):
|
||||
t = _t(i, q, a, "wrap-answer")
|
||||
t.tags = ["rule:wrap-answer", "rule:arxiv-id"]
|
||||
out.append(t)
|
||||
return out
|
||||
|
||||
|
||||
def programmer_persona() -> List[TaskRecord]:
|
||||
"""Programmer who wants imperative-mood commit subjects."""
|
||||
items = [
|
||||
("commit message for adding a login form", "Add login form"),
|
||||
("commit message for fixing the null pointer bug", "Fix null pointer in parser"),
|
||||
("commit message for updating the README", "Update README"),
|
||||
("commit message for removing dead code", "Remove dead code"),
|
||||
("commit message for bumping the version", "Bump version to 1.2.0"),
|
||||
("commit message for refactoring the auth module", "Refactor auth module"),
|
||||
("commit message for adding tests", "Add unit tests for scheduler"),
|
||||
("commit message for fixing the CI pipeline", "Fix CI pipeline"),
|
||||
]
|
||||
return [_t(i, q, a, "commit-imperative") for i, (q, a) in enumerate(items)]
|
||||
|
||||
|
||||
def harmful_edit_task() -> TaskRecord:
|
||||
"""A task whose 'fix' is a known-bad rule; used to prove the gate rejects
|
||||
regressions. The MockBackend proposes the harmful rule on this failure,
|
||||
but applying it does NOT raise the held-out score, so the gate must reject.
|
||||
"""
|
||||
t = _t(99, "answer this freely", "THIS_WILL_NOT_MATCH", "__harmful__")
|
||||
t.reference = "an-answer-that-the-harmful-rule-cannot-produce"
|
||||
return t
|
||||
|
||||
|
||||
PERSONAS = {
|
||||
"researcher": researcher_persona,
|
||||
"programmer": programmer_persona,
|
||||
}
|
||||
132
engineering/skillopt-sleep/skillopt_sleep/experiments/report.py
Normal file
132
engineering/skillopt-sleep/skillopt_sleep/experiments/report.py
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
"""SkillOpt-Sleep — turn a sweep JSONL into a presented Markdown scorecard.
|
||||
|
||||
Usage:
|
||||
python -m skillopt_sleep.experiments.report --in docs/sleep/sweep.jsonl \
|
||||
--out docs/sleep/benchmark_report.md
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, Dict, List
|
||||
|
||||
|
||||
def _load(path: str) -> List[Dict[str, Any]]:
|
||||
rows = []
|
||||
if os.path.exists(path):
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
try:
|
||||
rows.append(json.loads(line))
|
||||
except Exception:
|
||||
pass
|
||||
return rows
|
||||
|
||||
|
||||
def _fmt_model(backend: str, model: str) -> str:
|
||||
m = model or "default"
|
||||
return f"{backend}:{m}"
|
||||
|
||||
|
||||
def render(rows: List[Dict[str, Any]]) -> str:
|
||||
direct = [r for r in rows if r.get("cfg", {}).get("kind") in ("direct", "dual") and "error" not in r]
|
||||
transfer = [r for r in rows if r.get("cfg", {}).get("kind") == "transfer" and "error" not in r]
|
||||
errors = [r for r in rows if "error" in r]
|
||||
|
||||
out: List[str] = []
|
||||
out.append("# SkillOpt-Sleep — benchmark report")
|
||||
out.append("")
|
||||
out.append("Auto-generated from `sweep.jsonl`. Benchmark: "
|
||||
"[gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1` "
|
||||
"(deficient skills, train/held-out split, local rule judge — no judge-API).")
|
||||
out.append("Held-out scores are computed by the harness, not the optimizer.")
|
||||
out.append("")
|
||||
|
||||
# ── direct improvement table ──────────────────────────────────────────
|
||||
out.append("## Direct improvement (optimize, then deploy)")
|
||||
out.append("")
|
||||
out.append("| Optimizer → Target | Seed | Held-out before | Held-out after | Nights | Tokens |")
|
||||
out.append("|---|---|---|---|---|---|")
|
||||
for r in direct:
|
||||
c = r["cfg"]
|
||||
if c.get("kind") == "dual":
|
||||
label = (f"{_fmt_model(c['optimizer_backend'], c.get('optimizer_model',''))}"
|
||||
f" → {_fmt_model(c['target_backend'], c.get('target_model',''))}")
|
||||
else:
|
||||
m = _fmt_model(c["backend"], c.get("model", ""))
|
||||
label = f"{m} → {m}"
|
||||
out.append(f"| {label} | {c['seed']} | "
|
||||
f"{r['baseline']:.2f} | **{r['after']:.2f}** | {c['nights']} | "
|
||||
f"{r.get('tokens','?')} |")
|
||||
if direct:
|
||||
n_imp = sum(1 for r in direct if r.get("improved"))
|
||||
out.append("")
|
||||
out.append(f"**{n_imp}/{len(direct)} configurations improved on held-out.**")
|
||||
out.append("")
|
||||
|
||||
# ── transfer table ────────────────────────────────────────────────────
|
||||
if transfer:
|
||||
out.append("## Cross-model transfer (optimize on SOURCE, deploy frozen on TARGET)")
|
||||
out.append("")
|
||||
out.append("The price-difference story: spend cheap tokens optimizing overnight, "
|
||||
"then deploy the frozen skill on any model with no further optimization.")
|
||||
out.append("")
|
||||
out.append("| Source (optimizer) | Target (deploy) | Seed | Target baseline | Transferred | Gain |")
|
||||
out.append("|---|---|---|---|---|---|")
|
||||
for r in transfer:
|
||||
c = r["cfg"]
|
||||
s = _fmt_model(c["source_backend"], c.get("source_model", ""))
|
||||
t = _fmt_model(c["target_backend"], c.get("target_model", ""))
|
||||
out.append(f"| {s} | {t} | {c['seed']} | {r['baseline_target']:.2f} | "
|
||||
f"**{r['transferred']:.2f}** | {r['transfer_gain']:+.2f} |")
|
||||
n_pos = sum(1 for r in transfer if r.get("transfer_gain", 0) > 0)
|
||||
out.append("")
|
||||
out.append(f"**{n_pos}/{len(transfer)} transfers were positive** "
|
||||
"(frozen skill helped a different model than it was optimized on).")
|
||||
out.append("")
|
||||
|
||||
# ── errors (honest reporting) ─────────────────────────────────────────
|
||||
if errors:
|
||||
out.append("## Configs that errored (reported, not hidden)")
|
||||
out.append("")
|
||||
for r in errors:
|
||||
out.append(f"- `{json.dumps(r['cfg'])}` → {r['error']}")
|
||||
out.append("")
|
||||
|
||||
out.append("## How to reproduce")
|
||||
out.append("")
|
||||
out.append("```bash")
|
||||
out.append("git clone https://github.com/garrytan/gbrain-evals /tmp/gbrain-evals")
|
||||
out.append("python -m skillopt_sleep.experiments.sweep --plan full \\")
|
||||
out.append(" --data-root /tmp/gbrain-evals/eval/data/skillopt-v1 --out docs/sleep/sweep.jsonl")
|
||||
out.append("python -m skillopt_sleep.experiments.report \\")
|
||||
out.append(" --in docs/sleep/sweep.jsonl --out docs/sleep/benchmark_report.md")
|
||||
out.append("```")
|
||||
out.append("")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(description="Render SkillOpt-Sleep sweep report")
|
||||
ap.add_argument("--in", dest="inp", default="docs/sleep/sweep.jsonl")
|
||||
ap.add_argument("--out", default="docs/sleep/benchmark_report.md")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
rows = _load(args.inp)
|
||||
if not rows:
|
||||
print(f"no rows in {args.inp}", file=sys.stderr)
|
||||
return 1
|
||||
md = render(rows)
|
||||
os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
|
||||
with open(args.out, "w") as f:
|
||||
f.write(md)
|
||||
print(f"wrote {args.out} ({len(rows)} rows)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
"""SkillOpt-Sleep — validation experiment.
|
||||
|
||||
Answers the question the user posed: *does nightly offline self-evolution
|
||||
actually improve the agent?* Runs deterministically with the MockBackend
|
||||
(no API key, reproducible) and is the acceptance test for the whole idea.
|
||||
|
||||
What it proves:
|
||||
1. MONOTONIC LIFT — over N sleep nights, the held-out score rises from a
|
||||
baseline (empty skill/memory) toward 1.0 as the gate accepts the
|
||||
general rules the persona's tasks require.
|
||||
2. GATE SAFETY — an injected harmful edit is REJECTED (held-out score does
|
||||
not improve), so a bad nightly proposal can never be adopted.
|
||||
3. PLUMBING — harvest->mine->replay->consolidate->stage->adopt all run and
|
||||
the adopted artifact, re-scored, retains the lift.
|
||||
|
||||
Run:
|
||||
python -m skillopt_sleep.experiments.run_experiment
|
||||
python -m skillopt_sleep.experiments.run_experiment --persona programmer --nights 3
|
||||
python -m skillopt_sleep.experiments.run_experiment --backend anthropic # real lift
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from typing import List
|
||||
|
||||
from skillopt_sleep.backend import get_backend
|
||||
from skillopt_sleep.consolidate import consolidate
|
||||
from skillopt_sleep.experiments.personas import (
|
||||
PERSONAS,
|
||||
harmful_edit_task,
|
||||
researcher_persona,
|
||||
)
|
||||
from skillopt_sleep.memory import ensure_skill_scaffold
|
||||
from skillopt_sleep.replay import aggregate_scores, replay_batch
|
||||
from skillopt_sleep.types import TaskRecord
|
||||
|
||||
|
||||
def _score_holdout(backend, tasks: List[TaskRecord], skill: str, memory: str,
|
||||
metric: str = "mixed", w: float = 0.5) -> float:
|
||||
from skillopt_sleep.consolidate import select_gate_score
|
||||
# the persona experiment uses a 2-way split (train/val, no test); score on val
|
||||
holdout = [t for t in tasks if t.split in ("val", "holdout")] or tasks
|
||||
pairs = replay_batch(backend, holdout, skill, memory)
|
||||
h, s = aggregate_scores(pairs)
|
||||
return select_gate_score(h, s, metric, w)
|
||||
|
||||
|
||||
def run(persona: str = "researcher", nights: int = 4, backend_name: str = "mock",
|
||||
edit_budget: int = 4, seed: int = 42, model: str = "", codex_path: str = "",
|
||||
limit_tasks: int = 0) -> dict:
|
||||
from skillopt_sleep.mine import assign_splits
|
||||
|
||||
make = PERSONAS.get(persona, researcher_persona)
|
||||
items = make()
|
||||
if limit_tasks and limit_tasks < len(items):
|
||||
items = items[:limit_tasks]
|
||||
tasks = assign_splits(items, holdout_fraction=0.34, seed=seed)
|
||||
backend = get_backend(backend_name, model=model, codex_path=codex_path)
|
||||
is_mock = (backend.name == "mock")
|
||||
|
||||
# start from an empty managed skill + empty memory
|
||||
skill = ensure_skill_scaffold("", name="skillopt-sleep-learned",
|
||||
description="Learned preferences.")
|
||||
memory = ""
|
||||
|
||||
baseline = _score_holdout(backend, tasks, skill, memory)
|
||||
trace = [{"night": 0, "holdout_score": round(baseline, 4), "action": "baseline",
|
||||
"n_edits": 0}]
|
||||
|
||||
for night in range(1, nights + 1):
|
||||
res = consolidate(
|
||||
backend, tasks, skill, memory,
|
||||
edit_budget=edit_budget, gate_metric="mixed", gate_mixed_weight=0.5,
|
||||
evolve_skill=True, evolve_memory=True, night=night,
|
||||
)
|
||||
if res.accepted:
|
||||
skill, memory = res.new_skill, res.new_memory
|
||||
trace.append({
|
||||
"night": night,
|
||||
"holdout_score": round(res.candidate_score, 4),
|
||||
"action": res.gate_action,
|
||||
"accepted": res.accepted,
|
||||
"n_edits": len(res.applied_edits),
|
||||
"edits": [e.content for e in res.applied_edits],
|
||||
"n_rejected": len(res.rejected_edits),
|
||||
})
|
||||
# converged: stop early if perfect
|
||||
if res.candidate_score >= 0.999:
|
||||
break
|
||||
|
||||
after = _score_holdout(backend, tasks, skill, memory)
|
||||
|
||||
# ── gate-safety probe (mock only; it relies on the mock's known bad rule) ──
|
||||
harmful_rejected = None
|
||||
if is_mock:
|
||||
harmful_tasks = assign_splits([harmful_edit_task()] + make()[:3],
|
||||
holdout_fraction=0.5, seed=seed)
|
||||
_ = _score_holdout(backend, harmful_tasks, skill, memory)
|
||||
res_h = consolidate(backend, harmful_tasks, skill, memory,
|
||||
edit_budget=edit_budget, gate_metric="mixed",
|
||||
evolve_skill=True, evolve_memory=False, night=nights + 1)
|
||||
harmful_rule_text = get_backend("mock").RULE_TEXT["__harmful__"] # type: ignore[attr-defined]
|
||||
harmful_rejected = (harmful_rule_text not in res_h.new_skill)
|
||||
|
||||
result = {
|
||||
"persona": persona,
|
||||
"backend": backend.name,
|
||||
"model": model or "(default)",
|
||||
"n_tasks": len(tasks),
|
||||
"nights_run": len(trace) - 1,
|
||||
"baseline_holdout": round(baseline, 4),
|
||||
"after_holdout": round(after, 4),
|
||||
"lift": round(after - baseline, 4),
|
||||
"improved": after > baseline,
|
||||
"gate_blocks_harmful": harmful_rejected, # None for real backends
|
||||
"tokens_used": backend.tokens_used(),
|
||||
"final_skill_excerpt": skill[-500:],
|
||||
"trace": trace,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def _assert(cond: bool, msg: str) -> None:
|
||||
if not cond:
|
||||
print(f"FAIL: {msg}")
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(description="SkillOpt-Sleep validation experiment")
|
||||
ap.add_argument("--persona", default="researcher", choices=list(PERSONAS.keys()))
|
||||
ap.add_argument("--nights", type=int, default=4)
|
||||
ap.add_argument("--backend", default="mock", choices=["mock", "claude", "codex", "copilot"])
|
||||
ap.add_argument("--model", default="", help="backend model override")
|
||||
ap.add_argument("--codex-path", default="", help="path to the real @openai/codex binary")
|
||||
ap.add_argument("--edit-budget", type=int, default=4)
|
||||
ap.add_argument("--limit-tasks", type=int, default=0, help="cap #tasks (control API cost)")
|
||||
ap.add_argument("--json", action="store_true")
|
||||
ap.add_argument("--assert-improves", action="store_true",
|
||||
help="exit nonzero unless lift>0 (and, for mock, gate blocks harmful edit)")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
res = run(args.persona, nights=args.nights, backend_name=args.backend,
|
||||
edit_budget=args.edit_budget, model=args.model,
|
||||
codex_path=args.codex_path, limit_tasks=args.limit_tasks)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(res, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"=== SkillOpt-Sleep experiment: persona={res['persona']} "
|
||||
f"backend={res['backend']} model={res['model']} ===")
|
||||
print(f"tasks: {res['n_tasks']} tokens(approx): {res['tokens_used']}")
|
||||
print(f"baseline held-out : {res['baseline_holdout']}")
|
||||
print(f"after held-out : {res['after_holdout']} (lift {res['lift']:+.4f})")
|
||||
if res["gate_blocks_harmful"] is not None:
|
||||
print(f"gate blocks harmful edit: {res['gate_blocks_harmful']}")
|
||||
print("trace:")
|
||||
for row in res["trace"]:
|
||||
edits = "; ".join(row.get("edits", []))[:80]
|
||||
print(f" night {row['night']}: holdout={row['holdout_score']} "
|
||||
f"{row['action']} (+{row['n_edits']} edits) {edits}")
|
||||
|
||||
if args.assert_improves:
|
||||
_assert(res["improved"], "held-out score did not improve")
|
||||
if res["gate_blocks_harmful"] is not None:
|
||||
_assert(res["gate_blocks_harmful"], "gate failed to block harmful edit")
|
||||
print("\nPASS: nightly consolidation improves held-out score AND gate blocks regressions.")
|
||||
else:
|
||||
print("\nPASS: nightly consolidation improves held-out score (real backend).")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -0,0 +1,209 @@
|
|||
"""SkillOpt-Sleep — run the gbrain-evals skillopt-v1 benchmark with our engine.
|
||||
|
||||
Reproduces gbrain's "Result 1 — skills measurably improve" scorecard
|
||||
(docs/benchmarks/2026-06-03-skillopt.md) using SkillOpt-Sleep's
|
||||
consolidate() loop and either the claude or codex backend.
|
||||
|
||||
For each deficient seed skill:
|
||||
1. score the held-out tasks with the ORIGINAL skill -> before
|
||||
2. run N consolidation nights on the training tasks (gated) -> evolve skill
|
||||
3. score the held-out tasks with the EVOLVED skill -> after
|
||||
|
||||
Held-out scoring is done locally by the rule judge (no judge API). Only the
|
||||
agent's `attempt` (and the optimizer's `reflect`) spend tokens.
|
||||
|
||||
Usage:
|
||||
python -m skillopt_sleep.experiments.run_gbrain --backend mock
|
||||
python -m skillopt_sleep.experiments.run_gbrain --backend claude --seeds brief-writer --nights 2
|
||||
python -m skillopt_sleep.experiments.run_gbrain --backend codex --data-root /tmp/gbrain-evals/eval/data/skillopt-v1
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from skillopt_sleep.backend import build_backend, get_backend
|
||||
from skillopt_sleep.consolidate import consolidate, select_gate_score
|
||||
from skillopt_sleep.experiments.gbrain_bench import (
|
||||
available_seeds,
|
||||
find_data_root,
|
||||
load_seed,
|
||||
)
|
||||
from skillopt_sleep.replay import aggregate_scores, replay_batch
|
||||
|
||||
|
||||
def _score(backend, tasks, skill, memory, split="test", metric="mixed", w=0.5):
|
||||
sub = [t for t in tasks if t.split == split]
|
||||
if not sub: # fall back to val, then everything, so we never score on nothing
|
||||
sub = [t for t in tasks if t.split == "val"] or tasks
|
||||
pairs = replay_batch(backend, sub, skill, memory)
|
||||
h, s = aggregate_scores(pairs)
|
||||
return h, s, select_gate_score(h, s, metric, w)
|
||||
|
||||
|
||||
def run_seed(backend, seed: str, skill: str, tasks: List, *,
|
||||
nights: int = 3, edit_budget: int = 4, gate_mode: str = "on",
|
||||
slow_update: bool = True, rollouts_k: int = 1,
|
||||
limit_replay: int = 0, limit_holdout: int = 0) -> dict:
|
||||
memory = ""
|
||||
# optionally cap each split to control API cost / latency.
|
||||
# limit_replay caps train; limit_holdout caps BOTH val and test.
|
||||
if limit_replay or limit_holdout:
|
||||
train = [t for t in tasks if t.split == "train"]
|
||||
val = [t for t in tasks if t.split == "val"]
|
||||
test = [t for t in tasks if t.split == "test"]
|
||||
if limit_replay:
|
||||
train = train[:limit_replay]
|
||||
if limit_holdout:
|
||||
val = val[:limit_holdout]
|
||||
test = test[:limit_holdout]
|
||||
tasks = train + val + test
|
||||
# final measure is TEST (the gbrain held-out set); val gates internally
|
||||
bh, bs, bscore = _score(backend, tasks, skill, memory, split="test")
|
||||
trace = [{"night": 0, "test_hard": round(bh, 3), "action": "baseline"}]
|
||||
cur = skill
|
||||
first_night_skill = skill
|
||||
for night in range(1, nights + 1):
|
||||
res = consolidate(
|
||||
backend, tasks, cur, memory,
|
||||
edit_budget=edit_budget, gate_metric="mixed", gate_mixed_weight=0.5,
|
||||
gate_mode=gate_mode, rollouts_k=rollouts_k,
|
||||
evolve_skill=True, evolve_memory=False, night=night,
|
||||
)
|
||||
if res.accepted:
|
||||
cur = res.new_skill
|
||||
if night == 1:
|
||||
first_night_skill = cur
|
||||
# report the TEST score each night (independent of the val gate)
|
||||
th, _ts, _ = _score(backend, tasks, cur, memory, split="test")
|
||||
trace.append({
|
||||
"night": night,
|
||||
"val_hard": round(res.holdout_candidate, 3),
|
||||
"test_hard": round(th, 3),
|
||||
"action": res.gate_action,
|
||||
"accepted": res.accepted,
|
||||
"edits": [e.content for e in res.applied_edits],
|
||||
})
|
||||
if th >= 0.999:
|
||||
break
|
||||
|
||||
# ── SLOW UPDATE: consolidate cross-night experience into the protected
|
||||
# long-term field. Runs regardless of gate mode (it is what preserves
|
||||
# long-term memory even when the gate is OFF).
|
||||
slow_text = None
|
||||
if nights >= 2 and slow_update:
|
||||
try:
|
||||
from skillopt_sleep.slow_update import run_slow_update, replace_slow_field
|
||||
val_tasks = [t for t in tasks if t.split == "val"] or tasks
|
||||
prev_pairs = replay_batch(backend, val_tasks, first_night_skill, memory)
|
||||
curr_pairs = replay_batch(backend, val_tasks, cur, memory)
|
||||
slow_text = run_slow_update(
|
||||
backend, prev_skill=first_night_skill, curr_skill=cur,
|
||||
prev_pairs=[(t, r) for t, r in prev_pairs],
|
||||
curr_pairs=[(t, r) for t, r in curr_pairs],
|
||||
)
|
||||
if slow_text:
|
||||
cur = replace_slow_field(cur, slow_text)
|
||||
except Exception:
|
||||
slow_text = None
|
||||
|
||||
ah, as_, ascore = _score(backend, tasks, cur, memory, split="test")
|
||||
return {
|
||||
"seed": seed,
|
||||
"held_out_before": round(bh, 3),
|
||||
"held_out_after": round(ah, 3),
|
||||
"improved": ah > bh,
|
||||
"nights": len(trace) - 1,
|
||||
"trace": trace,
|
||||
"slow_update": slow_text,
|
||||
"final_skill_tail": cur[-400:],
|
||||
}
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(description="Run gbrain-evals skillopt-v1 with SkillOpt-Sleep")
|
||||
ap.add_argument("--backend", default="mock", choices=["mock", "claude", "codex"])
|
||||
ap.add_argument("--model", default="")
|
||||
ap.add_argument("--optimizer-backend", default="", help="route reflect/judge here (dual)")
|
||||
ap.add_argument("--optimizer-model", default="")
|
||||
ap.add_argument("--target-backend", default="", help="route attempt here (dual)")
|
||||
ap.add_argument("--target-model", default="")
|
||||
ap.add_argument("--codex-path", default="")
|
||||
ap.add_argument("--data-root", default="", help="path to eval/data/skillopt-v1")
|
||||
ap.add_argument("--seeds", default="", help="comma list; default = all available")
|
||||
ap.add_argument("--nights", type=int, default=3)
|
||||
ap.add_argument("--edit-budget", type=int, default=4)
|
||||
ap.add_argument("--gate", default="on", choices=["on", "off", "hard", "soft"],
|
||||
help="on/hard/soft = validation-gated; off = greedy (no hard filter)")
|
||||
ap.add_argument("--rollouts-k", type=int, default=1,
|
||||
help=">1 = multi-rollout contrastive reflection per task")
|
||||
ap.add_argument("--budget-tokens", type=int, default=0,
|
||||
help="approx token budget; auto-plans nights x rollouts when set")
|
||||
ap.add_argument("--budget-minutes", type=float, default=0.0)
|
||||
ap.add_argument("--preferences", default="", help="free-text user preferences (prior for reflect)")
|
||||
ap.add_argument("--limit-replay", type=int, default=0, help="cap #train tasks (cost control)")
|
||||
ap.add_argument("--limit-holdout", type=int, default=0, help="cap #val and #test tasks (cost control)")
|
||||
ap.add_argument("--json", action="store_true")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
data_root = find_data_root(args.data_root)
|
||||
if not data_root:
|
||||
print("ERROR: could not find eval/data/skillopt-v1. Clone gbrain-evals and pass --data-root.",
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
|
||||
seeds = [s.strip() for s in args.seeds.split(",") if s.strip()] or available_seeds(data_root)
|
||||
backend = build_backend(
|
||||
backend=args.backend, model=args.model,
|
||||
optimizer_backend=args.optimizer_backend, optimizer_model=args.optimizer_model,
|
||||
target_backend=args.target_backend, target_model=args.target_model,
|
||||
codex_path=args.codex_path, preferences=args.preferences,
|
||||
)
|
||||
|
||||
results = []
|
||||
for seed in seeds:
|
||||
skill, tasks = load_seed(data_root, seed)
|
||||
if not tasks:
|
||||
continue
|
||||
# budget auto-planning: derive nights x rollouts_k from a token budget
|
||||
nights, rollouts_k = args.nights, args.rollouts_k
|
||||
if args.budget_tokens:
|
||||
from skillopt_sleep.budget import Budget, plan_depth
|
||||
n_train = len([t for t in tasks if t.split == "train"]) or len(tasks)
|
||||
nights, rollouts_k = plan_depth(
|
||||
Budget(max_tokens=args.budget_tokens), n_tasks=n_train,
|
||||
default_nights=args.nights, default_k=args.rollouts_k,
|
||||
)
|
||||
if not args.json:
|
||||
print(f" [budget] {args.budget_tokens} tok -> nights={nights} rollouts_k={rollouts_k}")
|
||||
r = run_seed(backend, seed, skill, tasks, nights=nights,
|
||||
edit_budget=args.edit_budget, rollouts_k=rollouts_k,
|
||||
gate_mode=("off" if args.gate == "off" else "on"),
|
||||
limit_replay=args.limit_replay, limit_holdout=args.limit_holdout)
|
||||
results.append(r)
|
||||
if not args.json:
|
||||
print(f" {seed:<18} held-out {r['held_out_before']:.2f} -> {r['held_out_after']:.2f}"
|
||||
f" ({'IMPROVED' if r['improved'] else 'no change'}, {r['nights']} nights)")
|
||||
|
||||
n_improved = sum(1 for r in results if r["improved"])
|
||||
summary = {
|
||||
"benchmark": "gbrain-evals/skillopt-v1",
|
||||
"backend": backend.name,
|
||||
"model": args.model or "(default)",
|
||||
"n_seeds": len(results),
|
||||
"n_improved": n_improved,
|
||||
"tokens_used": backend.tokens_used(),
|
||||
"results": results,
|
||||
}
|
||||
if args.json:
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"\n=== {n_improved}/{len(results)} seeds improved on held-out "
|
||||
f"(backend={backend.name}, ~{backend.tokens_used()} tokens) ===")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
"""SkillOpt-Sleep — skill-transfer experiment (sleep scenario).
|
||||
|
||||
Answers: "if I optimize a skill while the agent sleeps using a CHEAP model,
|
||||
does the learned skill still help an EXPENSIVE model at deploy time?" — and the
|
||||
reverse. This is the SkillOpt paper's cross-model transfer result, reproduced
|
||||
in the sleep setting, and it is the core price-difference value proposition:
|
||||
spend cheap tokens overnight, deploy the frozen skill anywhere.
|
||||
|
||||
Protocol, per gbrain seed:
|
||||
1. baseline_target = held-out score of the DEFICIENT skill, run on TARGET model
|
||||
2. optimize the skill for N nights using the SOURCE model (attempt+reflect)
|
||||
3. transferred = held-out score of the LEARNED skill, run on TARGET model,
|
||||
with NO further optimization
|
||||
4. (reference) direct = held-out score of a skill optimized AND run on TARGET
|
||||
|
||||
Report baseline / direct / transferred, mirroring SkillOpt Table "transfer".
|
||||
|
||||
Usage:
|
||||
python -m skillopt_sleep.experiments.run_transfer \
|
||||
--source-backend claude --source-model haiku \
|
||||
--target-backend claude --target-model sonnet \
|
||||
--seeds brief-writer --nights 2
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from typing import List, Optional
|
||||
|
||||
from skillopt_sleep.backend import get_backend
|
||||
from skillopt_sleep.consolidate import consolidate, select_gate_score
|
||||
from skillopt_sleep.experiments.gbrain_bench import (
|
||||
available_seeds, find_data_root, load_seed,
|
||||
)
|
||||
from skillopt_sleep.replay import aggregate_scores, replay_batch
|
||||
|
||||
|
||||
def _holdout_hard(backend, tasks, skill, memory="") -> float:
|
||||
# transfer is measured on the true held-out TEST split
|
||||
ho = [t for t in tasks if t.split == "test"]
|
||||
if not ho:
|
||||
ho = [t for t in tasks if t.split in ("val", "holdout")] or tasks
|
||||
pairs = replay_batch(backend, ho, skill, memory)
|
||||
h, _s = aggregate_scores(pairs)
|
||||
return h
|
||||
|
||||
|
||||
def _optimize(backend, skill, tasks, *, nights, edit_budget) -> str:
|
||||
cur = skill
|
||||
for night in range(1, nights + 1):
|
||||
res = consolidate(backend, tasks, cur, "",
|
||||
edit_budget=edit_budget, gate_metric="mixed",
|
||||
evolve_skill=True, evolve_memory=False, night=night)
|
||||
if res.accepted:
|
||||
cur = res.new_skill
|
||||
if res.holdout_candidate >= 0.999:
|
||||
break
|
||||
return cur
|
||||
|
||||
|
||||
def run_seed(seed, skill, tasks, *, source, target, nights, edit_budget,
|
||||
limit_replay, limit_holdout, do_direct=True) -> dict:
|
||||
if limit_replay or limit_holdout:
|
||||
train = [t for t in tasks if t.split == "train"]
|
||||
val = [t for t in tasks if t.split == "val"]
|
||||
test = [t for t in tasks if t.split == "test"]
|
||||
if limit_replay:
|
||||
train = train[:limit_replay]
|
||||
if limit_holdout:
|
||||
val = val[:limit_holdout]
|
||||
test = test[:limit_holdout]
|
||||
tasks = train + val + test
|
||||
|
||||
baseline_target = _holdout_hard(target, tasks, skill)
|
||||
|
||||
# optimize on SOURCE, evaluate frozen skill on TARGET
|
||||
learned_on_source = _optimize(source, skill, tasks, nights=nights, edit_budget=edit_budget)
|
||||
transferred = _holdout_hard(target, tasks, learned_on_source)
|
||||
|
||||
direct = None
|
||||
if do_direct:
|
||||
learned_on_target = _optimize(target, skill, tasks, nights=nights, edit_budget=edit_budget)
|
||||
direct = _holdout_hard(target, tasks, learned_on_target)
|
||||
|
||||
return {
|
||||
"seed": seed,
|
||||
"baseline_target": round(baseline_target, 3),
|
||||
"direct_target": (round(direct, 3) if direct is not None else None),
|
||||
"transferred": round(transferred, 3),
|
||||
"transfer_gain": round(transferred - baseline_target, 3),
|
||||
"learned_skill_tail": learned_on_source[-300:],
|
||||
}
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(description="SkillOpt-Sleep cross-model transfer")
|
||||
ap.add_argument("--source-backend", default="claude")
|
||||
ap.add_argument("--source-model", default="haiku")
|
||||
ap.add_argument("--target-backend", default="claude")
|
||||
ap.add_argument("--target-model", default="sonnet")
|
||||
ap.add_argument("--codex-path", default="")
|
||||
ap.add_argument("--data-root", default="")
|
||||
ap.add_argument("--seeds", default="brief-writer")
|
||||
ap.add_argument("--nights", type=int, default=2)
|
||||
ap.add_argument("--edit-budget", type=int, default=4)
|
||||
ap.add_argument("--limit-replay", type=int, default=3)
|
||||
ap.add_argument("--limit-holdout", type=int, default=3)
|
||||
ap.add_argument("--no-direct", action="store_true", help="skip the direct reference (saves cost)")
|
||||
ap.add_argument("--json", action="store_true")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
data_root = find_data_root(args.data_root)
|
||||
if not data_root:
|
||||
print("ERROR: gbrain-evals skillopt-v1 data not found; pass --data-root", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
source = get_backend(args.source_backend, model=args.source_model, codex_path=args.codex_path)
|
||||
target = get_backend(args.target_backend, model=args.target_model, codex_path=args.codex_path)
|
||||
|
||||
seeds = [s.strip() for s in args.seeds.split(",") if s.strip()] or available_seeds(data_root)
|
||||
results = []
|
||||
for seed in seeds:
|
||||
skill, tasks = load_seed(data_root, seed)
|
||||
if not tasks:
|
||||
continue
|
||||
r = run_seed(seed, skill, tasks, source=source, target=target,
|
||||
nights=args.nights, edit_budget=args.edit_budget,
|
||||
limit_replay=args.limit_replay, limit_holdout=args.limit_holdout,
|
||||
do_direct=not args.no_direct)
|
||||
results.append(r)
|
||||
if not args.json:
|
||||
d = f" direct={r['direct_target']}" if r['direct_target'] is not None else ""
|
||||
print(f" {seed:<16} baseline={r['baseline_target']:.2f}"
|
||||
f" transferred={r['transferred']:.2f}{d}"
|
||||
f" (gain {r['transfer_gain']:+.2f})")
|
||||
|
||||
summary = {
|
||||
"experiment": "skillopt-sleep/transfer",
|
||||
"source": f"{args.source_backend}:{args.source_model}",
|
||||
"target": f"{args.target_backend}:{args.target_model}",
|
||||
"tokens_source": source.tokens_used(),
|
||||
"tokens_target": target.tokens_used(),
|
||||
"results": results,
|
||||
}
|
||||
if args.json:
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"\n=== transfer {summary['source']} -> {summary['target']}: "
|
||||
f"{sum(1 for r in results if r['transfer_gain'] > 0)}/{len(results)} positive ===")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
164
engineering/skillopt-sleep/skillopt_sleep/experiments/sweep.py
Normal file
164
engineering/skillopt-sleep/skillopt_sleep/experiments/sweep.py
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
"""SkillOpt-Sleep — benchmark sweep driver.
|
||||
|
||||
Runs many (backend, model, seed, transfer-pair) configurations SEQUENTIALLY in
|
||||
one process, appending each result to a JSONL file as it finishes. Designed to
|
||||
run unattended in the background; safe to interrupt (already-written rows
|
||||
survive) and resume (skip configs whose row already exists).
|
||||
|
||||
Then `report.py` turns the JSONL into a presented Markdown scorecard.
|
||||
|
||||
Usage:
|
||||
python -m skillopt_sleep.experiments.sweep --plan quick --out docs/sleep/sweep.jsonl
|
||||
python -m skillopt_sleep.experiments.sweep --plan full --out docs/sleep/sweep.jsonl
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from skillopt_sleep.backend import build_backend, get_backend
|
||||
from skillopt_sleep.experiments.gbrain_bench import find_data_root, load_seed
|
||||
from skillopt_sleep.experiments.run_gbrain import run_seed as bench_seed
|
||||
from skillopt_sleep.experiments.run_transfer import run_seed as transfer_seed
|
||||
|
||||
|
||||
# Plans: lists of config dicts. Kept small per-run to bound cost/latency.
|
||||
def _direct_cfg(backend, model, seed, nights=2):
|
||||
return {"kind": "direct", "backend": backend, "model": model, "seed": seed, "nights": nights}
|
||||
|
||||
|
||||
def _dual_cfg(opt_backend, opt_model, tgt_backend, tgt_model, seed, nights=2):
|
||||
# a 'direct' run on a DualBackend: strong optimizer proposes, weak target runs
|
||||
return {"kind": "dual", "optimizer_backend": opt_backend, "optimizer_model": opt_model,
|
||||
"target_backend": tgt_backend, "target_model": tgt_model, "seed": seed, "nights": nights}
|
||||
|
||||
|
||||
def _transfer_cfg(sb, sm, tb, tm, seed, nights=2):
|
||||
return {"kind": "transfer", "source_backend": sb, "source_model": sm,
|
||||
"target_backend": tb, "target_model": tm, "seed": seed, "nights": nights}
|
||||
|
||||
|
||||
PLANS: Dict[str, List[Dict[str, Any]]] = {
|
||||
# one cheap seed each, both backends — fast sanity
|
||||
"quick": [
|
||||
_direct_cfg("claude", "haiku", "brief-writer", 1),
|
||||
_direct_cfg("codex", "", "brief-writer", 2),
|
||||
],
|
||||
# SkillOpt-faithful: STRONG optimizer (sonnet) proposes, WEAK target (haiku)
|
||||
# runs — the reliable config. Plus Codex self-optimized. All 4 gbrain seeds,
|
||||
# including quick-answerer (real tool loop).
|
||||
"direct": [
|
||||
_dual_cfg("claude", "sonnet", "claude", "haiku", "brief-writer"),
|
||||
_dual_cfg("claude", "sonnet", "claude", "haiku", "advisor"),
|
||||
_dual_cfg("claude", "sonnet", "claude", "haiku", "thorough-analyst"),
|
||||
_dual_cfg("claude", "sonnet", "claude", "haiku", "quick-answerer"),
|
||||
_direct_cfg("codex", "", "brief-writer"),
|
||||
_direct_cfg("codex", "", "advisor"),
|
||||
_direct_cfg("codex", "", "quick-answerer"),
|
||||
],
|
||||
# the price-difference story: optimize cheap, deploy expensive (and reverse)
|
||||
"transfer": [
|
||||
_transfer_cfg("claude", "haiku", "claude", "sonnet", "brief-writer"),
|
||||
_transfer_cfg("claude", "sonnet", "claude", "haiku", "brief-writer"),
|
||||
_transfer_cfg("codex", "", "claude", "haiku", "brief-writer"),
|
||||
_transfer_cfg("claude", "haiku", "codex", "", "brief-writer"),
|
||||
],
|
||||
}
|
||||
PLANS["full"] = PLANS["direct"] + PLANS["transfer"]
|
||||
|
||||
|
||||
def _cfg_key(c: Dict[str, Any]) -> str:
|
||||
return json.dumps({k: c[k] for k in sorted(c)}, ensure_ascii=False)
|
||||
|
||||
|
||||
def _load_done(out_path: str) -> set:
|
||||
done = set()
|
||||
if os.path.exists(out_path):
|
||||
with open(out_path) as f:
|
||||
for line in f:
|
||||
try:
|
||||
row = json.loads(line)
|
||||
if "cfg_key" in row:
|
||||
done.add(row["cfg_key"])
|
||||
except Exception:
|
||||
pass
|
||||
return done
|
||||
|
||||
|
||||
def _append(out_path: str, row: Dict[str, Any]) -> None:
|
||||
os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
|
||||
with open(out_path, "a") as f:
|
||||
f.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def run_one(cfg: Dict[str, Any], data_root: str, codex_path: str,
|
||||
limit_replay: int, limit_holdout: int) -> Dict[str, Any]:
|
||||
seed = cfg["seed"]
|
||||
skill, tasks = load_seed(data_root, seed)
|
||||
t0 = time.time()
|
||||
if cfg["kind"] in ("direct", "dual"):
|
||||
if cfg["kind"] == "dual":
|
||||
be = build_backend(
|
||||
optimizer_backend=cfg["optimizer_backend"], optimizer_model=cfg.get("optimizer_model", ""),
|
||||
target_backend=cfg["target_backend"], target_model=cfg.get("target_model", ""),
|
||||
codex_path=codex_path,
|
||||
)
|
||||
else:
|
||||
be = get_backend(cfg["backend"], model=cfg.get("model", ""), codex_path=codex_path)
|
||||
r = bench_seed(be, seed, skill, tasks, nights=cfg["nights"],
|
||||
limit_replay=limit_replay, limit_holdout=limit_holdout)
|
||||
out = {"baseline": r["held_out_before"], "after": r["held_out_after"],
|
||||
"improved": r["improved"], "tokens": be.tokens_used()}
|
||||
else:
|
||||
src = get_backend(cfg["source_backend"], model=cfg.get("source_model", ""), codex_path=codex_path)
|
||||
tgt = get_backend(cfg["target_backend"], model=cfg.get("target_model", ""), codex_path=codex_path)
|
||||
r = transfer_seed(seed, skill, tasks, source=src, target=tgt, nights=cfg["nights"],
|
||||
edit_budget=4, limit_replay=limit_replay, limit_holdout=limit_holdout,
|
||||
do_direct=False)
|
||||
out = {"baseline_target": r["baseline_target"], "transferred": r["transferred"],
|
||||
"transfer_gain": r["transfer_gain"],
|
||||
"tokens": src.tokens_used() + tgt.tokens_used()}
|
||||
out.update({"cfg": cfg, "cfg_key": _cfg_key(cfg), "elapsed_s": round(time.time() - t0, 1)})
|
||||
return out
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(description="SkillOpt-Sleep benchmark sweep")
|
||||
ap.add_argument("--plan", default="quick", choices=list(PLANS.keys()))
|
||||
ap.add_argument("--out", default="docs/sleep/sweep.jsonl")
|
||||
ap.add_argument("--data-root", default="")
|
||||
ap.add_argument("--codex-path", default="")
|
||||
ap.add_argument("--limit-replay", type=int, default=3)
|
||||
ap.add_argument("--limit-holdout", type=int, default=3)
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
data_root = find_data_root(args.data_root)
|
||||
if not data_root:
|
||||
print("ERROR: gbrain-evals data not found; pass --data-root", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
plan = PLANS[args.plan]
|
||||
done = _load_done(args.out)
|
||||
print(f"[sweep] plan={args.plan} configs={len(plan)} already_done={len(done)} -> {args.out}")
|
||||
for i, cfg in enumerate(plan, 1):
|
||||
key = _cfg_key(cfg)
|
||||
if key in done:
|
||||
print(f"[sweep] ({i}/{len(plan)}) skip (done): {cfg}")
|
||||
continue
|
||||
print(f"[sweep] ({i}/{len(plan)}) running: {cfg}", flush=True)
|
||||
try:
|
||||
row = run_one(cfg, data_root, args.codex_path, args.limit_replay, args.limit_holdout)
|
||||
except Exception as e: # never let one config kill the sweep
|
||||
row = {"cfg": cfg, "cfg_key": key, "error": f"{type(e).__name__}: {e}"}
|
||||
_append(args.out, row)
|
||||
print(f"[sweep] -> {json.dumps({k: v for k, v in row.items() if k not in ('cfg','cfg_key')})}", flush=True)
|
||||
print(f"[sweep] done. rows in {args.out}: {len(_load_done(args.out))}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
50
engineering/skillopt-sleep/skillopt_sleep/gate.py
Normal file
50
engineering/skillopt-sleep/skillopt_sleep/gate.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
"""SkillOpt-Sleep — vendored validation gate.
|
||||
|
||||
This is a self-contained copy of the SkillOpt validation gate so the sleep
|
||||
engine has ZERO dependency on the research package (skillopt/*). The research
|
||||
repo's ``skillopt.evaluation.gate`` is the reference implementation and the two
|
||||
are kept behaviourally identical; vendoring keeps this open-source tool
|
||||
decoupled from the paper's experiment code.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GateResult:
|
||||
action: str # "accept_new_best" | "accept" | "reject"
|
||||
current_skill: str
|
||||
current_score: float
|
||||
best_skill: str
|
||||
best_score: float
|
||||
best_step: int
|
||||
|
||||
|
||||
def select_gate_score(hard: float, soft: float, metric: str = "hard",
|
||||
mixed_weight: float = 0.5) -> float:
|
||||
"""Project (hard, soft) onto a single comparison metric."""
|
||||
if metric == "hard":
|
||||
return float(hard)
|
||||
if metric == "soft":
|
||||
return float(soft)
|
||||
if metric == "mixed":
|
||||
w = max(0.0, min(1.0, float(mixed_weight)))
|
||||
return (1.0 - w) * float(hard) + w * float(soft)
|
||||
raise ValueError(f"unknown gate metric {metric!r}; expected hard/soft/mixed")
|
||||
|
||||
|
||||
def evaluate_gate(candidate_skill: str, cand_hard: float, current_skill: str,
|
||||
current_score: float, best_skill: str, best_score: float,
|
||||
best_step: int, global_step: int, *, cand_soft: float = 0.0,
|
||||
metric: str = "hard", mixed_weight: float = 0.5) -> GateResult:
|
||||
"""Pure gate decision: compare candidate score to current/best."""
|
||||
cand_score = select_gate_score(cand_hard, cand_soft, metric, mixed_weight)
|
||||
if cand_score > current_score:
|
||||
if cand_score > best_score:
|
||||
return GateResult("accept_new_best", candidate_skill, cand_score,
|
||||
candidate_skill, cand_score, global_step)
|
||||
return GateResult("accept", candidate_skill, cand_score,
|
||||
best_skill, best_score, best_step)
|
||||
return GateResult("reject", current_skill, current_score,
|
||||
best_skill, best_score, best_step)
|
||||
304
engineering/skillopt-sleep/skillopt_sleep/harvest.py
Normal file
304
engineering/skillopt-sleep/skillopt_sleep/harvest.py
Normal file
|
|
@ -0,0 +1,304 @@
|
|||
"""SkillOpt-Sleep — Stage 1: harvest.
|
||||
|
||||
Read the user's local Claude Code records (read-only) and normalize them
|
||||
into :class:`SessionDigest` objects.
|
||||
|
||||
Sources (verified schema):
|
||||
* ~/.claude/history.jsonl — one JSON/line:
|
||||
{"display": <prompt text>, "pastedContents": {...},
|
||||
"timestamp": <epoch ms>, "project": <abs path>}
|
||||
* ~/.claude/projects/<slug>/<sessionId>.jsonl — one record/line; the
|
||||
records we care about have type "user"/"assistant" and carry:
|
||||
message{role, content}, cwd, gitBranch, timestamp, sessionId, version
|
||||
|
||||
This module performs NO writes and NO network calls.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
from skillopt_sleep.types import SessionDigest
|
||||
|
||||
|
||||
# Heuristic phrases that signal the user (dis)approving of prior output.
|
||||
# English-only by default. Users whose sessions are in another language can add
|
||||
# their own phrases via the SKILLOPT_SLEEP_NEG_FEEDBACK / _POS_FEEDBACK env vars
|
||||
# (comma-separated), so the capability is extensible without hardcoding locales.
|
||||
_NEGATIVE_FEEDBACK = (
|
||||
"still broken", "still not", "still wrong", "doesn't work", "does not work",
|
||||
"not working", "that's wrong", "thats wrong", "incorrect", "wrong",
|
||||
"no,", "nope", "fix it", "didn't", "did not", "broken", "error again",
|
||||
"still failing", "still fails", "not fixed", "revert", "undo",
|
||||
)
|
||||
_POSITIVE_FEEDBACK = (
|
||||
"thanks", "thank you", "perfect", "great", "works now", "fixed",
|
||||
"that works", "lgtm", "looks good", "nice", "awesome", "correct",
|
||||
)
|
||||
|
||||
|
||||
def _extra_phrases(env_var: str) -> tuple:
|
||||
raw = os.environ.get(env_var, "")
|
||||
return tuple(p.strip().lower() for p in raw.split(",") if p.strip())
|
||||
|
||||
|
||||
_NEGATIVE_FEEDBACK = _NEGATIVE_FEEDBACK + _extra_phrases("SKILLOPT_SLEEP_NEG_FEEDBACK")
|
||||
_POSITIVE_FEEDBACK = _POSITIVE_FEEDBACK + _extra_phrases("SKILLOPT_SLEEP_POS_FEEDBACK")
|
||||
|
||||
|
||||
def _iter_jsonl(path: str) -> Iterable[Dict[str, Any]]:
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
yield json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
except (FileNotFoundError, IsADirectoryError, PermissionError):
|
||||
return
|
||||
|
||||
|
||||
def _text_from_content(content: Any) -> str:
|
||||
"""Flatten a message.content (str or list of blocks) into text."""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: List[str] = []
|
||||
for b in content:
|
||||
if isinstance(b, dict):
|
||||
if b.get("type") == "text" and b.get("text"):
|
||||
parts.append(str(b["text"]))
|
||||
return "\n".join(parts)
|
||||
return ""
|
||||
|
||||
|
||||
def _tool_names_from_content(content: Any) -> List[str]:
|
||||
names: List[str] = []
|
||||
if isinstance(content, list):
|
||||
for b in content:
|
||||
if isinstance(b, dict) and b.get("type") == "tool_use" and b.get("name"):
|
||||
names.append(str(b["name"]))
|
||||
return names
|
||||
|
||||
|
||||
def _detect_feedback(text: str) -> List[str]:
|
||||
low = text.lower()
|
||||
sig: List[str] = []
|
||||
for ph in _NEGATIVE_FEEDBACK:
|
||||
if ph in low:
|
||||
sig.append("neg:" + ph)
|
||||
for ph in _POSITIVE_FEEDBACK:
|
||||
if ph in low:
|
||||
sig.append("pos:" + ph)
|
||||
return sig
|
||||
|
||||
|
||||
def _is_meta_prompt(text: str) -> bool:
|
||||
"""Skip slash-commands / system noise that aren't real user intents."""
|
||||
t = text.strip()
|
||||
if not t:
|
||||
return True
|
||||
if t.startswith("<") and t.endswith(">"):
|
||||
return True
|
||||
if t.startswith("/") and len(t.split()) <= 3:
|
||||
return True
|
||||
if t.startswith("[Pasted text") or t.startswith("Caveat:"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ── Issue #62: filter headless replay sessions ─────────────────────────
|
||||
|
||||
# Prompt markers generated by the engine's own headless `claude -p` calls
|
||||
# (judge, reflect, attempt). If the sole user prompt in a single-turn
|
||||
# session matches any of these, the session is engine-generated, not a
|
||||
# real user task.
|
||||
_REPLAY_PROMPT_MARKERS = (
|
||||
"## CURRENT SKILL",
|
||||
"## FAILED TASKS",
|
||||
"## SUCCESSFUL TASKS",
|
||||
"## OUTPUT FORMAT",
|
||||
"You are a strict grader",
|
||||
"Score the response 0.0-1.0",
|
||||
"You are SkillOpt-Sleep",
|
||||
"## TASK\n",
|
||||
"## SKILL\n",
|
||||
)
|
||||
|
||||
|
||||
def _is_headless_replay(digest: "SessionDigest") -> bool:
|
||||
"""Detect sessions created by the engine's own headless replay calls.
|
||||
|
||||
Heuristics (conservatively applied):
|
||||
1. Session has exactly 1 user turn AND
|
||||
2. The sole prompt matches engine-generated patterns (grader/reflect),
|
||||
OR the session lasted < 3 seconds (programmatic, not interactive).
|
||||
Multi-turn sessions are always kept (interactive by definition).
|
||||
"""
|
||||
if digest.n_user_turns > 1:
|
||||
return False
|
||||
if digest.n_user_turns == 0:
|
||||
return True
|
||||
prompt = digest.user_prompts[0] if digest.user_prompts else ""
|
||||
for marker in _REPLAY_PROMPT_MARKERS:
|
||||
if marker in prompt:
|
||||
return True
|
||||
# Sub-3-second single-turn sessions with short prompts are almost
|
||||
# certainly programmatic (engine grader/judge calls). We require the
|
||||
# prompt to also be short (<200 chars) to avoid false-positives on
|
||||
# real one-shot questions that Claude happens to answer quickly.
|
||||
if digest.started_at and digest.ended_at and len(prompt) < 200:
|
||||
try:
|
||||
fmt = "%Y-%m-%dT%H:%M:%S"
|
||||
start = datetime.strptime(digest.started_at[:19], fmt)
|
||||
end = datetime.strptime(digest.ended_at[:19], fmt)
|
||||
if (end - start).total_seconds() < 3:
|
||||
return True
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def digest_transcript(path: str) -> Optional[SessionDigest]:
|
||||
"""Build a SessionDigest from one ``<sessionId>.jsonl`` transcript."""
|
||||
session_id = os.path.splitext(os.path.basename(path))[0]
|
||||
project = ""
|
||||
git_branch = ""
|
||||
started = ""
|
||||
ended = ""
|
||||
user_prompts: List[str] = []
|
||||
assistant_finals: List[str] = []
|
||||
tools: List[str] = []
|
||||
files: List[str] = []
|
||||
feedback: List[str] = []
|
||||
n_user = 0
|
||||
n_asst = 0
|
||||
|
||||
for rec in _iter_jsonl(path):
|
||||
rtype = rec.get("type")
|
||||
ts = rec.get("timestamp")
|
||||
if isinstance(ts, str) and ts:
|
||||
if not started:
|
||||
started = ts
|
||||
ended = ts
|
||||
if rec.get("cwd") and not project:
|
||||
project = str(rec.get("cwd"))
|
||||
if rec.get("gitBranch") and not git_branch:
|
||||
git_branch = str(rec.get("gitBranch"))
|
||||
if rtype == "file-history-snapshot":
|
||||
snap = rec.get("snapshot") or rec.get("files") or {}
|
||||
if isinstance(snap, dict):
|
||||
files.extend([str(k) for k in list(snap.keys())[:20]])
|
||||
msg = rec.get("message")
|
||||
if not isinstance(msg, dict):
|
||||
continue
|
||||
role = msg.get("role")
|
||||
content = msg.get("content")
|
||||
if role == "user":
|
||||
text = _text_from_content(content)
|
||||
if text and not _is_meta_prompt(text):
|
||||
n_user += 1
|
||||
user_prompts.append(text.strip())
|
||||
feedback.extend(_detect_feedback(text))
|
||||
elif role == "assistant":
|
||||
n_asst += 1
|
||||
tools.extend(_tool_names_from_content(content))
|
||||
text = _text_from_content(content)
|
||||
if text.strip():
|
||||
assistant_finals.append(text.strip())
|
||||
|
||||
if n_user == 0 and n_asst == 0:
|
||||
return None
|
||||
|
||||
# de-dup tools/files preserving order
|
||||
def _dedup(xs: List[str]) -> List[str]:
|
||||
seen = set()
|
||||
out = []
|
||||
for x in xs:
|
||||
if x not in seen:
|
||||
seen.add(x)
|
||||
out.append(x)
|
||||
return out
|
||||
|
||||
return SessionDigest(
|
||||
session_id=session_id,
|
||||
project=project,
|
||||
git_branch=git_branch,
|
||||
started_at=started,
|
||||
ended_at=ended,
|
||||
user_prompts=user_prompts,
|
||||
assistant_finals=assistant_finals[-5:], # last few finals are the useful ones
|
||||
tools_used=_dedup(tools),
|
||||
files_touched=_dedup(files),
|
||||
feedback_signals=feedback,
|
||||
n_user_turns=n_user,
|
||||
n_assistant_turns=n_asst,
|
||||
raw_path=path,
|
||||
)
|
||||
|
||||
|
||||
def _project_matches(project: str, scope: Any, invoked: str) -> bool:
|
||||
if scope == "all":
|
||||
return True
|
||||
if isinstance(scope, (list, tuple)):
|
||||
return any(os.path.abspath(project) == os.path.abspath(p) for p in scope)
|
||||
# "invoked": match the invoked project (or a subdir of it)
|
||||
if not invoked:
|
||||
return True
|
||||
a = os.path.abspath(project)
|
||||
b = os.path.abspath(invoked)
|
||||
return a == b or a.startswith(b + os.sep) or b.startswith(a + os.sep)
|
||||
|
||||
|
||||
def harvest(
|
||||
transcripts_dir: str,
|
||||
*,
|
||||
scope: Any = "all",
|
||||
invoked_project: str = "",
|
||||
since_iso: Optional[str] = None,
|
||||
limit: int = 0,
|
||||
) -> List[SessionDigest]:
|
||||
"""Walk ~/.claude/projects and return digests matching scope/time.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
transcripts_dir : str ~/.claude/projects
|
||||
scope : "all" | "invoked" | list[path]
|
||||
invoked_project : str used when scope == "invoked"
|
||||
since_iso : str|None ISO8601; only sessions ending after this are kept
|
||||
limit : int cap number of digests (0 = no cap)
|
||||
"""
|
||||
digests: List[SessionDigest] = []
|
||||
if not os.path.isdir(transcripts_dir):
|
||||
return digests
|
||||
|
||||
paths: List[str] = []
|
||||
for root, _dirs, files in os.walk(transcripts_dir):
|
||||
for fn in files:
|
||||
if fn.endswith(".jsonl"):
|
||||
paths.append(os.path.join(root, fn))
|
||||
# newest first by mtime
|
||||
paths.sort(key=lambda p: os.path.getmtime(p), reverse=True)
|
||||
|
||||
for p in paths:
|
||||
d = digest_transcript(p)
|
||||
if d is None:
|
||||
continue
|
||||
if _is_headless_replay(d):
|
||||
continue # Issue #62: skip engine's own headless replay sessions
|
||||
if not _project_matches(d.project or "", scope, invoked_project):
|
||||
continue
|
||||
if since_iso and d.ended_at and d.ended_at < since_iso:
|
||||
# Note: files are sorted by mtime but we compare the embedded
|
||||
# ended_at timestamp — mtime can diverge (copy/touch), so we
|
||||
# cannot break here; we must continue to check all files.
|
||||
continue
|
||||
digests.append(d)
|
||||
if limit and len(digests) >= limit:
|
||||
break
|
||||
return digests
|
||||
233
engineering/skillopt-sleep/skillopt_sleep/harvest_codex.py
Normal file
233
engineering/skillopt-sleep/skillopt_sleep/harvest_codex.py
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
"""SkillOpt-Sleep Codex Desktop session harvesting.
|
||||
|
||||
Reads Codex Desktop archived session JSONL files and normalizes them into
|
||||
``SessionDigest`` records without copying developer/system instructions, tool
|
||||
arguments, or raw tool outputs.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
from skillopt_sleep.harvest import (
|
||||
_detect_feedback,
|
||||
_is_meta_prompt,
|
||||
_iter_jsonl,
|
||||
_project_matches,
|
||||
)
|
||||
from skillopt_sleep.staging import _SECRET_PATTERNS
|
||||
from skillopt_sleep.types import SessionDigest
|
||||
|
||||
|
||||
def _payload(rec: Dict[str, Any]) -> Dict[str, Any]:
|
||||
payload = rec.get("payload")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def _timestamp(rec: Dict[str, Any], payload: Dict[str, Any]) -> str:
|
||||
for value in (
|
||||
payload.get("timestamp"),
|
||||
rec.get("timestamp"),
|
||||
payload.get("started_at"),
|
||||
payload.get("completed_at"),
|
||||
):
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return ""
|
||||
|
||||
|
||||
def _text_from_any(content: Any) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: List[str] = []
|
||||
for item in content:
|
||||
if isinstance(item, str):
|
||||
parts.append(item)
|
||||
elif isinstance(item, dict):
|
||||
if item.get("type") == "text" and item.get("text"):
|
||||
parts.append(str(item["text"]))
|
||||
elif item.get("text"):
|
||||
parts.append(str(item["text"]))
|
||||
return "\n".join(parts)
|
||||
if isinstance(content, dict):
|
||||
if content.get("text"):
|
||||
return str(content["text"])
|
||||
if content.get("content"):
|
||||
return _text_from_any(content["content"])
|
||||
return ""
|
||||
|
||||
|
||||
def _strip_codex_meta(text: str) -> str:
|
||||
stripped = text.strip()
|
||||
if not stripped:
|
||||
return ""
|
||||
if stripped.startswith("<codex_internal_context"):
|
||||
return ""
|
||||
if stripped.startswith("<environment_context"):
|
||||
return ""
|
||||
if stripped.startswith("# AGENTS.md instructions") or "--- project-doc ---" in stripped:
|
||||
for marker in ("</environment_context>", "</INSTRUCTIONS>"):
|
||||
idx = stripped.rfind(marker)
|
||||
if idx == -1:
|
||||
continue
|
||||
tail = stripped[idx + len(marker):].strip()
|
||||
if tail and not tail.startswith("<"):
|
||||
return tail
|
||||
return ""
|
||||
return stripped
|
||||
|
||||
|
||||
def _sanitize_text(text: str) -> str:
|
||||
sanitized = _strip_codex_meta(text).replace("\x00", "").strip()
|
||||
if not sanitized or _is_meta_prompt(sanitized):
|
||||
return ""
|
||||
for pattern, replacement in _SECRET_PATTERNS:
|
||||
sanitized = pattern.sub(replacement, sanitized)
|
||||
return sanitized
|
||||
|
||||
|
||||
def _sanitize_tool_name(name: str) -> str:
|
||||
return re.sub(r"[^A-Za-z0-9_.:-]+", "_", name)[:80]
|
||||
|
||||
|
||||
def _tool_name(payload: Dict[str, Any]) -> str:
|
||||
payload_type = payload.get("type")
|
||||
name = payload.get("name")
|
||||
if isinstance(name, str) and name:
|
||||
return _sanitize_tool_name(name)
|
||||
if payload_type == "exec_command_end":
|
||||
return "exec_command"
|
||||
if payload_type == "patch_apply_end":
|
||||
return "apply_patch"
|
||||
if payload_type == "web_search_call":
|
||||
return "web_search"
|
||||
if payload_type == "tool_search_call":
|
||||
return "tool_search"
|
||||
if isinstance(payload_type, str) and payload_type.endswith("_tool_call"):
|
||||
return _sanitize_tool_name(payload_type)
|
||||
return ""
|
||||
|
||||
|
||||
def _dedup(xs: Iterable[str]) -> List[str]:
|
||||
seen = set()
|
||||
out: List[str] = []
|
||||
for x in xs:
|
||||
if x not in seen:
|
||||
seen.add(x)
|
||||
out.append(x)
|
||||
return out
|
||||
|
||||
|
||||
def digest_codex_archived_session(path: str, project: str = "") -> Optional[SessionDigest]:
|
||||
"""Build a ``SessionDigest`` from one Codex Desktop archived session."""
|
||||
session_id = os.path.splitext(os.path.basename(path))[0]
|
||||
started = ""
|
||||
ended = ""
|
||||
session_project = ""
|
||||
user_prompts: List[str] = []
|
||||
assistant_finals: List[str] = []
|
||||
tools: List[str] = []
|
||||
feedback: List[str] = []
|
||||
n_user = 0
|
||||
n_asst = 0
|
||||
|
||||
for rec in _iter_jsonl(path):
|
||||
payload = _payload(rec)
|
||||
payload_type = payload.get("type")
|
||||
ts = _timestamp(rec, payload)
|
||||
if ts:
|
||||
if not started:
|
||||
started = ts
|
||||
ended = ts
|
||||
cwd = payload.get("cwd")
|
||||
if isinstance(cwd, str) and cwd:
|
||||
if not session_project:
|
||||
session_project = cwd
|
||||
if project and _project_matches(cwd, "invoked", project):
|
||||
session_project = cwd
|
||||
|
||||
role = payload.get("role")
|
||||
text = ""
|
||||
output_role = ""
|
||||
if payload_type == "user_message":
|
||||
text = _text_from_any(payload.get("message"))
|
||||
output_role = "user"
|
||||
elif payload_type == "agent_message":
|
||||
text = _text_from_any(payload.get("message"))
|
||||
output_role = "assistant"
|
||||
elif payload_type == "message" and role in {"user", "assistant"}:
|
||||
text = _text_from_any(payload.get("content"))
|
||||
output_role = str(role)
|
||||
else:
|
||||
tool = _tool_name(payload)
|
||||
if tool:
|
||||
tools.append(tool)
|
||||
continue
|
||||
|
||||
sanitized = _sanitize_text(text)
|
||||
if not sanitized:
|
||||
continue
|
||||
if output_role == "user":
|
||||
n_user += 1
|
||||
user_prompts.append(sanitized)
|
||||
feedback.extend(_detect_feedback(sanitized))
|
||||
elif output_role == "assistant":
|
||||
n_asst += 1
|
||||
assistant_finals.append(sanitized)
|
||||
|
||||
if project and not _project_matches(session_project or "", "invoked", project):
|
||||
return None
|
||||
if n_user == 0 and n_asst == 0:
|
||||
return None
|
||||
|
||||
return SessionDigest(
|
||||
session_id=session_id,
|
||||
project=session_project,
|
||||
started_at=started,
|
||||
ended_at=ended,
|
||||
user_prompts=user_prompts,
|
||||
assistant_finals=assistant_finals[-5:],
|
||||
tools_used=_dedup(tools),
|
||||
files_touched=[],
|
||||
feedback_signals=feedback,
|
||||
n_user_turns=n_user,
|
||||
n_assistant_turns=n_asst,
|
||||
raw_path=path,
|
||||
)
|
||||
|
||||
|
||||
def harvest_codex(
|
||||
archived_sessions_dir: str,
|
||||
*,
|
||||
scope: Any = "all",
|
||||
invoked_project: str = "",
|
||||
since_iso: Optional[str] = None,
|
||||
limit: int = 0,
|
||||
) -> List[SessionDigest]:
|
||||
"""Walk ``~/.codex/archived_sessions`` and return matching digests."""
|
||||
digests: List[SessionDigest] = []
|
||||
if not os.path.isdir(archived_sessions_dir):
|
||||
return digests
|
||||
|
||||
paths = [
|
||||
os.path.join(archived_sessions_dir, fn)
|
||||
for fn in os.listdir(archived_sessions_dir)
|
||||
if fn.endswith(".jsonl")
|
||||
]
|
||||
paths.sort(key=lambda p: os.path.getmtime(p), reverse=True)
|
||||
|
||||
project_hint = invoked_project if scope == "invoked" else ""
|
||||
for path in paths:
|
||||
digest = digest_codex_archived_session(path, project=project_hint)
|
||||
if digest is None:
|
||||
continue
|
||||
if not _project_matches(digest.project or "", scope, invoked_project):
|
||||
continue
|
||||
if since_iso and digest.ended_at and digest.ended_at < since_iso:
|
||||
continue
|
||||
digests.append(digest)
|
||||
if limit and len(digests) >= limit:
|
||||
break
|
||||
return digests
|
||||
41
engineering/skillopt-sleep/skillopt_sleep/harvest_sources.py
Normal file
41
engineering/skillopt-sleep/skillopt_sleep/harvest_sources.py
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
"""Source selection for SkillOpt-Sleep transcript harvesting."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from skillopt_sleep.harvest import harvest
|
||||
from skillopt_sleep.harvest_codex import harvest_codex
|
||||
from skillopt_sleep.types import SessionDigest
|
||||
|
||||
|
||||
def harvest_for_config(cfg, *, since_iso: Optional[str] = None, limit: int = 0) -> list[SessionDigest]:
|
||||
source = cfg.get("transcript_source", "claude")
|
||||
scope = cfg.get("projects", "invoked")
|
||||
invoked_project = cfg.get("invoked_project", "")
|
||||
|
||||
if source == "codex":
|
||||
return harvest_codex(
|
||||
cfg.codex_archived_sessions_dir,
|
||||
scope=scope,
|
||||
invoked_project=invoked_project,
|
||||
since_iso=since_iso,
|
||||
limit=limit,
|
||||
)
|
||||
if source == "auto":
|
||||
codex_digests = harvest_codex(
|
||||
cfg.codex_archived_sessions_dir,
|
||||
scope=scope,
|
||||
invoked_project=invoked_project,
|
||||
since_iso=since_iso,
|
||||
limit=limit,
|
||||
)
|
||||
if codex_digests:
|
||||
return codex_digests
|
||||
|
||||
return harvest(
|
||||
cfg.transcripts_dir,
|
||||
scope=scope,
|
||||
invoked_project=invoked_project,
|
||||
since_iso=since_iso,
|
||||
limit=limit,
|
||||
)
|
||||
84
engineering/skillopt-sleep/skillopt_sleep/judges.py
Normal file
84
engineering/skillopt-sleep/skillopt_sleep/judges.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
"""SkillOpt-Sleep — rule-based judges (gbrain-evals compatible).
|
||||
|
||||
Implements the programmatic check operators used by gbrain-evals'
|
||||
skillopt-v1 benchmark so we can score skill outputs locally, with NO judge
|
||||
API call:
|
||||
|
||||
* section_present <name> — a markdown heading containing <name> exists
|
||||
* regex <pattern> — the pattern matches the response
|
||||
* max_chars <n> — response length <= n
|
||||
* min_chars <n> — response length >= n
|
||||
* contains <text> — substring present (case-insensitive)
|
||||
* tool_called <name> — a tool with <name> was invoked (needs a tool loop;
|
||||
in single-shot replay we approximate via an
|
||||
explicit "TOOL_CALL: <name>" marker the agent emits)
|
||||
|
||||
A task whose judge is {"kind": "rule", "checks": [...]} passes (hard=1.0) iff
|
||||
ALL checks pass; soft = fraction of checks passed. This mirrors gbrain's
|
||||
all-checks-must-pass rule scoring and gives the gate a smooth signal.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
|
||||
def _section_present(response: str, name: str) -> bool:
|
||||
# a markdown heading line (#, ##, ...) or bold line that contains `name`
|
||||
pat = re.compile(
|
||||
r"(?im)^\s{0,3}(#{1,6}\s*.*%s|\*\*.*%s.*\*\*\s*:?)\s*$" % (re.escape(name), re.escape(name))
|
||||
)
|
||||
if pat.search(response or ""):
|
||||
return True
|
||||
# also accept "Name:" style label at line start
|
||||
label = re.compile(r"(?im)^\s*%s\s*:" % re.escape(name))
|
||||
return bool(label.search(response or ""))
|
||||
|
||||
|
||||
def _check(op: str, arg: Any, response: str, tools_called: List[str]) -> bool:
|
||||
r = response or ""
|
||||
if op == "section_present":
|
||||
return _section_present(r, str(arg))
|
||||
if op == "regex":
|
||||
try:
|
||||
return bool(re.search(str(arg), r))
|
||||
except re.error:
|
||||
return False
|
||||
if op == "max_chars":
|
||||
return len(r) <= int(arg)
|
||||
if op == "min_chars":
|
||||
return len(r) >= int(arg)
|
||||
if op == "contains":
|
||||
return str(arg).lower() in r.lower()
|
||||
if op == "tool_called":
|
||||
name = str(arg).lower()
|
||||
if any(name == t.lower() for t in tools_called):
|
||||
return True
|
||||
# single-shot approximation: the agent emits an explicit marker
|
||||
return bool(re.search(r"(?i)\btool_call\s*:\s*%s\b" % re.escape(name), r))
|
||||
# unknown op: do not block
|
||||
return True
|
||||
|
||||
|
||||
def score_rule_judge(
|
||||
judge: Dict[str, Any],
|
||||
response: str,
|
||||
tools_called: List[str] | None = None,
|
||||
) -> Tuple[float, float, str]:
|
||||
"""Return (hard, soft, rationale) for a gbrain-style rule judge."""
|
||||
checks = (judge or {}).get("checks", []) or []
|
||||
if not checks:
|
||||
return 0.0, 0.0, "no checks"
|
||||
tools_called = tools_called or []
|
||||
passed = 0
|
||||
failed_desc: List[str] = []
|
||||
for c in checks:
|
||||
ok = _check(c.get("op", ""), c.get("arg"), response, tools_called)
|
||||
if ok:
|
||||
passed += 1
|
||||
else:
|
||||
failed_desc.append(f"{c.get('op')}={c.get('arg')}")
|
||||
soft = passed / len(checks)
|
||||
hard = 1.0 if passed == len(checks) else 0.0
|
||||
rationale = "all checks passed" if hard else "failed: " + ", ".join(failed_desc)
|
||||
return hard, soft, rationale
|
||||
134
engineering/skillopt-sleep/skillopt_sleep/llm_miner.py
Normal file
134
engineering/skillopt-sleep/skillopt_sleep/llm_miner.py
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
"""SkillOpt-Sleep — LLM-backed task miner.
|
||||
|
||||
The heuristic miner (mine.py) produces TaskRecords without a checkable
|
||||
reference, so real harvested transcripts can't show measurable lift. This
|
||||
module uses an optimizer backend to turn session digests into TaskRecords
|
||||
WITH a checkable rubric judge — the missing piece for real-data improvement.
|
||||
|
||||
For each recurring intent it extracts:
|
||||
* a clean, generalized `intent` (the reusable task, stripped of one-off specifics)
|
||||
* a `rubric` (what a good answer must satisfy) -> stored as a rule judge of
|
||||
`contains`/`regex`/`section_present` checks the local judge can score, OR a
|
||||
free-text rubric scored by the backend's judge() when no programmatic check fits
|
||||
* a preference signal (was the user satisfied?) to weight failures
|
||||
|
||||
It is deliberately conservative: it only emits a task when it can name a
|
||||
concrete, checkable success criterion, so the gate has real signal. Tasks it
|
||||
can't make checkable are dropped (logged), not faked.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Callable, Dict, List
|
||||
|
||||
from skillopt_sleep.backend import Backend, _extract_json
|
||||
from skillopt_sleep.types import SessionDigest, TaskRecord
|
||||
|
||||
|
||||
_MINER_PROMPT = """You are mining a user's past AI-assistant sessions to find RECURRING tasks
|
||||
worth optimizing a skill for. From the session below, extract 0-3 reusable tasks.
|
||||
|
||||
A good task is something the user asks for repeatedly or had to correct, where a
|
||||
GENERAL rule would help next time (formatting, structure, tool-use, conventions).
|
||||
Skip one-off or purely exploratory requests.
|
||||
|
||||
For each task return:
|
||||
- "intent": the reusable request, generalized (no one-off specifics)
|
||||
- "checks": a list of programmatic success checks a grader can run on a future
|
||||
answer. Each check is one of:
|
||||
{"op":"section_present","arg":"<heading text>"}
|
||||
{"op":"regex","arg":"<python regex the answer must match>"}
|
||||
{"op":"contains","arg":"<substring the answer must contain>"}
|
||||
{"op":"max_chars","arg":<int>}
|
||||
Only include checks you are confident a GOOD answer must satisfy.
|
||||
- "rubric": a one-sentence description of what a good answer looks like
|
||||
- "satisfied": true/false — did the user seem satisfied with the assistant's answer?
|
||||
|
||||
Return ONLY a JSON array (possibly empty). No prose.
|
||||
|
||||
# Session
|
||||
project: __PROJECT__
|
||||
user prompts:
|
||||
__PROMPTS__
|
||||
assistant final (last):
|
||||
__FINAL__
|
||||
feedback signals: __FEEDBACK__
|
||||
"""
|
||||
|
||||
|
||||
def _digest_to_prompt(d: SessionDigest) -> str:
|
||||
prompts = "\n".join(f" - {p[:240]}" for p in d.user_prompts[:6]) or " (none)"
|
||||
final = (d.assistant_finals[-1][:400] if d.assistant_finals else "(none)")
|
||||
return (
|
||||
_MINER_PROMPT
|
||||
.replace("__PROJECT__", d.project or "(unknown)")
|
||||
.replace("__PROMPTS__", prompts)
|
||||
.replace("__FINAL__", final)
|
||||
.replace("__FEEDBACK__", ", ".join(d.feedback_signals[:6]) or "(none)")
|
||||
)
|
||||
|
||||
|
||||
def _mk_task(d: SessionDigest, obj: Dict[str, Any], idx: int) -> TaskRecord | None:
|
||||
intent = str(obj.get("intent", "")).strip()
|
||||
if len(intent) < 8:
|
||||
return None
|
||||
checks = obj.get("checks") or []
|
||||
rubric = str(obj.get("rubric", "")).strip()
|
||||
satisfied = bool(obj.get("satisfied", False))
|
||||
|
||||
# keep only well-formed checks
|
||||
clean_checks = []
|
||||
for c in checks:
|
||||
if isinstance(c, dict) and c.get("op") in {
|
||||
"section_present", "regex", "contains", "max_chars", "min_chars",
|
||||
}:
|
||||
clean_checks.append({"op": c["op"], "arg": c.get("arg")})
|
||||
|
||||
import hashlib
|
||||
tid = "llm_" + hashlib.sha256((d.project + intent).encode()).hexdigest()[:12]
|
||||
|
||||
if clean_checks:
|
||||
return TaskRecord(
|
||||
id=tid, project=d.project, intent=intent,
|
||||
reference_kind="rule", judge={"kind": "rule", "checks": clean_checks},
|
||||
outcome="success" if satisfied else "fail",
|
||||
tags=["mined:llm"], source_sessions=[d.session_id],
|
||||
)
|
||||
if rubric:
|
||||
return TaskRecord(
|
||||
id=tid, project=d.project, intent=intent,
|
||||
reference_kind="rubric", reference=rubric,
|
||||
outcome="success" if satisfied else "fail",
|
||||
tags=["mined:llm"], source_sessions=[d.session_id],
|
||||
)
|
||||
return None # not checkable -> drop
|
||||
|
||||
|
||||
def make_llm_miner(
|
||||
backend: Backend,
|
||||
*,
|
||||
max_sessions: int = 20,
|
||||
max_tasks: int = 40,
|
||||
) -> Callable[[List[SessionDigest]], List[TaskRecord]]:
|
||||
"""Return an llm_miner(digests) -> list[TaskRecord] bound to a backend."""
|
||||
|
||||
def _miner(digests: List[SessionDigest]) -> List[TaskRecord]:
|
||||
out: List[TaskRecord] = []
|
||||
for d in digests[:max_sessions]:
|
||||
if not d.user_prompts:
|
||||
continue
|
||||
raw = backend._call(_digest_to_prompt(d), max_tokens=800) # type: ignore[attr-defined]
|
||||
arr = _extract_json(raw, "array")
|
||||
if not isinstance(arr, list):
|
||||
continue
|
||||
for i, obj in enumerate(arr[:3]):
|
||||
if isinstance(obj, dict):
|
||||
t = _mk_task(d, obj, i)
|
||||
if t is not None:
|
||||
out.append(t)
|
||||
if len(out) >= max_tasks:
|
||||
return out
|
||||
return out
|
||||
|
||||
return _miner
|
||||
129
engineering/skillopt-sleep/skillopt_sleep/memory.py
Normal file
129
engineering/skillopt-sleep/skillopt_sleep/memory.py
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
"""SkillOpt-Sleep — skill/memory document manipulation.
|
||||
|
||||
Applies bounded EditRecords to a skill (SKILL.md body) or memory (CLAUDE.md)
|
||||
document, and provides Dream-style consolidation helpers (dedup near-identical
|
||||
lines, drop contradictions). All edits live inside a protected, clearly-marked
|
||||
region so the sleep cycle never clobbers the user's hand-written content.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import List, Tuple
|
||||
|
||||
from skillopt_sleep.types import EditRecord
|
||||
|
||||
LEARNED_START = "<!-- SKILLOPT-SLEEP:LEARNED START -->"
|
||||
LEARNED_END = "<!-- SKILLOPT-SLEEP:LEARNED END -->"
|
||||
_BANNER = (
|
||||
"_This block is maintained by SkillOpt-Sleep. Edits here are proposed "
|
||||
"offline, validated against your past tasks, and adopted only after you "
|
||||
"approve them. Hand-edits outside this block are never touched._"
|
||||
)
|
||||
|
||||
|
||||
def extract_learned(doc: str) -> str:
|
||||
s = doc.find(LEARNED_START)
|
||||
e = doc.find(LEARNED_END)
|
||||
if s == -1 or e == -1:
|
||||
return ""
|
||||
return doc[s + len(LEARNED_START):e].strip()
|
||||
|
||||
|
||||
def _strip_learned(doc: str) -> str:
|
||||
while True:
|
||||
s = doc.find(LEARNED_START)
|
||||
if s == -1:
|
||||
break
|
||||
e = doc.find(LEARNED_END, s)
|
||||
if e == -1:
|
||||
doc = doc[:s]
|
||||
break
|
||||
doc = doc[:s] + doc[e + len(LEARNED_END):]
|
||||
while "\n\n\n" in doc:
|
||||
doc = doc.replace("\n\n\n", "\n\n")
|
||||
return doc.rstrip()
|
||||
|
||||
|
||||
def set_learned(doc: str, learned_lines: List[str]) -> str:
|
||||
"""Replace the protected learned region with the given bullet lines."""
|
||||
base = _strip_learned(doc)
|
||||
body = "\n".join(f"- {ln.strip().lstrip('- ').strip()}" for ln in learned_lines if ln.strip())
|
||||
block = (
|
||||
f"\n\n{LEARNED_START}\n"
|
||||
f"## Learned preferences & procedures\n\n{_BANNER}\n\n{body}\n"
|
||||
f"{LEARNED_END}\n"
|
||||
)
|
||||
return (base + block).lstrip("\n")
|
||||
|
||||
|
||||
def current_learned_lines(doc: str) -> List[str]:
|
||||
inner = extract_learned(doc)
|
||||
lines: List[str] = []
|
||||
for ln in inner.splitlines():
|
||||
ln = ln.strip()
|
||||
if ln.startswith("- "):
|
||||
lines.append(ln[2:].strip())
|
||||
return lines
|
||||
|
||||
|
||||
def _norm(s: str) -> str:
|
||||
return re.sub(r"\s+", " ", (s or "").lower()).strip()
|
||||
|
||||
|
||||
def apply_edits(doc: str, edits: List[EditRecord]) -> Tuple[str, List[EditRecord]]:
|
||||
"""Apply add/delete/replace edits to the protected learned region.
|
||||
|
||||
Returns (new_doc, applied_edits). Dedups: an `add` whose content already
|
||||
exists (normalized) is skipped. `delete`/`replace` match on normalized
|
||||
anchor substring.
|
||||
"""
|
||||
lines = current_learned_lines(doc)
|
||||
norm_set = {_norm(line) for line in lines}
|
||||
applied: List[EditRecord] = []
|
||||
|
||||
for e in edits:
|
||||
op = (e.op or "add").lower()
|
||||
if op == "add":
|
||||
if _norm(e.content) in norm_set or not e.content.strip():
|
||||
continue
|
||||
lines.append(e.content.strip())
|
||||
norm_set.add(_norm(e.content))
|
||||
applied.append(e)
|
||||
elif op == "delete":
|
||||
anchor = _norm(e.anchor or e.content)
|
||||
keep = [line for line in lines if anchor not in _norm(line)]
|
||||
if len(keep) != len(lines):
|
||||
lines = keep
|
||||
norm_set = {_norm(line) for line in lines}
|
||||
applied.append(e)
|
||||
elif op == "replace":
|
||||
anchor = _norm(e.anchor)
|
||||
new_lines = []
|
||||
changed = False
|
||||
for line in lines:
|
||||
if anchor and anchor in _norm(line):
|
||||
new_lines.append(e.content.strip())
|
||||
changed = True
|
||||
else:
|
||||
new_lines.append(line)
|
||||
if changed:
|
||||
lines = new_lines
|
||||
norm_set = {_norm(line) for line in lines}
|
||||
applied.append(e)
|
||||
|
||||
return set_learned(doc, lines), applied
|
||||
|
||||
|
||||
def ensure_skill_scaffold(doc: str, *, name: str, description: str) -> str:
|
||||
"""Ensure a SKILL.md has YAML frontmatter so local agents load it."""
|
||||
if doc.lstrip().startswith("---"):
|
||||
return doc
|
||||
fm = (
|
||||
"---\n"
|
||||
f"name: {name}\n"
|
||||
f"description: {description}\n"
|
||||
"---\n\n"
|
||||
f"# {name}\n\n"
|
||||
"Preferences and procedures learned from your past local agent sessions.\n"
|
||||
)
|
||||
return fm + doc
|
||||
312
engineering/skillopt-sleep/skillopt_sleep/mine.py
Normal file
312
engineering/skillopt-sleep/skillopt_sleep/mine.py
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
"""SkillOpt-Sleep — Stage 2: mine.
|
||||
|
||||
Turn :class:`SessionDigest` objects into :class:`TaskRecord` training units.
|
||||
|
||||
Two miners:
|
||||
* heuristic_mine — deterministic, no API. Detects retry chains (a prompt
|
||||
re-asked after negative feedback => the early attempt failed), extracts
|
||||
the user's recurring intents, and labels outcomes from feedback signals.
|
||||
* llm_mine — optional; uses an optimizer backend to produce richer
|
||||
TaskRecords with checkable references. Falls back to heuristic on error.
|
||||
|
||||
The heuristic miner is what makes the whole cycle runnable offline and is the
|
||||
basis of the deterministic experiment.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
from collections import Counter
|
||||
from typing import Any, Callable, List, Optional, Set, Tuple
|
||||
|
||||
from skillopt_sleep.types import SessionDigest, TaskRecord
|
||||
|
||||
|
||||
def _tid(project: str, intent: str) -> str:
|
||||
h = hashlib.sha256((project + "::" + intent).encode("utf-8")).hexdigest()[:12]
|
||||
return "task_" + h
|
||||
|
||||
|
||||
def _short(text: str, n: int = 600) -> str:
|
||||
text = (text or "").strip()
|
||||
return text if len(text) <= n else text[:n] + " …"
|
||||
|
||||
|
||||
def _looks_negative(signals: List[str]) -> bool:
|
||||
return any(s.startswith("neg:") for s in signals)
|
||||
|
||||
|
||||
def _looks_positive(signals: List[str]) -> bool:
|
||||
return any(s.startswith("pos:") for s in signals)
|
||||
|
||||
|
||||
_TARGET_STOPWORDS = {
|
||||
"about", "after", "again", "agent", "agents", "all", "also", "always",
|
||||
"and", "any", "are", "before", "being", "but", "can", "codex",
|
||||
"current", "default", "docs", "does", "done", "each", "file", "files",
|
||||
"for", "from", "have", "into", "keep", "must", "not", "only", "path",
|
||||
"paths", "project", "read", "repo", "request", "requests", "rule",
|
||||
"rules", "same", "should", "skill", "skills", "source", "start",
|
||||
"task", "tasks", "that", "the", "their", "then", "this", "unless",
|
||||
"update", "user", "users", "when", "with", "work", "workflow",
|
||||
}
|
||||
|
||||
|
||||
def _target_tokens(text: str) -> List[str]:
|
||||
tokens: List[str] = []
|
||||
for raw in re.findall(r"[\w][\w.-]*", (text or "").lower(), flags=re.UNICODE):
|
||||
parts = [raw] + re.split(r"[\W_]+", raw, flags=re.UNICODE)
|
||||
for part in parts:
|
||||
if len(part) < 3 or part.isdigit() or part in _TARGET_STOPWORDS:
|
||||
continue
|
||||
tokens.append(part)
|
||||
return tokens
|
||||
|
||||
|
||||
def _expand_target_keywords(keywords: Set[str]) -> None:
|
||||
if "mcp" in keywords:
|
||||
keywords.update({
|
||||
"configure", "configuration", "connect", "connected", "enable",
|
||||
"enabled", "install", "installed", "server", "servers",
|
||||
"настрой", "настроить", "подключи", "подключить",
|
||||
})
|
||||
if {"conflict", "conflicts"} & keywords:
|
||||
keywords.update({
|
||||
"cherry", "conflict", "conflicts", "git", "merge", "rebase",
|
||||
"unmerged", "конфликт", "конфликты",
|
||||
})
|
||||
|
||||
|
||||
def target_task_keywords(
|
||||
target_skill_text: str,
|
||||
target_skill_path: str = "",
|
||||
*,
|
||||
limit: int = 180,
|
||||
) -> Tuple[Set[str], Set[str]]:
|
||||
"""Return (strong, weak) keywords that describe a target skill."""
|
||||
path_text = (target_skill_path or "").replace(os.sep, " ")
|
||||
headings = "\n".join(re.findall(r"(?m)^#+\s+(.+)$", target_skill_text or ""))
|
||||
strong = set(_target_tokens(path_text + "\n" + headings))
|
||||
weak = set(strong)
|
||||
counts = Counter(_target_tokens(target_skill_text or ""))
|
||||
for token, _count in counts.most_common(limit):
|
||||
weak.add(token)
|
||||
_expand_target_keywords(strong)
|
||||
_expand_target_keywords(weak)
|
||||
return strong, weak
|
||||
|
||||
|
||||
def _task_search_text(task: TaskRecord) -> str:
|
||||
return "\n".join([
|
||||
task.intent or "",
|
||||
task.context_excerpt or "",
|
||||
" ".join(task.tags or []),
|
||||
])
|
||||
|
||||
|
||||
def filter_tasks_for_target(
|
||||
tasks: List[TaskRecord],
|
||||
target_skill_text: str,
|
||||
target_skill_path: str = "",
|
||||
) -> List[TaskRecord]:
|
||||
"""Prefer tasks whose language overlaps the explicit target skill.
|
||||
|
||||
If nothing matches, return the original list. This keeps a target run useful
|
||||
even when transcripts are too sparse or the skill is too generic.
|
||||
"""
|
||||
strong, weak = target_task_keywords(target_skill_text, target_skill_path)
|
||||
if not tasks or not (strong or weak):
|
||||
return tasks
|
||||
|
||||
ranked = []
|
||||
for idx, task in enumerate(tasks):
|
||||
tokens = set(_target_tokens(_task_search_text(task)))
|
||||
strong_hits = tokens & strong
|
||||
weak_hits = tokens & weak
|
||||
if not strong_hits and len(weak_hits) < 2:
|
||||
continue
|
||||
score = len(strong_hits) * 3 + len(weak_hits)
|
||||
ranked.append((score, idx, task))
|
||||
if not ranked:
|
||||
return tasks
|
||||
ranked.sort(key=lambda item: (-item[0], item[1]))
|
||||
return [task for _score, _idx, task in ranked]
|
||||
|
||||
|
||||
def heuristic_mine(
|
||||
digests: List[SessionDigest],
|
||||
*,
|
||||
max_tasks: int = 40,
|
||||
) -> List[TaskRecord]:
|
||||
"""Deterministic miner — no API calls.
|
||||
|
||||
Strategy:
|
||||
* Each session with >=1 real user prompt yields one TaskRecord whose
|
||||
intent is the FIRST substantive prompt (the original ask).
|
||||
* Outcome is inferred:
|
||||
- negative feedback present and no later positive -> "fail"
|
||||
- positive feedback present -> "success"
|
||||
- re-asks (multiple user turns) without resolution -> "mixed"
|
||||
- otherwise -> "unknown"
|
||||
* attempted_solution = the last assistant final (what was produced).
|
||||
* reference_kind defaults to "none"; the consolidation step will use a
|
||||
rubric judge for these. (Exact refs are added by the experiment data
|
||||
or by the LLM miner when it can derive a checkable answer.)
|
||||
"""
|
||||
tasks: List[TaskRecord] = []
|
||||
for d in digests:
|
||||
if not d.user_prompts:
|
||||
continue
|
||||
intent = d.user_prompts[0]
|
||||
if len(intent.strip()) < 8:
|
||||
continue
|
||||
if _looks_positive(d.feedback_signals) and not _looks_negative(d.feedback_signals):
|
||||
outcome = "success"
|
||||
elif _looks_negative(d.feedback_signals):
|
||||
outcome = "fail"
|
||||
elif d.n_user_turns >= 3:
|
||||
outcome = "mixed"
|
||||
else:
|
||||
outcome = "unknown"
|
||||
|
||||
attempted = d.assistant_finals[-1] if d.assistant_finals else ""
|
||||
context = ""
|
||||
if len(d.user_prompts) > 1:
|
||||
# later prompts often carry the corrective detail / real constraints
|
||||
context = "Follow-up constraints from the same session:\n- " + "\n- ".join(
|
||||
_short(p, 200) for p in d.user_prompts[1:4]
|
||||
)
|
||||
tags = []
|
||||
if d.tools_used:
|
||||
tags.append("tools:" + "+".join(d.tools_used[:4]))
|
||||
if d.git_branch:
|
||||
tags.append("branch:" + d.git_branch)
|
||||
|
||||
tasks.append(
|
||||
TaskRecord(
|
||||
id=_tid(d.project, intent),
|
||||
project=d.project,
|
||||
intent=_short(intent, 800),
|
||||
context_excerpt=_short(context, 600),
|
||||
attempted_solution=_short(attempted, 600),
|
||||
outcome=outcome,
|
||||
reference_kind="none",
|
||||
reference="",
|
||||
tags=tags,
|
||||
source_sessions=[d.session_id],
|
||||
)
|
||||
)
|
||||
if len(tasks) >= max_tasks:
|
||||
break
|
||||
return tasks
|
||||
|
||||
|
||||
def dedup_tasks(tasks: List[TaskRecord]) -> List[TaskRecord]:
|
||||
"""Merge tasks sharing an id (same project+intent across sessions)."""
|
||||
by_id: dict = {}
|
||||
for t in tasks:
|
||||
if t.id in by_id:
|
||||
ex = by_id[t.id]
|
||||
ex.source_sessions = list(dict.fromkeys(ex.source_sessions + t.source_sessions))
|
||||
# prefer a resolved outcome if either session resolved it
|
||||
order = {"success": 3, "fail": 2, "mixed": 1, "unknown": 0}
|
||||
if order.get(t.outcome, 0) > order.get(ex.outcome, 0):
|
||||
ex.outcome = t.outcome
|
||||
else:
|
||||
by_id[t.id] = t
|
||||
return list(by_id.values())
|
||||
|
||||
|
||||
def assign_splits(
|
||||
tasks: List[TaskRecord],
|
||||
*,
|
||||
val_fraction: float = 0.34,
|
||||
test_fraction: float = 0.0,
|
||||
holdout_fraction: float | None = None, # legacy alias for val_fraction
|
||||
seed: int = 42,
|
||||
) -> List[TaskRecord]:
|
||||
"""Deterministically split tasks into train / val / test.
|
||||
|
||||
Anti-overfitting contract (the user's design):
|
||||
* ``val`` and ``test`` are drawn ONLY from REAL mined tasks (origin=='real')
|
||||
and never overlap. val gates updates; test is the final held-out measure.
|
||||
* ``train`` may include DREAM-augmented tasks (origin=='dream'); those are
|
||||
NEVER placed in val/test.
|
||||
|
||||
A stable hash of the task id keeps the same real task in the same split across
|
||||
nights (a fixed held-out gate, like SkillOpt's D_sel/D_test).
|
||||
|
||||
Back-compat: if ``test_fraction`` is 0 (default), this behaves like the old
|
||||
two-way replay/holdout split — real tasks divide into train + val, no test.
|
||||
``holdout_fraction`` is accepted as an alias for ``val_fraction``.
|
||||
"""
|
||||
if holdout_fraction is not None:
|
||||
val_fraction = holdout_fraction
|
||||
|
||||
dream = [t for t in tasks if t.origin == "dream"]
|
||||
real = [t for t in tasks if t.origin != "dream"]
|
||||
|
||||
# all dream tasks go to train, unconditionally
|
||||
for t in dream:
|
||||
t.split = "train"
|
||||
|
||||
val_cut = int(round(val_fraction * 100))
|
||||
test_cut = val_cut + int(round(test_fraction * 100))
|
||||
for t in real:
|
||||
bucket = int(hashlib.sha256((str(seed) + t.id).encode()).hexdigest(), 16) % 100
|
||||
if bucket < val_cut:
|
||||
t.split = "val"
|
||||
elif bucket < test_cut:
|
||||
t.split = "test"
|
||||
else:
|
||||
t.split = "train"
|
||||
|
||||
# guarantee val (the gate) is non-empty when we have >=2 real tasks
|
||||
real_splits = {t.split for t in real}
|
||||
if len(real) >= 2 and "val" not in real_splits:
|
||||
real[-1].split = "val"
|
||||
# guarantee a train pool exists (dream or real) when possible
|
||||
if not any(t.split == "train" for t in tasks) and len(real) >= 2:
|
||||
real[0].split = "train"
|
||||
# if test was requested but ended up empty with >=3 real tasks, carve one
|
||||
if test_fraction > 0 and len(real) >= 3 and not any(t.split == "test" for t in real):
|
||||
for t in real:
|
||||
if t.split == "train":
|
||||
t.split = "test"
|
||||
break
|
||||
return tasks
|
||||
|
||||
|
||||
def normalize_legacy_split(value: str) -> str:
|
||||
"""Map old split names to the new vocabulary."""
|
||||
return {"replay": "train", "holdout": "val"}.get(value, value)
|
||||
|
||||
|
||||
def mine(
|
||||
digests: List[SessionDigest],
|
||||
*,
|
||||
max_tasks: int = 40,
|
||||
candidate_limit: int = 0,
|
||||
holdout_fraction: float = 0.34,
|
||||
seed: int = 42,
|
||||
llm_miner: Optional[Callable[[List[SessionDigest]], List[TaskRecord]]] = None,
|
||||
target_skill_text: str = "",
|
||||
target_skill_path: str = "",
|
||||
) -> List[TaskRecord]:
|
||||
"""Top-level miner. Uses ``llm_miner`` if provided, else heuristic."""
|
||||
candidate_limit = candidate_limit or max_tasks
|
||||
tasks: List[TaskRecord] = []
|
||||
if llm_miner is not None:
|
||||
try:
|
||||
tasks = llm_miner(digests) or []
|
||||
except Exception:
|
||||
tasks = []
|
||||
if not tasks:
|
||||
tasks = heuristic_mine(digests, max_tasks=candidate_limit)
|
||||
tasks = dedup_tasks(tasks)
|
||||
if target_skill_text or target_skill_path:
|
||||
tasks = filter_tasks_for_target(tasks, target_skill_text, target_skill_path)
|
||||
tasks = tasks[:max_tasks]
|
||||
tasks = assign_splits(tasks, holdout_fraction=holdout_fraction, seed=seed)
|
||||
return tasks
|
||||
146
engineering/skillopt-sleep/skillopt_sleep/replay.py
Normal file
146
engineering/skillopt-sleep/skillopt_sleep/replay.py
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
"""SkillOpt-Sleep — Stage 3: replay.
|
||||
|
||||
Re-run mined TaskRecords offline under a given (skill, memory) and score
|
||||
them, producing the (hard, soft) signal SkillOpt's gate consumes.
|
||||
|
||||
Single-shot text replay by default. Tasks whose rule judge requires a tool
|
||||
call (gbrain's `tool_called`) are run through the backend's real tool loop
|
||||
(attempt_with_tools), so tool use is verified honestly rather than self-reported.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Tuple
|
||||
|
||||
from skillopt_sleep.backend import Backend
|
||||
from skillopt_sleep.types import ReplayResult, TaskRecord
|
||||
|
||||
|
||||
def _required_tools(task: TaskRecord) -> List[str]:
|
||||
"""Tool names a rule judge requires (op == 'tool_called')."""
|
||||
if task.reference_kind != "rule" or not task.judge:
|
||||
return []
|
||||
tools = []
|
||||
for c in task.judge.get("checks", []) or []:
|
||||
if isinstance(c, dict) and c.get("op") == "tool_called" and c.get("arg"):
|
||||
tools.append(str(c["arg"]))
|
||||
return tools
|
||||
|
||||
|
||||
def replay_one(backend: Backend, task: TaskRecord, skill: str, memory: str,
|
||||
sample_id: int = 0) -> ReplayResult:
|
||||
"""``sample_id`` distinguishes repeated dream rollouts of the same
|
||||
(task, skill, memory) in the attempt cache — without it all K rollouts
|
||||
collapse to one cached response and the contrastive signal is always 0."""
|
||||
import time
|
||||
tools = _required_tools(task)
|
||||
tools_called: List[str] = []
|
||||
t0 = time.time()
|
||||
tok_before = backend.tokens_used()
|
||||
if tools:
|
||||
response, tools_called = backend.attempt_with_tools(task, skill, memory, tools)
|
||||
else:
|
||||
response = backend.attempt(task, skill, memory, sample_id=sample_id)
|
||||
latency_ms = (time.time() - t0) * 1000.0
|
||||
tokens = max(0, backend.tokens_used() - tok_before)
|
||||
# if the backend doesn't track tokens (e.g. mock), approximate from text length
|
||||
if tokens == 0:
|
||||
tokens = (len(skill) + len(memory) + len(task.intent) + len(response)) // 4
|
||||
|
||||
# rule judges may need the detected tool calls; score locally when possible
|
||||
if task.reference_kind == "rule" and task.judge:
|
||||
from skillopt_sleep.judges import score_rule_judge
|
||||
hard, soft, rationale = score_rule_judge(task.judge, response, tools_called)
|
||||
else:
|
||||
hard, soft, rationale = backend.judge(task, response)
|
||||
|
||||
return ReplayResult(
|
||||
id=task.id,
|
||||
hard=float(hard),
|
||||
soft=float(soft),
|
||||
response=response,
|
||||
fail_reason="" if hard >= 1.0 else (rationale or "below threshold"),
|
||||
task_type=(task.tags[0] if task.tags else "task"),
|
||||
judge_rationale=rationale,
|
||||
tools_called=tools_called,
|
||||
tokens=int(tokens),
|
||||
latency_ms=round(latency_ms, 1),
|
||||
)
|
||||
|
||||
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
|
||||
def replay_batch(
|
||||
backend: Backend,
|
||||
tasks: List[TaskRecord],
|
||||
skill: str,
|
||||
memory: str,
|
||||
*,
|
||||
workers: int = 0,
|
||||
) -> List[Tuple[TaskRecord, ReplayResult]]:
|
||||
"""Replay tasks, optionally in parallel.
|
||||
|
||||
Real backends are network-bound, so a thread pool gives a large speedup on
|
||||
big test sets (like the research harness's --workers). ``workers`` defaults
|
||||
to env SKILLOPT_SLEEP_WORKERS or 1 (sequential). Mock stays sequential
|
||||
(deterministic) unless asked otherwise.
|
||||
"""
|
||||
if workers <= 0:
|
||||
workers = int(os.environ.get("SKILLOPT_SLEEP_WORKERS", "1") or "1")
|
||||
if workers <= 1 or len(tasks) <= 1:
|
||||
return [(t, replay_one(backend, t, skill, memory)) for t in tasks]
|
||||
results: List = [None] * len(tasks)
|
||||
with ThreadPoolExecutor(max_workers=min(workers, len(tasks))) as ex:
|
||||
futs = {ex.submit(replay_one, backend, t, skill, memory): i
|
||||
for i, t in enumerate(tasks)}
|
||||
for fut in futs:
|
||||
i = futs[fut]
|
||||
results[i] = (tasks[i], fut.result())
|
||||
return results
|
||||
|
||||
|
||||
def aggregate_scores(pairs: List[Tuple[TaskRecord, ReplayResult]]) -> Tuple[float, float]:
|
||||
if not pairs:
|
||||
return 0.0, 0.0
|
||||
hard = sum(r.hard for _t, r in pairs) / len(pairs)
|
||||
soft = sum(r.soft for _t, r in pairs) / len(pairs)
|
||||
return hard, soft
|
||||
|
||||
|
||||
def aggregate_cost(pairs: List[Tuple[TaskRecord, ReplayResult]]) -> Tuple[float, float]:
|
||||
"""Mean (tokens, latency_ms) per task — the cost objectives."""
|
||||
if not pairs:
|
||||
return 0.0, 0.0
|
||||
tok = sum(r.tokens for _t, r in pairs) / len(pairs)
|
||||
lat = sum(r.latency_ms for _t, r in pairs) / len(pairs)
|
||||
return tok, lat
|
||||
|
||||
|
||||
def multi_objective_reward(
|
||||
pairs: List[Tuple[TaskRecord, ReplayResult]],
|
||||
*,
|
||||
w_acc: float = 1.0,
|
||||
w_tokens: float = 0.0,
|
||||
w_latency: float = 0.0,
|
||||
token_ref: float = 2000.0,
|
||||
latency_ref_ms: float = 15000.0,
|
||||
) -> float:
|
||||
"""Weighted reward = accuracy↑, tokens↓, latency↓.
|
||||
|
||||
Cost terms are normalized against a reference and clamped to [0,1], so a
|
||||
response at/under the reference cost contributes ~1.0 and an expensive one
|
||||
less. Weights let the user trade off (default = accuracy only, backward
|
||||
compatible).
|
||||
"""
|
||||
if not pairs:
|
||||
return 0.0
|
||||
acc, _soft = aggregate_scores(pairs)
|
||||
tok, lat = aggregate_cost(pairs)
|
||||
tok_score = max(0.0, 1.0 - tok / max(1.0, token_ref)) if token_ref else 0.0
|
||||
lat_score = max(0.0, 1.0 - lat / max(1.0, latency_ref_ms)) if latency_ref_ms else 0.0
|
||||
total_w = w_acc + w_tokens + w_latency
|
||||
if total_w <= 0:
|
||||
return acc
|
||||
return (w_acc * acc + w_tokens * tok_score + w_latency * lat_score) / total_w
|
||||
|
||||
153
engineering/skillopt-sleep/skillopt_sleep/rollout.py
Normal file
153
engineering/skillopt-sleep/skillopt_sleep/rollout.py
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
"""SkillOpt-Sleep — multi-rollout + contrastive reflection (the imagination core).
|
||||
|
||||
The core idea: let the agent re-run the SAME task many times, then look at
|
||||
which rollouts went well vs badly and distill a rule from the *contrast*. This
|
||||
is a much stronger learning signal than a single failure, and it is the essence
|
||||
of the offline "dream/imagination" process — train-time rollouts are synthetic,
|
||||
so doing many is fine.
|
||||
|
||||
Pieces:
|
||||
* multi_rollout — run one task K times under (skill, memory), return scored attempts
|
||||
* contrastive_reflect — given good vs bad attempts of the same tasks, ask the
|
||||
optimizer what distinguishes them and propose a general rule
|
||||
|
||||
Driven through the Backend abstraction (mock/claude/codex), import-light.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from skillopt_sleep.backend import Backend, _extract_json
|
||||
from skillopt_sleep.replay import replay_one
|
||||
from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord
|
||||
|
||||
|
||||
@dataclass
|
||||
class RolloutSet:
|
||||
"""K scored attempts at one task under a fixed (skill, memory)."""
|
||||
task: TaskRecord
|
||||
attempts: List[ReplayResult] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def best(self) -> Optional[ReplayResult]:
|
||||
return max(self.attempts, key=lambda r: r.hard, default=None)
|
||||
|
||||
@property
|
||||
def worst(self) -> Optional[ReplayResult]:
|
||||
return min(self.attempts, key=lambda r: r.hard, default=None)
|
||||
|
||||
@property
|
||||
def spread(self) -> float:
|
||||
if not self.attempts:
|
||||
return 0.0
|
||||
hs = [r.hard for r in self.attempts]
|
||||
return max(hs) - min(hs)
|
||||
|
||||
@property
|
||||
def pass_rate(self) -> float:
|
||||
if not self.attempts:
|
||||
return 0.0
|
||||
return sum(1 for r in self.attempts if r.hard >= 1.0) / len(self.attempts)
|
||||
|
||||
|
||||
def multi_rollout(
|
||||
backend: Backend,
|
||||
task: TaskRecord,
|
||||
skill: str,
|
||||
memory: str,
|
||||
*,
|
||||
k: int = 3,
|
||||
workers: int = 0,
|
||||
) -> RolloutSet:
|
||||
"""Run ``task`` K times. replay_one is deterministic for mock; for real
|
||||
backends the model's own sampling yields variation across attempts.
|
||||
|
||||
The K attempts are independent, so they run concurrently (this is the dream
|
||||
phase's dominant cost). ``workers`` defaults to the SKILLOPT_SLEEP_WORKERS
|
||||
env (capped at k); set to 1 to force serial (used by the mock tests).
|
||||
"""
|
||||
import os
|
||||
rs = RolloutSet(task=task)
|
||||
k = max(1, k)
|
||||
if workers <= 0:
|
||||
try:
|
||||
workers = int(os.environ.get("SKILLOPT_SLEEP_WORKERS", "1"))
|
||||
except ValueError:
|
||||
workers = 1
|
||||
workers = max(1, min(workers, k))
|
||||
if workers == 1:
|
||||
for i in range(k):
|
||||
rs.attempts.append(replay_one(backend, task, skill, memory, sample_id=i))
|
||||
return rs
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||||
futs = [ex.submit(replay_one, backend, task, skill, memory, sample_id=i)
|
||||
for i in range(k)]
|
||||
for f in futs:
|
||||
rs.attempts.append(f.result())
|
||||
return rs
|
||||
|
||||
|
||||
def contrastive_reflect(
|
||||
backend: Backend,
|
||||
rollout_sets: List[RolloutSet],
|
||||
skill: str,
|
||||
memory: str,
|
||||
*,
|
||||
edit_budget: int = 4,
|
||||
target: str = "skill",
|
||||
) -> List[EditRecord]:
|
||||
"""Distill a rule from the contrast between good and bad attempts.
|
||||
|
||||
We pick tasks with the highest score *spread* (some attempts passed, some
|
||||
failed) — those are the most informative — and show the optimizer a
|
||||
high-scoring vs a low-scoring attempt of each, asking what general rule makes
|
||||
the good behavior reliable.
|
||||
"""
|
||||
informative = [rs for rs in rollout_sets if rs.spread > 0 and rs.best and rs.worst]
|
||||
informative.sort(key=lambda rs: rs.spread, reverse=True)
|
||||
informative = informative[:6]
|
||||
if not informative:
|
||||
return []
|
||||
|
||||
blocks = []
|
||||
for rs in informative:
|
||||
blocks.append(
|
||||
f"## Task: {rs.task.intent[:160]}\n"
|
||||
f"- GOOD attempt (score {rs.best.hard:.1f}): {rs.best.response[:200]}\n"
|
||||
f"- BAD attempt (score {rs.worst.hard:.1f}): {rs.worst.response[:200]}\n"
|
||||
f" (bad failed: {rs.worst.fail_reason[:100]})"
|
||||
)
|
||||
# the output contract the proposed rules must not violate (same guardrail the
|
||||
# single-shot reflect uses — prevents harness-violating rules like "return VBA"
|
||||
# or "ask the user for the range" on SpreadsheetBench).
|
||||
from skillopt_sleep.backend import _task_guardrail
|
||||
guard = _task_guardrail([(rs.task, rs.best) for rs in informative])
|
||||
prompt = (
|
||||
"You are SkillOpt's optimizer doing CONTRASTIVE reflection. For each task "
|
||||
"below the agent was run multiple times; some attempts succeeded and some "
|
||||
"failed. Identify what the GOOD attempts did that the BAD ones did not, "
|
||||
f"and propose at most {edit_budget} SHORT, GENERAL, reusable rules for the "
|
||||
f"{target} that would make the good behavior reliable every time. Quote "
|
||||
"concrete thresholds/formats verbatim; do not paraphrase vaguely. "
|
||||
"Every rule MUST obey the task output contract (if shown) — never propose "
|
||||
"a rule that changes the required output format/language or tells the agent "
|
||||
"to ask the user a question; such a rule scores ZERO.\n"
|
||||
f"{guard}"
|
||||
'Return ONLY a JSON array: '
|
||||
'[{"op":"add","content":"<rule>","rationale":"<what good did that bad didnt>"}].\n\n'
|
||||
+ "\n\n".join(blocks)
|
||||
)
|
||||
raw = backend._call(prompt, max_tokens=1024) # type: ignore[attr-defined]
|
||||
arr = _extract_json(raw, "array")
|
||||
edits: List[EditRecord] = []
|
||||
if isinstance(arr, list):
|
||||
for e in arr[:edit_budget]:
|
||||
if isinstance(e, dict) and str(e.get("content", "")).strip():
|
||||
edits.append(EditRecord(
|
||||
target=target, op=str(e.get("op", "add")).strip().lower(),
|
||||
content=str(e["content"]).strip(),
|
||||
rationale=str(e.get("rationale", "")).strip(),
|
||||
))
|
||||
return edits
|
||||
175
engineering/skillopt-sleep/skillopt_sleep/scheduler.py
Normal file
175
engineering/skillopt-sleep/skillopt_sleep/scheduler.py
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
"""SkillOpt-Sleep — built-in nightly scheduler.
|
||||
|
||||
Installs/removes a crontab entry that runs the sleep cycle automatically, so the
|
||||
user doesn't have to wire cron themselves. Idempotent: a managed block delimited
|
||||
by marker comments is added/replaced/removed in the user's crontab.
|
||||
|
||||
Design choices:
|
||||
* Off-:00 minute (3:17 local by default) so many users don't all hit the API
|
||||
at the same instant.
|
||||
* The entry runs `python -m skillopt_sleep run` for a specific project and
|
||||
appends to <project>/.skillopt-sleep/cron.log.
|
||||
* `schedule` is additive per project (keyed by project path); `unschedule`
|
||||
removes the project's line (or the whole managed block with --all).
|
||||
|
||||
cron is the portable mechanism on Linux/macOS. On systems without `crontab`,
|
||||
`schedule` prints the line and instructions instead of failing.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
_BEGIN = "# >>> skillopt-sleep (managed) >>>"
|
||||
_END = "# <<< skillopt-sleep (managed) <<<"
|
||||
|
||||
|
||||
def _have_crontab() -> bool:
|
||||
return shutil.which("crontab") is not None
|
||||
|
||||
|
||||
def _read_crontab() -> str:
|
||||
try:
|
||||
proc = subprocess.run(["crontab", "-l"], capture_output=True, text=True)
|
||||
return proc.stdout if proc.returncode == 0 else ""
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _write_crontab(content: str) -> bool:
|
||||
try:
|
||||
proc = subprocess.run(["crontab", "-"], input=content, text=True,
|
||||
capture_output=True)
|
||||
return proc.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _split_managed(crontab: str) -> Tuple[str, List[str]]:
|
||||
"""Return (text_outside_block, managed_lines_inside_block)."""
|
||||
lines = crontab.splitlines()
|
||||
outside: List[str] = []
|
||||
managed: List[str] = []
|
||||
in_block = False
|
||||
for ln in lines:
|
||||
if ln.strip() == _BEGIN:
|
||||
in_block = True
|
||||
continue
|
||||
if ln.strip() == _END:
|
||||
in_block = False
|
||||
continue
|
||||
(managed if in_block else outside).append(ln)
|
||||
return "\n".join(outside).rstrip(), managed
|
||||
|
||||
|
||||
def _runner_cmd(project: str, backend: str, extra: str, python: str) -> str:
|
||||
logdir = os.path.join(project, ".skillopt-sleep")
|
||||
log = os.path.join(logdir, "cron.log")
|
||||
# This line is written straight into the user's real crontab, which cron
|
||||
# runs through `sh -c` on every fire — shell-quote every interpolated
|
||||
# path (not just wrap in "..."), since a project dir containing a `"`,
|
||||
# `` ` ``, `$( )`, or `;` would otherwise break out of the quoted context.
|
||||
project_q, logdir_q, log_q = shlex.quote(project), shlex.quote(logdir), shlex.quote(log)
|
||||
repo_root_q = shlex.quote(_repo_root())
|
||||
# `extra` is only ever a hardcoded flag literal today ("" or
|
||||
# "--auto-adopt" from __main__.py), but quote it defensively token-by-
|
||||
# token (not the whole string as one blob, which would break a future
|
||||
# multi-flag `extra`) so this call site can't silently reopen the same
|
||||
# quoting gap just closed above.
|
||||
extra_q = " ".join(shlex.quote(t) for t in shlex.split(extra)) if extra else ""
|
||||
# use absolute python + -m so cron's minimal env still works
|
||||
cmd = (f'{shlex.quote(python)} -m skillopt_sleep run --project {project_q} '
|
||||
f'--scope invoked --backend {shlex.quote(backend)} {extra_q}'.rstrip())
|
||||
# cron.log accumulates this command's real stdout/stderr indefinitely,
|
||||
# which (unlike state.json/staged files) was never covered by this
|
||||
# plugin's chmod 0700/0600 hardening -- tighten it here too, best-effort
|
||||
# (2>/dev/null so a chmod failure, e.g. a non-POSIX filesystem, doesn't
|
||||
# block the actual run).
|
||||
return (f'mkdir -p {logdir_q} && chmod 700 {logdir_q} 2>/dev/null; '
|
||||
f'touch {log_q} && chmod 600 {log_q} 2>/dev/null; '
|
||||
f'cd {repo_root_q} && {cmd} >> {log_q} 2>&1')
|
||||
|
||||
|
||||
def _repo_root() -> str:
|
||||
# the package lives at <repo>/skillopt_sleep/; repo root is its parent
|
||||
return os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
|
||||
def _project_marker(project: str) -> str:
|
||||
return f"# project={os.path.abspath(project)}"
|
||||
|
||||
|
||||
def _line_matches_project(ln: str, marker: str) -> bool:
|
||||
"""True if ``ln`` is the managed cron line for this project's marker.
|
||||
|
||||
The marker is always appended as the last token of a generated line
|
||||
(see ``schedule()``'s ``cron_line`` construction), so anchor on that
|
||||
rather than a bare substring test -- ``marker not in ln`` would also
|
||||
match a *different* project whose absolute path happens to be a
|
||||
prefix of this one (e.g. scheduling/unscheduling ``/home/user/app``
|
||||
would silently drop ``/home/user/app-v2``'s line too, since
|
||||
"# project=/home/user/app" is a literal substring of
|
||||
"# project=/home/user/app-v2"). Mirrors the anchored comparison
|
||||
``harvest.py``'s ``_project_matches()`` already uses for the same
|
||||
class of path-prefix ambiguity.
|
||||
"""
|
||||
return ln.rstrip().endswith(marker)
|
||||
|
||||
|
||||
def schedule(project: str, *, backend: str = "mock", hour: int = 3, minute: int = 17,
|
||||
extra: str = "", python: Optional[str] = None) -> Tuple[bool, str]:
|
||||
"""Install (or replace) the nightly entry for ``project``.
|
||||
|
||||
Returns (installed, message). If crontab is unavailable, installed=False and
|
||||
the message contains the line to add manually.
|
||||
"""
|
||||
project = os.path.abspath(project)
|
||||
python = python or sys.executable or "python3"
|
||||
cron_line = f"{minute} {hour} * * * {_runner_cmd(project, backend, extra, python)} {_project_marker(project)}"
|
||||
|
||||
if not _have_crontab():
|
||||
return False, ("crontab not found on this system. Add this line to your "
|
||||
"scheduler manually:\n" + cron_line)
|
||||
|
||||
outside, managed = _split_managed(_read_crontab())
|
||||
# drop any existing line for this project, then add the new one
|
||||
marker = _project_marker(project)
|
||||
managed = [ln for ln in managed if not _line_matches_project(ln, marker) and ln.strip()]
|
||||
managed.append(cron_line)
|
||||
|
||||
block = _BEGIN + "\n" + "\n".join(managed) + "\n" + _END
|
||||
new_crontab = (outside + "\n\n" + block + "\n").lstrip("\n")
|
||||
ok = _write_crontab(new_crontab)
|
||||
if ok:
|
||||
return True, (f"Scheduled nightly at {hour:02d}:{minute:02d} for {project} "
|
||||
f"(backend={backend}). Logs -> {project}/.skillopt-sleep/cron.log\n"
|
||||
f"Runs `skillopt_sleep run`; it only STAGES a proposal — adopt is still manual.")
|
||||
return False, "Failed to write crontab. Line to add manually:\n" + cron_line
|
||||
|
||||
|
||||
def unschedule(project: Optional[str] = None, *, all_projects: bool = False) -> Tuple[bool, str]:
|
||||
"""Remove the entry for ``project`` (or the whole managed block with all_projects)."""
|
||||
if not _have_crontab():
|
||||
return False, "crontab not found; nothing to remove."
|
||||
outside, managed = _split_managed(_read_crontab())
|
||||
if all_projects:
|
||||
managed = []
|
||||
elif project:
|
||||
marker = _project_marker(project)
|
||||
managed = [ln for ln in managed if not _line_matches_project(ln, marker) and ln.strip()]
|
||||
if managed:
|
||||
block = _BEGIN + "\n" + "\n".join(managed) + "\n" + _END
|
||||
new_crontab = (outside + "\n\n" + block + "\n").lstrip("\n")
|
||||
else:
|
||||
new_crontab = outside.rstrip() + "\n"
|
||||
ok = _write_crontab(new_crontab)
|
||||
return ok, ("Removed." if ok else "Failed to update crontab.")
|
||||
|
||||
|
||||
def list_scheduled() -> List[str]:
|
||||
_outside, managed = _split_managed(_read_crontab())
|
||||
return [ln for ln in managed if ln.strip()]
|
||||
142
engineering/skillopt-sleep/skillopt_sleep/slow_update.py
Normal file
142
engineering/skillopt-sleep/skillopt_sleep/slow_update.py
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
"""SkillOpt-Sleep — slow update (cross-night long-term memory).
|
||||
|
||||
This is the deployment-time analogue of SkillOpt's epoch-wise slow/meta update
|
||||
(paper §3.6). Step-level edits (consolidate) learn from one night's batch; the
|
||||
slow update learns across nights and writes a durable "longitudinal guidance"
|
||||
block into a PROTECTED field of the skill that step-level edits never touch.
|
||||
|
||||
It reuses the exact protected-field marker convention from the main repo
|
||||
(``skillopt/optimizer/slow_update.py``) so the artifact is compatible:
|
||||
|
||||
<!-- SLOW_UPDATE_START --> ... <!-- SLOW_UPDATE_END -->
|
||||
|
||||
Why it matters: even when the user turns the validation gate OFF (greedy mode),
|
||||
the slow update still runs at the end of the run, so short-term nightly
|
||||
experience is consolidated into long-term memory rather than lost. The cross-night
|
||||
content is carried in ``state.slow_memory``.
|
||||
|
||||
Driven through the Backend abstraction (mock/claude/codex), so it stays
|
||||
import-light — no `openai` dependency.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from skillopt_sleep.backend import Backend, _extract_json
|
||||
from skillopt_sleep.types import ReplayResult, TaskRecord
|
||||
|
||||
|
||||
SLOW_UPDATE_START = "<!-- SLOW_UPDATE_START -->"
|
||||
SLOW_UPDATE_END = "<!-- SLOW_UPDATE_END -->"
|
||||
|
||||
|
||||
# ── protected-field helpers (mirror skillopt/optimizer/slow_update.py) ─────────
|
||||
|
||||
def has_slow_field(skill: str) -> bool:
|
||||
return SLOW_UPDATE_START in skill and SLOW_UPDATE_END in skill
|
||||
|
||||
|
||||
def extract_slow_field(skill: str) -> str:
|
||||
s = skill.find(SLOW_UPDATE_START)
|
||||
e = skill.find(SLOW_UPDATE_END)
|
||||
if s == -1 or e == -1:
|
||||
return ""
|
||||
return skill[s + len(SLOW_UPDATE_START):e].strip()
|
||||
|
||||
|
||||
def _strip_slow_fields(skill: str) -> str:
|
||||
while True:
|
||||
s = skill.find(SLOW_UPDATE_START)
|
||||
if s == -1:
|
||||
break
|
||||
e = skill.find(SLOW_UPDATE_END, s)
|
||||
if e == -1:
|
||||
skill = skill[:s]
|
||||
break
|
||||
skill = skill[:s] + skill[e + len(SLOW_UPDATE_END):]
|
||||
skill = skill.replace(SLOW_UPDATE_END, "")
|
||||
while "\n\n\n" in skill:
|
||||
skill = skill.replace("\n\n\n", "\n\n")
|
||||
return skill.rstrip()
|
||||
|
||||
|
||||
def replace_slow_field(skill: str, content: str) -> str:
|
||||
"""Set the protected slow-update field to ``content`` (exactly one block)."""
|
||||
base = _strip_slow_fields(skill)
|
||||
if not content.strip():
|
||||
return base
|
||||
block = f"\n\n{SLOW_UPDATE_START}\n{content.strip()}\n{SLOW_UPDATE_END}\n"
|
||||
return base + block
|
||||
|
||||
|
||||
# ── the slow-update synthesis ──────────────────────────────────────────────────
|
||||
|
||||
def _summarize_pairs(
|
||||
prev_pairs: List[Tuple[TaskRecord, ReplayResult]],
|
||||
curr_pairs: List[Tuple[TaskRecord, ReplayResult]],
|
||||
) -> str:
|
||||
"""Group adjacent-version outcomes into improved/regressed/persistent/stable."""
|
||||
prev_by = {t.id: r for t, r in prev_pairs}
|
||||
lines: List[str] = []
|
||||
counts = {"improved": 0, "regressed": 0, "persistent_fail": 0, "stable_success": 0}
|
||||
for t, r in curr_pairs:
|
||||
p = prev_by.get(t.id)
|
||||
if p is None:
|
||||
continue
|
||||
a, b = p.hard, r.hard
|
||||
if b > a:
|
||||
cat = "improved"
|
||||
elif b < a:
|
||||
cat = "regressed"
|
||||
elif b >= 1.0:
|
||||
cat = "stable_success"
|
||||
else:
|
||||
cat = "persistent_fail"
|
||||
counts[cat] += 1
|
||||
if cat in ("regressed", "persistent_fail") and len(lines) < 8:
|
||||
lines.append(f"- [{cat}] {t.intent[:120]} (why: {r.fail_reason[:80]})")
|
||||
head = ", ".join(f"{k}={v}" for k, v in counts.items())
|
||||
return head + ("\n" + "\n".join(lines) if lines else ""), counts # type: ignore[return-value]
|
||||
|
||||
|
||||
def run_slow_update(
|
||||
backend: Backend,
|
||||
*,
|
||||
prev_skill: str,
|
||||
curr_skill: str,
|
||||
prev_pairs: List[Tuple[TaskRecord, ReplayResult]],
|
||||
curr_pairs: List[Tuple[TaskRecord, ReplayResult]],
|
||||
prev_slow_content: str = "",
|
||||
) -> Optional[str]:
|
||||
"""Produce durable longitudinal guidance text (or None).
|
||||
|
||||
Compares behavior under the previous vs current skill across the same tasks
|
||||
and asks the optimizer to distill a short, durable guidance block — what to
|
||||
keep doing, what regressions to avoid — refining any prior slow-update text.
|
||||
"""
|
||||
summary, counts = _summarize_pairs(prev_pairs, curr_pairs) # type: ignore[misc]
|
||||
# nothing changed and no prior guidance to refine → skip
|
||||
if counts["regressed"] == 0 and counts["persistent_fail"] == 0 and not prev_slow_content:
|
||||
return None
|
||||
|
||||
prompt = (
|
||||
"You are SkillOpt's SLOW UPDATE — the long-term memory pass that runs "
|
||||
"across nights. Write a SHORT, durable guidance block (2-5 bullet "
|
||||
"points) capturing the longitudinal lessons: behaviors that reliably "
|
||||
"help and should be preserved, and regressions/persistent failures to "
|
||||
"avoid. Keep it GENERAL and stable (not tied to one task). If prior "
|
||||
"guidance is given, refine it rather than restate it.\n"
|
||||
'Return ONLY JSON: {"guidance": "<bullet list as one string>"}.\n\n'
|
||||
f"# Cross-night outcome summary\n{summary}\n\n"
|
||||
f"# Prior long-term guidance (refine this)\n{prev_slow_content or '(none)'}"
|
||||
)
|
||||
raw = backend._call(prompt, max_tokens=600) # type: ignore[attr-defined]
|
||||
obj = _extract_json(raw, "object")
|
||||
if isinstance(obj, dict):
|
||||
g = str(obj.get("guidance", "")).strip()
|
||||
if g:
|
||||
return g
|
||||
# fallback: if the model returned prose, keep the first ~400 chars
|
||||
text = (raw or "").strip()
|
||||
return text[:400] if text else None
|
||||
235
engineering/skillopt-sleep/skillopt_sleep/staging.py
Normal file
235
engineering/skillopt-sleep/skillopt_sleep/staging.py
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
"""SkillOpt-Sleep — Stage 5/6: staging and adoption.
|
||||
|
||||
Implements the Dreams safety contract: the cycle never mutates the user's
|
||||
live CLAUDE.md / SKILL.md. It writes proposals + a human-readable report into
|
||||
a staging directory; a separate, explicit `adopt` step copies them over the
|
||||
live files after taking a backup.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from skillopt_sleep.types import SleepReport
|
||||
|
||||
# Secret patterns scrubbed from any free-text we persist to the staging dir
|
||||
# (diagnostics, reports). Kept here so every on-disk artifact shares one
|
||||
# redaction pass; harvest_codex reuses these for session text too.
|
||||
_SECRET_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
# "sk-" is not OpenAI-specific: Anthropic (sk-ant-...) and other vendors
|
||||
# share the prefix, so the placeholder names the shape, not one vendor.
|
||||
(re.compile(r"sk-[A-Za-z0-9_-]{10,}"), "[REDACTED_API_KEY]"),
|
||||
# Distinctive vendor token prefixes (low false-positive: these prefixes do
|
||||
# not occur in normal diagnostic prose).
|
||||
(re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "[REDACTED_AWS_KEY]"),
|
||||
(re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b"), "[REDACTED_GITHUB_TOKEN]"),
|
||||
(re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{10,}\b"), "[REDACTED_SLACK_TOKEN]"),
|
||||
(re.compile(r"\bAIza[0-9A-Za-z_-]{20,}\b"), "[REDACTED_GOOGLE_KEY]"),
|
||||
# Bare JWT (three base64url segments) — e.g. a leaked bearer body without
|
||||
# the "Authorization:" prefix.
|
||||
(re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b"),
|
||||
"[REDACTED_JWT]"),
|
||||
(re.compile(r"(?i)(Authorization:\s*Bearer\s+)[^\s\"']+"), r"\1[REDACTED]"),
|
||||
(re.compile(r"(?i)(Authorization:\s*Basic\s+)[^\s\"']+"), r"\1[REDACTED]"),
|
||||
(
|
||||
re.compile(r"(?i)\b(api[_-]?key|token|password|secret)\b(\s*[:=]\s*)[^\s\"']+"),
|
||||
r"\1\2[REDACTED]",
|
||||
),
|
||||
(
|
||||
re.compile(r"(?i)\b(api[_-]?key|token|password|secret)\b(\s+)[^\s\"']+"),
|
||||
r"\1\2[REDACTED]",
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----",
|
||||
re.DOTALL,
|
||||
),
|
||||
"[REDACTED_PRIVATE_KEY]",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def redact_secrets(value: Any) -> Any:
|
||||
"""Scrub secret-looking substrings (API keys, bearer tokens, private keys)
|
||||
from a string, or recursively from the string leaves of a list/dict.
|
||||
|
||||
Used before writing backend stderr / optimizer replies / task responses to
|
||||
on-disk diagnostics: those are surfaced for debugging, but the underlying
|
||||
text (e.g. a codex 401 stderr dump) can carry credentials. Non-string
|
||||
scalars pass through unchanged.
|
||||
"""
|
||||
if isinstance(value, str):
|
||||
out = value
|
||||
for pattern, replacement in _SECRET_PATTERNS:
|
||||
out = pattern.sub(replacement, out)
|
||||
return out
|
||||
if isinstance(value, list):
|
||||
return [redact_secrets(v) for v in value]
|
||||
if isinstance(value, dict):
|
||||
return {k: redact_secrets(v) for k, v in value.items()}
|
||||
return value
|
||||
|
||||
|
||||
def _ts_dir() -> str:
|
||||
return time.strftime("%Y%m%d-%H%M%S", time.localtime())
|
||||
|
||||
|
||||
def _secure_dir(path: str) -> None:
|
||||
"""Restrict to owner-only. These directories hold real harvested session
|
||||
content (task intents, code excerpts, project context) in plaintext,
|
||||
which lands world-readable by default on a typical multi-user box unless
|
||||
we tighten it ourselves."""
|
||||
try:
|
||||
os.chmod(path, 0o700)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _secure_file(path: str) -> None:
|
||||
try:
|
||||
os.chmod(path, 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def staging_root(project: str) -> str:
|
||||
return os.path.join(project, ".skillopt-sleep", "staging")
|
||||
|
||||
|
||||
def latest_staging(project: str) -> Optional[str]:
|
||||
root = staging_root(project)
|
||||
if not os.path.isdir(root):
|
||||
return None
|
||||
subs = sorted(
|
||||
(os.path.join(root, d) for d in os.listdir(root)),
|
||||
key=lambda p: os.path.getmtime(p),
|
||||
reverse=True,
|
||||
)
|
||||
return subs[0] if subs else None
|
||||
|
||||
|
||||
def write_staging(
|
||||
project: str,
|
||||
*,
|
||||
report: SleepReport,
|
||||
proposed_skill: Optional[str],
|
||||
proposed_memory: Optional[str],
|
||||
live_skill_path: str,
|
||||
live_memory_path: str,
|
||||
report_md: str,
|
||||
redact: bool = True,
|
||||
) -> str:
|
||||
"""Write proposals + report into staging/<ts>/ and return that path.
|
||||
|
||||
``redact`` mirrors the config's ``redact_secrets`` flag (default True).
|
||||
Disabling it is honored — the user asked for it — but never silently:
|
||||
the caller is expected to have already logged a loud warning (see
|
||||
cycle.py) before passing False.
|
||||
"""
|
||||
root = staging_root(project)
|
||||
out = os.path.join(root, _ts_dir())
|
||||
# mode= closes the create-then-chmod race window for this leaf dir on a
|
||||
# first run (intermediate parents and pre-existing dirs still need the
|
||||
# explicit chmod below -- mode= only governs mkdir()'s own leaf, and is
|
||||
# itself subject to umask).
|
||||
os.makedirs(out, mode=0o700, exist_ok=True)
|
||||
# secure both the per-run leaf dir and the shared .skillopt-sleep root
|
||||
# (root may already exist from a prior night; still worth tightening).
|
||||
_secure_dir(os.path.dirname(root)) # <project>/.skillopt-sleep
|
||||
_secure_dir(out)
|
||||
|
||||
# reflect()'s prompt is built from real harvested session text, which can
|
||||
# carry anything a user pasted while debugging (an API key, a .env dump).
|
||||
# These are the files `adopt()` copies over the LIVE CLAUDE.md / SKILL.md
|
||||
# (with --auto-adopt, with no human in the loop) — scrub them the same as
|
||||
# every other on-disk artifact rather than only the diagnostics dump.
|
||||
#
|
||||
# report.md / report.json carry the same risk: EditRecord.content /
|
||||
# .rationale come from the optimizer's reflect() output over real failing
|
||||
# task responses, and report.md is the file the SKILL.md's own workflow
|
||||
# tells a human to read FIRST ("show the user the exact proposed edits").
|
||||
# Redact both — report_md as the already-rendered string, report.to_dict()
|
||||
# recursively (redact_secrets walks dict/list/str) before it hits JSON.
|
||||
if redact:
|
||||
proposed_skill = redact_secrets(proposed_skill)
|
||||
proposed_memory = redact_secrets(proposed_memory)
|
||||
report_md = redact_secrets(report_md)
|
||||
report_dict = redact_secrets(report.to_dict())
|
||||
else:
|
||||
report_dict = report.to_dict()
|
||||
|
||||
manifest = {
|
||||
"live_skill_path": live_skill_path,
|
||||
"live_memory_path": live_memory_path,
|
||||
"has_skill": proposed_skill is not None,
|
||||
"has_memory": proposed_memory is not None,
|
||||
"accepted": report.accepted,
|
||||
}
|
||||
if proposed_skill is not None:
|
||||
p = os.path.join(out, "proposed_SKILL.md")
|
||||
with open(p, "w", encoding="utf-8") as f:
|
||||
f.write(proposed_skill)
|
||||
_secure_file(p)
|
||||
if proposed_memory is not None:
|
||||
p = os.path.join(out, "proposed_CLAUDE.md")
|
||||
with open(p, "w", encoding="utf-8") as f:
|
||||
f.write(proposed_memory)
|
||||
_secure_file(p)
|
||||
p = os.path.join(out, "report.json")
|
||||
with open(p, "w", encoding="utf-8") as f:
|
||||
json.dump(report_dict, f, ensure_ascii=False, indent=2)
|
||||
_secure_file(p)
|
||||
p = os.path.join(out, "report.md")
|
||||
with open(p, "w", encoding="utf-8") as f:
|
||||
f.write(report_md)
|
||||
_secure_file(p)
|
||||
p = os.path.join(out, "manifest.json")
|
||||
with open(p, "w", encoding="utf-8") as f:
|
||||
json.dump(manifest, f, ensure_ascii=False, indent=2)
|
||||
_secure_file(p)
|
||||
return out
|
||||
|
||||
|
||||
def _backup(path: str, backup_dir: str) -> None:
|
||||
if os.path.exists(path):
|
||||
os.makedirs(backup_dir, mode=0o700, exist_ok=True)
|
||||
_secure_dir(backup_dir)
|
||||
shutil.copy2(path, os.path.join(backup_dir, os.path.basename(path)))
|
||||
|
||||
|
||||
def _adopt_one(staged_path: str, live: str, backup_dir: str) -> None:
|
||||
os.makedirs(os.path.dirname(live), exist_ok=True)
|
||||
_backup(live, backup_dir)
|
||||
# Defense-in-depth: write_staging() already redacted this content, but
|
||||
# re-scrub here too rather than trust that nothing touched the staged
|
||||
# file between `stage` and `adopt` (a human editing the proposal by hand
|
||||
# is exactly the case the staging step exists to allow).
|
||||
with open(staged_path, encoding="utf-8") as f:
|
||||
content = redact_secrets(f.read())
|
||||
with open(live, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
|
||||
|
||||
def adopt(staging_dir: str) -> List[str]:
|
||||
"""Copy staged proposals over the live files, backing up first.
|
||||
|
||||
Returns the list of live paths that were updated.
|
||||
"""
|
||||
with open(os.path.join(staging_dir, "manifest.json")) as f:
|
||||
manifest = json.load(f)
|
||||
backup_dir = os.path.join(staging_dir, "backup")
|
||||
updated: List[str] = []
|
||||
|
||||
if manifest.get("has_skill"):
|
||||
live = manifest["live_skill_path"]
|
||||
_adopt_one(os.path.join(staging_dir, "proposed_SKILL.md"), live, backup_dir)
|
||||
updated.append(live)
|
||||
if manifest.get("has_memory"):
|
||||
live = manifest["live_memory_path"]
|
||||
_adopt_one(os.path.join(staging_dir, "proposed_CLAUDE.md"), live, backup_dir)
|
||||
updated.append(live)
|
||||
return updated
|
||||
114
engineering/skillopt-sleep/skillopt_sleep/state.py
Normal file
114
engineering/skillopt-sleep/skillopt_sleep/state.py
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
"""SkillOpt-Sleep — persistent cross-night state.
|
||||
|
||||
state.json lives in ~/.skillopt-sleep and is the "long-term" store that
|
||||
turns nightly episodes into durable competence (the Agent-Sleep paper's
|
||||
short-term -> long-term transfer). It records:
|
||||
|
||||
- night counter
|
||||
- last harvest timestamp per project (so each night only sees new data)
|
||||
- cross-night "slow/meta" memory (lessons that persisted across nights)
|
||||
- per-night history (scores, accept/reject) for trend reporting
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
def _now_iso(clock: Optional[float] = None) -> str:
|
||||
# caller passes a timestamp; we avoid importing time at module import
|
||||
import time as _t
|
||||
return _t.strftime("%Y-%m-%dT%H:%M:%S", _t.localtime(clock if clock is not None else _t.time()))
|
||||
|
||||
|
||||
DEFAULT_STATE: Dict[str, Any] = {
|
||||
"version": 1,
|
||||
"night": 0,
|
||||
"last_harvest": {}, # project -> iso timestamp of last harvested record
|
||||
"slow_memory": "", # cross-night consolidated lessons (meta-skill analogue)
|
||||
"history": [], # list of per-night summaries
|
||||
"task_archive": [], # capped list of past mined tasks (for associative recall)
|
||||
}
|
||||
|
||||
|
||||
class SleepState:
|
||||
def __init__(self, path: str, data: Optional[Dict[str, Any]] = None) -> None:
|
||||
self.path = path
|
||||
self.data = data if data is not None else dict(DEFAULT_STATE)
|
||||
|
||||
# io ---------------------------------------------------------------------
|
||||
@classmethod
|
||||
def load(cls, path: str) -> "SleepState":
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
merged = dict(DEFAULT_STATE)
|
||||
merged.update(data if isinstance(data, dict) else {})
|
||||
return cls(path, merged)
|
||||
except Exception:
|
||||
pass
|
||||
return cls(path, dict(DEFAULT_STATE))
|
||||
|
||||
def save(self) -> None:
|
||||
# state.json carries real harvested session content (task intents,
|
||||
# code excerpts, project context) in plaintext, kept indefinitely —
|
||||
# tighten permissions rather than leave them at the process umask
|
||||
# default, which is world-readable on a typical multi-user box.
|
||||
state_dir = os.path.dirname(self.path)
|
||||
# mode= only governs the leaf dir mkdir() creates (intermediate
|
||||
# parents still fall back to the umask default) and is itself
|
||||
# subject to umask, so the chmod below still matters -- but passing
|
||||
# it here closes the window between creation and chmod for the
|
||||
# common case where state_dir doesn't already exist.
|
||||
os.makedirs(state_dir, mode=0o700, exist_ok=True)
|
||||
try:
|
||||
os.chmod(state_dir, 0o700)
|
||||
except OSError:
|
||||
pass
|
||||
tmp = self.path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(self.data, f, ensure_ascii=False, indent=2)
|
||||
try:
|
||||
os.chmod(tmp, 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
os.replace(tmp, self.path)
|
||||
|
||||
# accessors --------------------------------------------------------------
|
||||
@property
|
||||
def night(self) -> int:
|
||||
return int(self.data.get("night", 0))
|
||||
|
||||
def last_harvest_for(self, project: str) -> Optional[str]:
|
||||
return self.data.get("last_harvest", {}).get(project)
|
||||
|
||||
def set_last_harvest(self, project: str, iso_ts: str) -> None:
|
||||
self.data.setdefault("last_harvest", {})[project] = iso_ts
|
||||
|
||||
@property
|
||||
def slow_memory(self) -> str:
|
||||
return str(self.data.get("slow_memory", ""))
|
||||
|
||||
def set_slow_memory(self, content: str) -> None:
|
||||
self.data["slow_memory"] = content
|
||||
|
||||
def begin_night(self, clock: Optional[float] = None) -> int:
|
||||
self.data["night"] = self.night + 1
|
||||
return self.night
|
||||
|
||||
def record_night(self, summary: Dict[str, Any]) -> None:
|
||||
self.data.setdefault("history", []).append(summary)
|
||||
|
||||
# ── task archive (associative-recall memory) ──────────────────────────
|
||||
def task_archive(self) -> list:
|
||||
"""Past mined tasks as plain dicts (newest last)."""
|
||||
return list(self.data.get("task_archive", []))
|
||||
|
||||
def add_to_archive(self, task_dicts: list, cap: int = 300) -> None:
|
||||
"""Append tonight's tasks; keep only the most recent ``cap``."""
|
||||
arc = self.data.setdefault("task_archive", [])
|
||||
arc.extend(task_dicts)
|
||||
if len(arc) > cap:
|
||||
self.data["task_archive"] = arc[-cap:]
|
||||
81
engineering/skillopt-sleep/skillopt_sleep/tasks_file.py
Normal file
81
engineering/skillopt-sleep/skillopt_sleep/tasks_file.py
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
"""Reviewed task-file helpers for privacy-safe SkillOpt-Sleep runs."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from skillopt_sleep.mine import assign_splits, normalize_legacy_split
|
||||
from skillopt_sleep.types import TaskRecord
|
||||
|
||||
|
||||
def make_tasks_payload(
|
||||
tasks: List[TaskRecord],
|
||||
*,
|
||||
project: str,
|
||||
transcript_source: str = "",
|
||||
n_sessions: int = 0,
|
||||
target_skill_path: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
return {
|
||||
"format": "skillopt_sleep.tasks.v1",
|
||||
"project": project,
|
||||
"transcript_source": transcript_source,
|
||||
"n_sessions": n_sessions,
|
||||
"target_skill_path": target_skill_path,
|
||||
"reviewed": False,
|
||||
"tasks": [t.to_dict() for t in tasks],
|
||||
}
|
||||
|
||||
|
||||
def write_tasks_file(path: str, payload: Dict[str, Any]) -> str:
|
||||
out = os.path.abspath(os.path.expanduser(path))
|
||||
parent = os.path.dirname(out)
|
||||
if parent:
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
with open(out, "w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, ensure_ascii=False, indent=2)
|
||||
f.write("\n")
|
||||
return out
|
||||
|
||||
|
||||
def _normalize_tasks(
|
||||
tasks: List[TaskRecord],
|
||||
*,
|
||||
holdout_fraction: float,
|
||||
seed: int,
|
||||
) -> List[TaskRecord]:
|
||||
for task in tasks:
|
||||
task.split = normalize_legacy_split(task.split or "train")
|
||||
if len(tasks) >= 2 and not any(task.split in {"val", "test"} for task in tasks):
|
||||
tasks = assign_splits(tasks, holdout_fraction=holdout_fraction, seed=seed)
|
||||
return tasks
|
||||
|
||||
|
||||
def load_tasks_file(
|
||||
path: str,
|
||||
*,
|
||||
holdout_fraction: float = 0.34,
|
||||
seed: int = 42,
|
||||
) -> Tuple[List[TaskRecord], Dict[str, Any]]:
|
||||
source = os.path.abspath(os.path.expanduser(path))
|
||||
with open(source, encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
if isinstance(payload, list):
|
||||
meta: Dict[str, Any] = {"format": "skillopt_sleep.tasks.v1", "tasks_file": source}
|
||||
raw_tasks = payload
|
||||
elif isinstance(payload, dict):
|
||||
meta = {k: v for k, v in payload.items() if k != "tasks"}
|
||||
meta["tasks_file"] = source
|
||||
raw_tasks = payload.get("tasks", [])
|
||||
else:
|
||||
raise ValueError("tasks file must contain a JSON object with tasks or a JSON task array")
|
||||
if not isinstance(raw_tasks, list):
|
||||
raise ValueError("tasks file field 'tasks' must be an array")
|
||||
|
||||
tasks: List[TaskRecord] = []
|
||||
for item in raw_tasks:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("each task entry must be a JSON object")
|
||||
tasks.append(TaskRecord.from_dict(item))
|
||||
return _normalize_tasks(tasks, holdout_fraction=holdout_fraction, seed=seed), meta
|
||||
146
engineering/skillopt-sleep/skillopt_sleep/types.py
Normal file
146
engineering/skillopt-sleep/skillopt_sleep/types.py
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
"""SkillOpt-Sleep — core data types.
|
||||
|
||||
These dataclasses are the interfaces between the sleep-cycle stages
|
||||
(harvest -> mine -> replay -> consolidate -> stage). They are intentionally
|
||||
plain (no slots, no heavy deps) so the package imports cleanly on any
|
||||
Python 3.8+ interpreter and the deterministic experiment runs with zero
|
||||
external dependencies.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import Any, Dict, List
|
||||
|
||||
# ── Stage 1: harvest ──────────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class SessionDigest:
|
||||
"""A normalized summary of one local agent session transcript.
|
||||
|
||||
Produced by source-specific harvesters from Claude Code transcripts or
|
||||
Codex Desktop archived sessions.
|
||||
"""
|
||||
|
||||
session_id: str
|
||||
project: str
|
||||
git_branch: str = ""
|
||||
started_at: str = ""
|
||||
ended_at: str = ""
|
||||
user_prompts: List[str] = field(default_factory=list)
|
||||
assistant_finals: List[str] = field(default_factory=list)
|
||||
tools_used: List[str] = field(default_factory=list)
|
||||
files_touched: List[str] = field(default_factory=list)
|
||||
feedback_signals: List[str] = field(default_factory=list) # "still broken", "perfect", ...
|
||||
n_user_turns: int = 0
|
||||
n_assistant_turns: int = 0
|
||||
raw_path: str = ""
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
# ── Stage 2: mine ─────────────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class TaskRecord:
|
||||
"""A self-contained recurring task mined from one or more sessions.
|
||||
|
||||
This is the *training unit* of the sleep cycle — the analogue of a
|
||||
SkillOpt benchmark item.
|
||||
"""
|
||||
|
||||
id: str
|
||||
project: str
|
||||
intent: str # what the user wanted (the "question")
|
||||
context_excerpt: str = "" # minimal context needed to attempt it
|
||||
# Optional system framing for the rollout. When set (e.g. real benchmarks
|
||||
# carrying the research repo's exact rollout_system), the backend uses THIS
|
||||
# verbatim instead of its generic instruction wrapper — this keeps scoring
|
||||
# faithful to the source task and avoids re-deriving framing the benchmark
|
||||
# already bakes in.
|
||||
system: str = ""
|
||||
attempted_solution: str = "" # what the agent produced before
|
||||
outcome: str = "unknown" # success | fail | mixed | unknown
|
||||
reference_kind: str = "none" # exact | rubric | rule | none
|
||||
reference: str = "" # exact answer, or rubric text
|
||||
judge: Dict[str, Any] = field(default_factory=dict) # gbrain-style rule judge
|
||||
tags: List[str] = field(default_factory=list)
|
||||
source_sessions: List[str] = field(default_factory=list)
|
||||
# split ∈ {train, val, test}. val + test come ONLY from real mined tasks and
|
||||
# never overlap (val gates updates, test is the final held-out measure). train
|
||||
# may be dream-augmented (see origin). Legacy values replay->train,
|
||||
# holdout->val are normalized on load.
|
||||
split: str = "train"
|
||||
# origin ∈ {real, dream}. 'real' = mined from the user's actual sessions;
|
||||
# 'dream' = synthetic/augmented for the training pool. Dream tasks are NEVER
|
||||
# allowed into val/test, which is the anti-overfitting guarantee.
|
||||
origin: str = "real"
|
||||
derived_from: str = "" # for dream tasks: the real task id it varies
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: Dict[str, Any]) -> "TaskRecord":
|
||||
known = {f for f in cls.__dataclass_fields__} # type: ignore[attr-defined]
|
||||
return cls(**{k: v for k, v in d.items() if k in known})
|
||||
|
||||
|
||||
# ── Stage 3: replay ───────────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class ReplayResult:
|
||||
"""Outcome of re-running one TaskRecord offline under a given skill+memory."""
|
||||
|
||||
id: str
|
||||
hard: float = 0.0 # 0/1 exact, or continuous reward
|
||||
soft: float = 0.0 # partial credit / judge score 0..1
|
||||
response: str = ""
|
||||
fail_reason: str = ""
|
||||
task_type: str = "task"
|
||||
judge_rationale: str = ""
|
||||
tools_called: List[str] = field(default_factory=list)
|
||||
tokens: int = 0 # approx tokens this rollout cost (for token objective)
|
||||
latency_ms: float = 0.0 # wall-clock for this rollout (for latency objective)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
# ── Stage 4/5: consolidation report ───────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class EditRecord:
|
||||
"""One bounded edit proposed/applied to skill or memory."""
|
||||
|
||||
target: str # "skill" | "memory"
|
||||
op: str # add | delete | replace
|
||||
content: str = ""
|
||||
anchor: str = "" # for replace/delete: text being changed
|
||||
rationale: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SleepReport:
|
||||
"""Everything one night produced — written to staging for review."""
|
||||
|
||||
night: int
|
||||
project: str
|
||||
started_at: str = ""
|
||||
ended_at: str = ""
|
||||
n_sessions: int = 0
|
||||
n_tasks: int = 0
|
||||
n_replayed: int = 0
|
||||
baseline_score: float = 0.0
|
||||
candidate_score: float = 0.0
|
||||
accepted: bool = False
|
||||
gate_action: str = ""
|
||||
no_edits_reason: str = ""
|
||||
edits: List[EditRecord] = field(default_factory=list)
|
||||
rejected_edits: List[EditRecord] = field(default_factory=list)
|
||||
tokens_used: int = 0
|
||||
notes: List[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
d = asdict(self)
|
||||
return d
|
||||
128
engineering/skillopt-sleep/skills/skillopt-sleep/SKILL.md
Normal file
128
engineering/skillopt-sleep/skills/skillopt-sleep/SKILL.md
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
---
|
||||
name: skillopt-sleep
|
||||
description: "Use when the user wants their Claude agent to self-improve from past usage, asks about a nightly/offline 'sleep' or 'dream' cycle, memory/skill consolidation, or says things like 'make my agent better the more I use it', 'review my past sessions', 'learn my preferences', 'consolidate what you learned', 'run the sleep cycle', or wants to schedule offline self-optimization. Drives the skillopt_sleep engine: harvest past sessions -> mine recurring tasks -> replay offline -> consolidate validated CLAUDE.md and SKILL.md behind a held-out gate."
|
||||
---
|
||||
|
||||
# SkillOpt-Sleep: offline self-evolution for a local Claude agent
|
||||
|
||||
SkillOpt-Sleep gives the user's agent a **sleep cycle**. While the user is
|
||||
offline (e.g. nightly), it reviews their real past Claude Code sessions,
|
||||
re-runs recurring tasks on their own API budget, and consolidates what it
|
||||
learns into **memory** (`CLAUDE.md`) and **skills** (`SKILL.md`) — but only
|
||||
keeps changes that pass a held-out validation gate, and only after the user
|
||||
adopts them. The agent gets measurably better at *this* user's recurring work,
|
||||
with no model-weight training. It is the deployment-time analogue of training:
|
||||
short-term experience → long-term competence.
|
||||
|
||||
It synthesizes three ideas:
|
||||
- **SkillOpt** — the skill/memory doc is trainable text; bounded add/delete/replace
|
||||
edits; accepted only through a held-out gate; rejected edits become negative feedback.
|
||||
- **Claude Dreams** — offline consolidation that reads past sessions and rebuilds
|
||||
memory (dedup/merge/resolve); the input is never mutated; output is reviewed then adopted.
|
||||
- **Agent sleep** — periodic offline replay turns episodes into durable skill.
|
||||
|
||||
## When to use this skill
|
||||
|
||||
Trigger when the user wants any of:
|
||||
- "make my agent learn from how I use it" / "get better the more I use it" / "remember my preferences across sessions"
|
||||
- a nightly/scheduled or on-demand **offline self-improvement / dream / sleep** run
|
||||
- to **review past sessions/trajectories** and distill recurring tasks
|
||||
- to **consolidate** feedback into `CLAUDE.md` or a managed skill
|
||||
- to **schedule** the cycle (cron) or **adopt** a staged proposal
|
||||
|
||||
## The cycle (six stages)
|
||||
|
||||
1. **Harvest** — read `~/.claude/projects/*/<session>.jsonl` + `~/.claude/history.jsonl` (READ-ONLY) → session digests.
|
||||
2. **Mine** — digests → `TaskRecord`s (recurring intents + outcome labels + checkable refs where possible).
|
||||
3. **Replay** — re-run tasks offline under the *current* skill+memory → (hard, soft) scores.
|
||||
4. **Consolidate** — reflect on failures → propose bounded edits → **gate** on a held-out slice; accept only if it strictly improves.
|
||||
5. **Stage** — write `proposed_CLAUDE.md`, `proposed_SKILL.md`, a diff, and `report.md` into `<project>/.skillopt-sleep/staging/<date>/`. **Nothing live changes.**
|
||||
6. **Adopt** — explicit (or opt-in auto): copy staged files over live ones, backing up first.
|
||||
|
||||
## How to drive it
|
||||
|
||||
Prefer the `/skillopt-sleep` command. Under the hood it calls the bundled runner:
|
||||
|
||||
```bash
|
||||
"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" status # what's happened
|
||||
"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" dry-run --project "$(pwd)" # safe preview
|
||||
"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" run --project "$(pwd)" # full cycle, stages a proposal
|
||||
"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" adopt --project "$(pwd)" # apply staged proposal (with backup)
|
||||
```
|
||||
|
||||
- Default backend is `mock` (deterministic, **no API spend**) — good for trying the plumbing.
|
||||
- Add `--backend claude` or `--backend codex` to spend the user's real budget for genuine improvement.
|
||||
- Scope defaults to the invoked project; `--scope all` harvests every project.
|
||||
|
||||
### Scheduling
|
||||
|
||||
```bash
|
||||
"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" schedule --project "$(pwd)" --hour 3 --minute 17
|
||||
"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" unschedule --project "$(pwd)"
|
||||
```
|
||||
|
||||
Installs a nightly cron entry. `unschedule --all` removes every managed entry.
|
||||
|
||||
## All CLI flags
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--project PATH` | cwd | Project directory to evolve |
|
||||
| `--scope all\|invoked` | invoked | Harvest scope |
|
||||
| `--backend mock\|claude\|codex\|copilot` | mock | Replay backend (mock = no API spend) |
|
||||
| `--model NAME` | backend default | Override the model used for replay |
|
||||
| `--source claude\|codex\|auto` | claude | Transcript source |
|
||||
| `--lookback-hours N` | 72 | Harvest window |
|
||||
| `--max-sessions N` | unlimited | Cap harvested sessions |
|
||||
| `--max-tasks N` | 40 | Cap mined tasks |
|
||||
| `--target-skill-path PATH` | auto | Explicit SKILL.md to evolve |
|
||||
| `--tasks-file PATH` | — | Reviewed TaskRecord JSON (skip harvest) |
|
||||
| `--progress` | off | Print phase progress to stderr |
|
||||
| `--auto-adopt` | off | Auto-adopt if gate passes |
|
||||
| `--edit-budget N` | 4 | Max bounded edits per night |
|
||||
| `--json` | off | Machine-readable JSON output |
|
||||
|
||||
## Config keys (`~/.skillopt-sleep/config.json`)
|
||||
|
||||
Beyond the CLI flags, advanced behavior is controlled via config:
|
||||
|
||||
- **`preferences`** — free-text house rules injected into the optimizer's reflect step (e.g. "Always use async/await", "Answers in `\boxed{}`").
|
||||
- **`gate_mode`** — `on` (default, validation-gated) or `off` (greedy, accept all edits).
|
||||
- **`gate_metric`** — `hard`, `soft`, or `mixed` (default). Controls how the held-out gate scores.
|
||||
- **`dream_rollouts`** — >1 enables multi-rollout contrastive reflection per task.
|
||||
- **`recall_k`** — >0 recalls K similar past tasks into the dream (long-term memory).
|
||||
- **`evolve_memory`** / **`evolve_skill`** — independently toggle CLAUDE.md vs SKILL.md consolidation.
|
||||
|
||||
## Memory consolidation
|
||||
|
||||
The sleep cycle can consolidate both:
|
||||
- **SKILL.md** — the managed skill file (bounded edits: add/delete/replace)
|
||||
- **CLAUDE.md** — the project memory (same bounded edits)
|
||||
|
||||
Both are gated by the same held-out validation score. Set `evolve_memory: false` to consolidate only skills, or `evolve_skill: false` for only memory.
|
||||
|
||||
## Hard rules
|
||||
|
||||
- **Never** hand-edit the user's `CLAUDE.md` / `SKILL.md` as part of this skill.
|
||||
Only the `adopt` action changes live files, and it backs them up first.
|
||||
- Harvest is read-only. `mock` replay has no side effects.
|
||||
- Always show the user the **held-out baseline → candidate** score and the
|
||||
exact proposed edits before suggesting adoption. Evidence before adoption.
|
||||
- If asked whether it really helps, run
|
||||
`python -m skillopt_sleep.experiments.run_experiment --persona researcher --json`
|
||||
— a deterministic demo that proves held-out lift and that the gate blocks
|
||||
harmful edits.
|
||||
|
||||
## Validate / demo
|
||||
|
||||
```bash
|
||||
# deterministic proof (no API): held-out score rises, gate blocks regressions
|
||||
python -m skillopt_sleep.experiments.run_experiment --persona researcher --assert-improves
|
||||
python -m skillopt_sleep.experiments.run_experiment --persona programmer --assert-improves
|
||||
```
|
||||
|
||||
See the upstream SkillOpt-Sleep guide section
|
||||
(https://microsoft.github.io/SkillOpt/docs/guideline.html#sleep) for recorded
|
||||
output and the full design. (The original repo-relative design-doc path,
|
||||
`docs/superpowers/specs/...`, is not vendored into this repo — see this
|
||||
skill's README.md "What was and wasn't vendored" table.)
|
||||
22
productivity/fable-goal/.claude-plugin/plugin.json
Normal file
22
productivity/fable-goal/.claude-plugin/plugin.json
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
{
|
||||
"name": "fable-goal",
|
||||
"description": "Convert a rambling description of a desired outcome into one polished, autonomous /goal prompt ready to paste into a fresh session. Use when the user says '/fable-goal', 'turn this into a goal prompt', 'write me a fable prompt', or rambles about something they want made and asks for the prompt that makes it happen. Extracts deliverable/quantity/stakes/tools/destination, asks at most one question batch, verifies every named resource against the live environment, writes a 150-350 word prose prompt with the seven-part anatomy (desire + stakes, quality bar, tool inventory + discovery mandate, creative-freedom grant, medium-matched verification loop, delivery, goal line + autonomy directive), and self-checks six binary criteria before delivering. The output is a single copy-paste prompt, never the build itself.",
|
||||
"version": "2.11.1",
|
||||
"author": {
|
||||
"name": "Alireza Rezvani",
|
||||
"url": "https://alirezarezvani.com"
|
||||
},
|
||||
"homepage": "https://github.com/alirezarezvani/claude-skills/tree/main/productivity/fable-goal",
|
||||
"repository": "https://github.com/alirezarezvani/claude-skills",
|
||||
"license": "MIT",
|
||||
"skills": [
|
||||
"./skills/fable-goal"
|
||||
],
|
||||
"attribution": {
|
||||
"upstream": "https://github.com/duncan-buildroom/freeskills",
|
||||
"upstream_skill": "fable-goal",
|
||||
"upstream_license": "No formal license file; repo states 'Free to use and modify. Made for builders.' — an informal use-and-modify grant, not an SPDX license.",
|
||||
"changes": "Substantial restructure: wrong-tool check, observable-done principle, six-slot extraction table, verification-defaults-by-medium table, six-point pre-delivery self-check, failure-mode catalog reference doc, second worked example (non-web medium), /cs:fable-goal command, goal_prompt_self_check.py (stdlib runner for the mechanically checkable self-check subset).",
|
||||
"license_note": "The MIT license declared above covers the rewritten text and structure authored in this repository. The upstream's informal grant is quoted verbatim; it is not relicensed as MIT."
|
||||
}
|
||||
}
|
||||
33
productivity/fable-goal/README.md
Normal file
33
productivity/fable-goal/README.md
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
# fable-goal
|
||||
|
||||
Turn a ramble about something you want made into **one polished, autonomous /goal prompt** — copy-paste ready for a fresh session. The skill writes the prompt, never the build.
|
||||
|
||||
## What this skill does
|
||||
|
||||
You describe an outcome in fragments ("I want like 5 landing pages for my prompt pack, crazy good, put them up somewhere"). The skill:
|
||||
|
||||
1. **Extracts** the six slots a goal prompt needs: deliverable, quantity, audience/stakes, tools named, quality bar, destination
|
||||
2. **Fills gaps** from your brand profile (if you keep one) or sensible defaults — asking at most ONE question batch, never interviewing in rounds
|
||||
3. **Verifies before naming** — every skill, path, or MCP the prompt mentions is checked against the live environment so the fresh session never chases phantoms
|
||||
4. **Writes** a 150–350 word flowing-prose prompt with the seven-part anatomy: desire + stakes → quality bar → tool inventory + discovery mandate → creative-freedom grant → verification loop matched to the medium → delivery → goal line + autonomy directive
|
||||
5. **Self-checks** against six binary criteria before delivering, then outputs the prompt in one fenced block plus a short Assumptions list
|
||||
|
||||
## The philosophy
|
||||
|
||||
Get out of the model's way. A great /goal prompt nails the *what*, grants explicit freedom on the *how*, and demands self-verification before done — with a completion condition the session can observe itself (load the page, run the script, watch the render).
|
||||
|
||||
## Usage
|
||||
|
||||
```
|
||||
/cs:fable-goal I keep manually sorting the podcast files my editor sends, write me a prompt so a session builds me something for that
|
||||
```
|
||||
|
||||
Or just ramble and say "turn this into a goal prompt."
|
||||
|
||||
## Design notes
|
||||
|
||||
Unlike sibling `productivity/*` plugins, this one intentionally ships no `agents/` persona and no `assets/`: the skill is a single reasoning pass with nothing to orchestrate in parallel, invoked via `/cs:fable-goal`, and its output is one prose block with no templates to fill. `scripts/goal_prompt_self_check.py` covers the mechanically checkable subset of the pre-delivery self-check (word count, goal line, autonomy directive, verification/freedom/destination language); deliverable concreteness and live-environment resource verification remain author judgment.
|
||||
|
||||
## Attribution
|
||||
|
||||
Derived from [duncan-buildroom/freeskills](https://github.com/duncan-buildroom/freeskills) `fable-goal` ("free to use and modify. Made for builders."). This version is a substantial restructure: wrong-tool check, observable-done principle, verification-defaults-by-medium table, six-point pre-delivery self-check, failure-mode catalog, and a second worked example in a non-web medium.
|
||||
33
productivity/fable-goal/commands/cs-fable-goal.md
Normal file
33
productivity/fable-goal/commands/cs-fable-goal.md
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
---
|
||||
name: "cs-fable-goal"
|
||||
argument-hint: "[ramble about the thing you want made]"
|
||||
description: "/cs:fable-goal — Turn a ramble about something you want made into one polished, autonomous /goal prompt (copy-paste ready). Extracts deliverable/quantity/stakes/tools/destination, asks at most one question batch, verifies every named resource exists, writes a 150–350 word prose prompt with the seven-part anatomy, and self-checks before delivering."
|
||||
---
|
||||
|
||||
# /cs:fable-goal — Goal Prompt Writer
|
||||
|
||||
**Command:** `/cs:fable-goal <ramble>`
|
||||
|
||||
Converts a rambling description of a desired outcome into a single polished /goal prompt for a fresh autonomous session. The output is the prompt, never the build.
|
||||
|
||||
## When to Run
|
||||
|
||||
- You know what you want made but not how to ask for it well
|
||||
- Voice-to-text rambles ("I want like 5 landing pages, crazy good, put them up somewhere")
|
||||
- You're about to kick off a fresh autonomous session and want the prompt engineered first
|
||||
|
||||
## When NOT to Run
|
||||
|
||||
- You want the thing built right now in this session — just ask for it directly
|
||||
- You already have a well-formed prompt and want it executed
|
||||
|
||||
## What You Get
|
||||
|
||||
1. One fenced code block containing the finished /goal prompt (150–350 words, flowing first-person prose) with all seven anatomy parts: desire + stakes, quality bar, verified tool inventory + discovery mandate, creative-freedom grant, medium-matched verification loop, delivery destination, and the closing goal line + autonomy directive
|
||||
2. A 2–4 bullet **Assumptions** list so you can correct any gap-fill with one line instead of re-rambling
|
||||
|
||||
## Process (enforced by the skill)
|
||||
|
||||
Extract the six slots from the ramble → fill gaps from your brand profile or defaults, asking at most ONE question batch → verify every resource the prompt will name actually exists → write the prompt → run the six-point self-check → deliver.
|
||||
|
||||
See `skills/fable-goal/SKILL.md` for the full anatomy, verification-by-medium table, and anti-pattern list.
|
||||
79
productivity/fable-goal/skills/fable-goal/SKILL.md
Normal file
79
productivity/fable-goal/skills/fable-goal/SKILL.md
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
---
|
||||
name: fable-goal
|
||||
description: Convert a rambling description of a desired outcome into one polished, autonomous /goal prompt ready to paste into a fresh session. Use when the user says "/fable-goal", "turn this into a goal prompt", "write me a fable prompt", "write the prompt that builds X", or rambles about something they want made and asks for the prompt that makes it happen. The output is a single copy-paste prompt, never the build itself. Do NOT use when the user wants the thing built right now in this session — only when they want the PROMPT that will make it happen in a fresh session.
|
||||
---
|
||||
|
||||
# Fable Goal Prompt Writer
|
||||
|
||||
Turn the user's ramble into one exceptional /goal prompt they can paste into a fresh autonomous session. You are not building the thing. You are designing the prompt that builds the thing.
|
||||
|
||||
**Wrong-tool check first.** If the user actually wants the deliverable built now, say so in one line and offer to build it — don't write a prompt nobody will paste.
|
||||
|
||||
## The philosophy
|
||||
|
||||
**Get out of the model's way.** A capable model can do almost anything if the prompt (1) articulates the desire clearly, (2) hands it tools, and (3) gives it a way to verify its own work. A great /goal prompt does not micromanage the how. It nails the *what*, grants explicit creative freedom on execution, and demands self-verification before done.
|
||||
|
||||
The corollary most prompts miss: **done must be observable.** Every deliverable needs a completion condition the session can check itself — a page that loads, a script that runs on real input, a link that resolves. "Make it good" is a wish; "load each page and click every element before you ok it" is a verification loop.
|
||||
|
||||
**Brand profile (optional).** If the user keeps one — a `brand.md` in this folder, or `~/.claude/CLAUDE.md` / `~/.claude/brand-profile.md` — read it once at the start: proof points, audience numbers, design system, asset paths, default destinations, voice rules, preferred MCPs. Pull in ONLY entries the task touches. No profile is fine; a profile just removes questions.
|
||||
|
||||
## Process
|
||||
|
||||
### 1. Extract what the ramble already contains
|
||||
|
||||
People think in fragments, especially over voice-to-text. Interpret intent over literal words ("Quad MD" means CLAUDE.md, "Netlefi" means Netlify). Pull out six slots: **deliverable** (the concrete thing), **quantity**, **audience/stakes** (who sees it, real numbers), **tools named**, **quality bar** (adjectives, comparisons), **destination** (hosted link, folder, post, file).
|
||||
|
||||
### 2. Fill gaps with defaults; ask only when it matters
|
||||
|
||||
Synthesize small gaps yourself, using the brand profile if one exists. Ask ONLY when the answer would meaningfully change the prompt: **outcome** (can't tell what the deliverable is), **scale** (5 vs 50 changes the shape and nothing implies it), **destination** (can't infer where results land), **assets** (a needed input like a logo or source file that nothing supplies), **brand facts** (public-facing output with no profile — ask for the one or two that raise the bar).
|
||||
|
||||
If you ask, ask everything in ONE AskUserQuestion batch, then write. Never interview in rounds. If the ramble (plus profile) covers the basics, ask nothing and note assumptions instead.
|
||||
|
||||
### 3. Verify before you name
|
||||
|
||||
A prompt that points at a path, capability, or MCP that does not exist sends the fresh session on a dead-end hunt. Spend 30 seconds confirming every resource you plan to name: `ls` the paths, glance at the available-tools list. The live environment is the source of truth, not the profile. Name only what you verified AND what is load-bearing (usually 2–4 things); everything else is the discovery mandate's job.
|
||||
|
||||
### 4. Write the prompt: the seven-part anatomy
|
||||
|
||||
Weave all seven parts as natural flowing prose — no headers, no bulleted spec. First person, as the user speaking to the session:
|
||||
|
||||
1. **Desire + stakes.** Concrete deliverable, concrete quantity, why it matters. If an audience will see it, say so with the real number. Never invent stakes — real stakes make the model try harder; fake ones are noise.
|
||||
2. **Quality bar.** What excellent looks like, in a sentence or two. For creative work, vivid adjectives beat specs; for functional work, concrete behavior beats adjectives.
|
||||
3. **Tool inventory + discovery mandate.** Name the verified resources, sketch ONE example workflow as a suggestion, then release it: "you can accomplish this many ways." Then grant discovery — "before you start, take stock of the tools and MCPs you actually have, and go find or fetch any references, libraries, or assets you need along the way; the internet is available to you."
|
||||
4. **Creative freedom + decision authority.** Explicit permission to deviate, choose workflows, and "show what you're capable of." Never skip this. Anything named is a suggestion the session may swap for something better; every mid-run judgment call gets decided by the session with taste, not deferred back.
|
||||
5. **Verification loop.** Default: at least three iteration passes — going back through the finished output with a fine-toothed comb for problems and improvements. Define the pass in the medium's own terms: load the page and click through it, run the script on real input, render and watch the video. See [references/goal_prompt_patterns.md](references/goal_prompt_patterns.md) for per-medium defaults.
|
||||
6. **Delivery.** Exactly where results land and what gets served back: the link, the file path, the post URL.
|
||||
7. **Goal line + autonomy directive.** Close with one sentence: "[X with Y and Z] is your /goal. Work completely autonomously and do not ask me for anything until you are all done." For 3+ independent units, add a nudge to parallelize with subagents.
|
||||
|
||||
**Length target: 150–350 words** — long enough for stakes, tools, and verification; short enough that nothing dilutes the goal line.
|
||||
|
||||
### 5. Self-check, then deliver
|
||||
|
||||
Fix any miss before shipping: (1) deliverable + quantity concrete enough that a stranger could judge done; (2) every named resource verified in step 3; (3) creative-freedom clause present; (4) verification loop present and matched to the medium; (5) destination explicit; (6) ends with the goal line + autonomy directive, 150–350 words.
|
||||
|
||||
The mechanically checkable subset (word count, goal line, autonomy directive, verification/freedom/destination language) can be run: `python3 scripts/goal_prompt_self_check.py -` with the draft on stdin — exit 0 means those checks pass; judgment calls (1) and (2) stay yours.
|
||||
|
||||
Output the prompt in a single fenced code block, then a short **Assumptions** list (2–4 bullets) so the user can correct any gap-fill with one line. Nothing else — no preamble above the code block.
|
||||
|
||||
## Anti-patterns
|
||||
|
||||
Never ship a prompt that: **micromanages the how** (step-by-step strangles the session); **names phantom resources** (unverified = dead-end hunt); **has unverifiable done-ness** (session declares victory early); **buries the goal line** (nothing comes after it); **invents stakes** (false urgency trains the user to write fiction); or follows an **interview spiral** (one question batch max). Failure-mode detail and a second worked example (non-web medium) live in [references/goal_prompt_patterns.md](references/goal_prompt_patterns.md).
|
||||
|
||||
## Example
|
||||
|
||||
**Ramble:** "I want like 5 different landing pages for my free prompt pack thing, they should all look totally different and crazy good, put them up somewhere I can look at them"
|
||||
|
||||
**Output:**
|
||||
|
||||
```
|
||||
I want you to build 5 landing pages for my free prompt pack, each one fundamentally different from the others, as a way to show me the strongest possible range of directions before I pick one. These will be seen by my audience, so the bar is high: exceptional typography, striking layouts, and motion that feels designed rather than templated. Each page needs a headline, proof, and a single email-capture CTA for the prompt pack. You have total creative freedom on the visual direction of each one. You can generate any imagery you need with whatever image tools you have available, and you can accomplish this in many ways using many workflows, so before you start, take stock of the tools and MCPs you actually have, go find or fetch any references, libraries, or assets you need along the way, and show me what you are capable of. Before you ok each page, do at least three iteration passes: load the live page, click through every element, check it at mobile width, and go back through with a fine-toothed comb looking for design problems and opportunities to improve. Parallelize across subagents so the pages develop independently. When all 5 are done, deploy them to Netlify and serve me the 5 links with a one-line description of each direction. 5 fundamentally different prompt pack landing pages, live on Netlify with three iteration passes each, is your /goal. Work completely autonomously and do not ask me for anything until you are all done.
|
||||
```
|
||||
|
||||
**Assumptions:**
|
||||
- CTA is email capture for the prompt pack (review candidates, not live pages — no marketing-automation wiring)
|
||||
- Netlify for hosting since you said "put them up somewhere"
|
||||
- Each page gets a distinct visual direction so you see the full range before committing
|
||||
|
||||
---
|
||||
|
||||
*Derived from [duncan-buildroom/freeskills](https://github.com/duncan-buildroom/freeskills) `fable-goal` ("free to use and modify"). Substantially restructured: wrong-tool check, observable-done principle, six-slot extraction, per-medium verification defaults, six-point self-check, anti-pattern list, second worked example.*
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
# Goal Prompt Patterns — why the anatomy works
|
||||
|
||||
Supporting rationale for the seven-part anatomy in SKILL.md, plus the failure-mode catalog in long form. Read this when adapting the skill to a new medium or when a generated prompt underperforms.
|
||||
|
||||
## Why each part exists
|
||||
|
||||
**1. Desire + stakes.** Autonomous sessions calibrate effort to perceived importance. A deliverable with a named audience and a real number ("my 40k newsletter subscribers will see this") measurably shifts output quality versus "make me a page." The constraint: stakes must be true. A user who learns to invent audience numbers to juice quality is being trained to lie to their own tools, and the model eventually calibrates to the inflation.
|
||||
|
||||
**2. Quality bar.** Adjectives outperform specs for creative work because specs anchor the model to a checklist floor while adjectives set a ceiling to reach for. "Otherworldly beautiful animations" produces more ambitious work than "use CSS transitions of at least 300ms." For non-creative work (a CLI, a pipeline), invert this: concrete behavior beats adjectives ("a dry-run mode that shows me what it would do" > "make it safe").
|
||||
|
||||
**3. Tool inventory + discovery mandate.** Two failure modes bracket this part. Naming too much: every named tool is an implicit instruction, and ten named tools reads as a workflow spec — the thing the philosophy says not to write. Naming nothing: the session works from memory, guesses at what exists, and stops short when it hits a gap. The fix is 2–4 verified, load-bearing names plus one sentence of discovery mandate. The discovery sentence matters more than the inventory: it converts "I don't have X" from a stopping condition into a search task.
|
||||
|
||||
**4. Creative freedom + decision authority.** Without an explicit grant, capable models default to conservative interpretation and mid-run check-ins — both poison for an autonomous run. The grant has two halves that are easy to conflate: *creative* freedom (visual/structural choices) and *decision* authority (judgment calls like naming, scope edges, tradeoffs get made, not deferred). A prompt can grant the first and still get a session that stops to ask about the second.
|
||||
|
||||
**5. Verification loop.** The single highest-leverage part. An autonomous session with no self-check declares victory at first plausible output. "Three iteration passes" works because it is countable — the session can verify its own compliance — and because pass 1 catches errors, pass 2 catches what pass 1's fixes broke, and pass 3 is where polish happens. The pass must be defined in the medium's own terms (load the page, run the script, watch the render); "review your work" without a medium-specific action degrades into re-reading the code.
|
||||
|
||||
**6. Delivery.** Ambiguous destinations produce orphaned work: builds that finish in a scratch directory nobody looks at. Naming the destination also names the hand-back artifact (the link, the path), which becomes part of the observable done condition.
|
||||
|
||||
**7. Goal line + autonomy directive.** Restating the deliverable as the final sentence exploits recency: it is the last thing in context before the session begins, and it survives context compression in long runs better than a spec buried mid-prompt. The autonomy directive ("do not ask me for anything until you are all done") is load-bearing — without it, sessions surface questions the prompt already answered.
|
||||
|
||||
## Failure-mode catalog
|
||||
|
||||
| Failure | Symptom in the fresh session | Root cause in the prompt |
|
||||
|---------|------------------------------|--------------------------|
|
||||
| Micromanaged how | Mechanical, joyless output; session ignores better paths it noticed | Step-by-step instructions instead of outcome + freedom |
|
||||
| Phantom resources | Session burns early turns hunting a path/MCP that doesn't exist, sometimes hallucinating around it | Named resource never verified against the live environment |
|
||||
| Unverifiable done | Session ships first plausible draft; quality lottery | No observable completion condition, no defined pass |
|
||||
| Buried goal line | Session over-weights a mid-prompt detail, under-delivers the actual goal | Closing sentence isn't the deliverable restated; content after the goal line |
|
||||
| Invented stakes | Short-term quality bump, long-term calibration rot | Fake audience numbers or false urgency |
|
||||
| Interview spiral | User abandons the skill; ramble-to-prompt loses its speed advantage | Multi-round questioning instead of one batch + assumptions |
|
||||
| Scope creep grant | Session builds adjacent things the user never asked for | Freedom clause written without a concrete deliverable to bound it |
|
||||
|
||||
The last row is the counterweight to part 4: creative freedom is safe only when part 1 nailed the *what*. Freedom without a concrete deliverable is a scope-creep license.
|
||||
|
||||
## Verification defaults by medium
|
||||
|
||||
| Medium | The pass is… |
|
||||
|--------|--------------|
|
||||
| Web page / app | Load the live page, click every element, check mobile width |
|
||||
| Script / CLI / automation | Run it on real input end to end; feed it a bad input too |
|
||||
| Video / animation | Render and watch the whole thing at full speed |
|
||||
| Written content | Read it aloud pass; check every claim and link |
|
||||
| Data pipeline / analysis | Run on a real sample; sanity-check the numbers against a known figure |
|
||||
| Design / image assets | View at actual size and in context, not just the editor |
|
||||
|
||||
For anything else, derive the pass from one question: **what would the end user do with this on day one?** The pass is the session doing exactly that. A Slack bot's pass is messaging it. A spreadsheet template's pass is filling it in with real-ish data. An API's pass is calling every endpoint including with malformed input. If you cannot phrase the pass as an action on the finished artifact, the deliverable is not concrete enough yet — go back to part 1.
|
||||
|
||||
## Worked example: a non-web medium
|
||||
|
||||
**Ramble:** "ugh I keep manually renaming and sorting all the podcast episode files my editor sends me, can you write me a prompt so fable just builds me something for that"
|
||||
|
||||
**Output:**
|
||||
|
||||
```
|
||||
I want you to build me a small command-line tool that takes a folder of podcast episode files from my editor and renames and sorts them into my archive automatically, because I currently lose time every week doing this by hand. The tool should figure out the episode number and title from the messy filenames my editor uses, rename each file to a clean consistent pattern, and move it into a per-season folder structure, with a dry-run mode that shows me what it would do before touching anything. You have full freedom on language, structure, and approach — pick whatever makes this most robust and easiest for me to run, and take stock of the tools you actually have before you start, fetching any libraries you need along the way. Before you call it done, do at least three iteration passes: run it end to end on a realistic set of sample files you generate yourself, including at least a few badly named ones and one duplicate, and make sure the dry-run output is honest about what the real run will do. When it works, put the tool and a short README with the exact command I run in a folder called podcast-sorter in my home directory, and show me the dry-run output from your final test. A working, tested podcast file renaming and sorting CLI with dry-run mode, in ~/podcast-sorter with a README, is your /goal. Work completely autonomously and do not ask me for anything until you are all done.
|
||||
```
|
||||
|
||||
**Assumptions:**
|
||||
- Local CLI in your home directory since you didn't name a destination
|
||||
- Dry-run mode included because the tool moves your real files
|
||||
- The session generates its own realistic test fixtures since you didn't provide sample filenames
|
||||
|
||||
Note how the quality bar inverted for functional work (concrete behavior — dry-run mode, duplicate handling — instead of adjectives) and the verification pass matched the medium (run end to end on generated fixtures, including bad input).
|
||||
|
||||
## Sources
|
||||
|
||||
- Duncan / buildroom, [`freeskills/fable-goal`](https://github.com/duncan-buildroom/freeskills) — the upstream skill and the "25 websites" reference example that defines the genre
|
||||
- Anthropic, [Claude Code best practices](https://www.anthropic.com/engineering/claude-code-best-practices) — verification loops and target-based iteration for agentic runs
|
||||
- Anthropic, [Effective harnesses for long-running agents](https://www.anthropic.com/engineering/effective-harnesses-for-long-running-agents) — observable completion conditions; why sessions need self-checkable done states
|
||||
- Anthropic, [Prompt engineering overview](https://docs.claude.com/en/docs/build-with-claude/prompt-engineering/overview) — clarity of desired outcome over prescriptive process
|
||||
- This repo's `engineering/agent-harness` references — verifier's law: work you can verify mechanically gets done reliably; work you can't, doesn't
|
||||
|
|
@ -0,0 +1,169 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Self-check runner for generated /goal prompts (fable-goal).
|
||||
|
||||
Mechanically verifies the checkable subset of the six-point pre-delivery
|
||||
self-check from SKILL.md step 5 against a drafted /goal prompt:
|
||||
|
||||
1. Word count inside the 150-350 target band
|
||||
2. Goal line present ("... is your /goal")
|
||||
3. Autonomy directive present ("work completely autonomously ...")
|
||||
4. Verification-loop language present (iteration passes / medium-matched pass)
|
||||
5. Creative-freedom grant present
|
||||
6. Delivery destination named (link / path / deploy / serve / save)
|
||||
|
||||
Points that need judgment (deliverable concreteness, whether named resources
|
||||
were verified against the live environment) stay with the author — this tool
|
||||
checks form, not taste.
|
||||
|
||||
Deterministic logic. No LLM calls. Stdlib only.
|
||||
|
||||
Usage:
|
||||
python goal_prompt_self_check.py prompt.txt
|
||||
cat prompt.txt | python goal_prompt_self_check.py -
|
||||
python goal_prompt_self_check.py --sample
|
||||
python goal_prompt_self_check.py prompt.txt --output json
|
||||
|
||||
Exit codes: 0 all checks pass, 1 one or more checks fail, 2 usage error.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
WORD_MIN, WORD_MAX = 150, 350
|
||||
|
||||
SAMPLE = (
|
||||
"I want you to build 5 landing pages for my free prompt pack, each one "
|
||||
"fundamentally different from the others, as a way to show me the strongest "
|
||||
"possible range of directions before I pick one. These will be seen by my "
|
||||
"audience, so the bar is high: exceptional typography, striking layouts, and "
|
||||
"motion that feels designed rather than templated. Each page needs a headline, "
|
||||
"proof, and a single email-capture CTA for the prompt pack. You have total "
|
||||
"creative freedom on the visual direction of each one. You can generate any "
|
||||
"imagery you need with whatever image tools you have available, and you can "
|
||||
"accomplish this in many ways using many workflows, so before you start, take "
|
||||
"stock of the tools and MCPs you actually have, go find or fetch any references, "
|
||||
"libraries, or assets you need along the way, and show me what you are capable "
|
||||
"of. Before you ok each page, do at least three iteration passes: load the live "
|
||||
"page, click through every element, check it at mobile width, and go back "
|
||||
"through with a fine-toothed comb looking for design problems and opportunities "
|
||||
"to improve. Parallelize across subagents so the pages develop independently. "
|
||||
"When all 5 are done, deploy them to Netlify and serve me the 5 links with a "
|
||||
"one-line description of each direction. 5 fundamentally different prompt pack "
|
||||
"landing pages, live on Netlify with three iteration passes each, is your "
|
||||
"/goal. Work completely autonomously and do not ask me for anything until you "
|
||||
"are all done."
|
||||
)
|
||||
|
||||
CHECKS = [
|
||||
(
|
||||
"goal_line",
|
||||
"Goal line present ('... is your /goal')",
|
||||
re.compile(r"is your /goal", re.IGNORECASE),
|
||||
),
|
||||
(
|
||||
"autonomy_directive",
|
||||
"Autonomy directive present ('work ... autonomously' / 'do not ask me')",
|
||||
re.compile(r"(work\s+(completely\s+)?autonomously|do not ask me)", re.IGNORECASE),
|
||||
),
|
||||
(
|
||||
"verification_loop",
|
||||
"Verification loop present (iteration passes / run / load / render / watch / click)",
|
||||
re.compile(
|
||||
r"(iteration pass|before you (ok|call it done|finish)|"
|
||||
r"run it (end to end|on real)|load the (live )?page|render and watch|"
|
||||
r"click through)",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
),
|
||||
(
|
||||
"creative_freedom",
|
||||
"Creative-freedom grant present ('creative freedom' / 'many ways' / 'full freedom')",
|
||||
re.compile(
|
||||
r"(creative freedom|full freedom|many ways|show me what you (are capable|can do))",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
),
|
||||
(
|
||||
"destination",
|
||||
"Delivery destination named (deploy / serve / link / folder / path / save / publish)",
|
||||
re.compile(
|
||||
r"(deploy|serve me|the \d*\s*links?\b|a folder called|in my home directory|"
|
||||
r"save (it|them|the)|publish|file path|~/[\w./-]+)",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def run_checks(text: str) -> dict:
|
||||
words = len(re.findall(r"\S+", text))
|
||||
results = [
|
||||
{
|
||||
"check": "word_count",
|
||||
"label": f"Word count in {WORD_MIN}-{WORD_MAX} band",
|
||||
"passed": WORD_MIN <= words <= WORD_MAX,
|
||||
"detail": f"{words} words",
|
||||
}
|
||||
]
|
||||
for key, label, pattern in CHECKS:
|
||||
match = pattern.search(text)
|
||||
results.append(
|
||||
{
|
||||
"check": key,
|
||||
"label": label,
|
||||
"passed": match is not None,
|
||||
"detail": f"matched: '{match.group(0)}'" if match else "no match",
|
||||
}
|
||||
)
|
||||
passed = sum(1 for r in results if r["passed"])
|
||||
return {
|
||||
"results": results,
|
||||
"passed": passed,
|
||||
"total": len(results),
|
||||
"verdict": "PASS" if passed == len(results) else "FAIL",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Mechanically verify a drafted /goal prompt against the "
|
||||
"checkable subset of the fable-goal pre-delivery self-check."
|
||||
)
|
||||
parser.add_argument("path", nargs="?", help="prompt file to check, or '-' for stdin")
|
||||
parser.add_argument("--sample", action="store_true", help="run against the embedded example prompt")
|
||||
parser.add_argument("--output", choices=["text", "json"], default="text")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.sample and args.path:
|
||||
parser.error("pass either a prompt file or --sample, not both")
|
||||
if args.sample:
|
||||
text = SAMPLE
|
||||
elif args.path == "-":
|
||||
text = sys.stdin.read()
|
||||
elif args.path:
|
||||
try:
|
||||
with open(args.path, encoding="utf-8") as f:
|
||||
text = f.read()
|
||||
except OSError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
else:
|
||||
parser.print_help()
|
||||
return 2
|
||||
|
||||
report = run_checks(text)
|
||||
if args.output == "json":
|
||||
print(json.dumps(report, indent=2))
|
||||
else:
|
||||
for r in report["results"]:
|
||||
mark = "x" if r["passed"] else " "
|
||||
print(f" [{mark}] {r['label']} ({r['detail']})")
|
||||
print(f"\nVerdict: {report['verdict']} ({report['passed']}/{report['total']})")
|
||||
return 0 if report["verdict"] == "PASS" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -18,3 +18,44 @@ engineering/autoresearch-agent/evaluators/test_pass_rate.py # fixed-contract au
|
|||
engineering/security-guidance/hooks/security_reminder_hook.py # PreToolUse stdin hook
|
||||
productivity/handoff/hooks/session_start.py # SessionStart stdin hook
|
||||
productivity/handoff/hooks/session_end.py # SessionEnd stdin hook
|
||||
|
||||
# skillopt_sleep engine (engineering/skillopt-sleep, vendored from
|
||||
# microsoft/SkillOpt): internal package modules, not standalone CLI scripts.
|
||||
# The real entry point is `python -m skillopt_sleep` (verified: exits 0 on
|
||||
# --help). Running any single file directly breaks two ways: (1)
|
||||
# skillopt_sleep/types.py shadows the stdlib `types` module once the file's
|
||||
# own directory is prepended to sys.path, which cascades into
|
||||
# "cannot import name 'GenericAlias'/'asdict' from partially initialized
|
||||
# module" for every sibling that (transitively) imports dataclasses/typing;
|
||||
# (2) experiments/*.py use absolute `from skillopt_sleep.x import y` imports
|
||||
# that require the package's parent dir on sys.path, which standalone
|
||||
# execution doesn't provide.
|
||||
engineering/skillopt-sleep/skillopt_sleep/__main__.py # python -m skillopt_sleep entry point only
|
||||
engineering/skillopt-sleep/skillopt_sleep/backend.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/budget.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/config.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/consolidate.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/cycle.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/dream.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/experiments/gbrain_bench.py # requires skillopt_sleep on sys.path
|
||||
engineering/skillopt-sleep/skillopt_sleep/experiments/personas.py # requires skillopt_sleep on sys.path
|
||||
engineering/skillopt-sleep/skillopt_sleep/experiments/run_experiment.py # requires skillopt_sleep on sys.path
|
||||
engineering/skillopt-sleep/skillopt_sleep/experiments/run_gbrain.py # requires skillopt_sleep on sys.path
|
||||
engineering/skillopt-sleep/skillopt_sleep/experiments/run_transfer.py # requires skillopt_sleep on sys.path
|
||||
engineering/skillopt-sleep/skillopt_sleep/experiments/sweep.py # requires skillopt_sleep on sys.path
|
||||
engineering/skillopt-sleep/skillopt_sleep/gate.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/harvest.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/harvest_codex.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/harvest_sources.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/judges.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/llm_miner.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/memory.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/mine.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/replay.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/rollout.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/scheduler.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/slow_update.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/staging.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/state.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/tasks_file.py # internal module, types.py shadow
|
||||
engineering/skillopt-sleep/skillopt_sleep/types.py # internal module; module itself IS the stdlib-name collision
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue