From 74e1251ddcd212205788f21ccd6b5ebd82ade7c3 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Sat, 17 Oct 2026 22:55:13 +0800 Subject: [PATCH] docs: consolidate bilingual compaction benchmark guides --- benchmark/compaction/README.md | 336 +++++++++++++++++++++++--- benchmark/compaction/README_ZH.md | 311 ++++++++++++++++++++++-- benchmark/compaction/RESULTS.md | 97 -------- benchmark/compaction/RESULTS_ZH.md | 94 ------- benchmark/compaction/SETTINGS.md | 158 ------------ benchmark/compaction/SETTINGS_ZH.md | 149 ------------ benchmark/compaction/STUDY.md | 114 --------- benchmark/compaction/STUDY_ZH.md | 110 --------- benchmark/compaction/suite_collect.py | 2 +- benchmark/compaction/suite_fusion.py | 2 +- benchmark/compaction/suite_native.py | 4 +- benchmark/compaction/suite_run.py | 2 +- 12 files changed, 599 insertions(+), 780 deletions(-) delete mode 100644 benchmark/compaction/RESULTS.md delete mode 100644 benchmark/compaction/RESULTS_ZH.md delete mode 100644 benchmark/compaction/SETTINGS.md delete mode 100644 benchmark/compaction/SETTINGS_ZH.md delete mode 100644 benchmark/compaction/STUDY.md delete mode 100644 benchmark/compaction/STUDY_ZH.md diff --git a/benchmark/compaction/README.md b/benchmark/compaction/README.md index fa299346..e17720bc 100644 --- a/benchmark/compaction/README.md +++ b/benchmark/compaction/README.md @@ -1,23 +1,234 @@ -# Context compaction: fixed summary + selected evidence +# Context compression: Jev across two summary strategies **English** | [简体中文](./README_ZH.md) -A **14-query fixture panel** with predefined answers and rule scoring: four primary tasks and ten short/query-only -controls. See [queries, answers and frozen settings](./STUDY.md) and [measured results](./RESULTS.md). +Summaries can omit identifiers, configuration values or log details needed for later answers. This experiment asks whether +appending a few Jev-selected original evidence blocks improves information retention across two summary strategies, +while keeping mean input shorter than the full history. -Hermes collects the source through real `read_file` calls. ReMe's experiment then compares Full context, a frozen Qwen -summary S and **Hermes compaction summary H (small-budget setting)**, each with Jev Top 0–5. Scored answering calls Qwen directly; -there is no resumed Hermes tool loop or archive recovery. Production `compact_context` is separate. +This is a **synthetic-task comparison from 2026-10-16**. Hermes collects trajectories; evaluation calls Qwen directly. +There is no resumed agent/tool loop, archive recovery or task-completion measurement. Production `compact_context` is separate. -Models, framework/runtime versions, request parameters, artifact hashes and replay requirements: -[Experiment settings and reproducibility](./SETTINGS.md). +## Experiment design -## Run +### Data and cohorts -Use ReMe's development environment plus optional `tokenizers`. Collection and native generation need a separately -installed Hermes environment, also able to import `tokenizers`. Credentials come from ReMe's `.env`: -`LLM_API_KEY`, `LLM_BASE_URL`, `DECISION_MODEL_API_KEY`, and optional `DECISION_MODEL_NAME` / `DECISION_MODEL_BASE_URL`. -All commands require `--live`; outputs must be new directories outside this repository. +There are **14 predefined query/answer templates**: one reused original payment trajectory and thirteen newly collected +read-only Hermes trajectories over synthetic upstream fixtures. For collection, queries asking for file changes or cron jobs +are adapted to evidence investigation and proposed actions. Source workspaces are immutable; only allowlisted `read_file` +is exposed. The appendix lists all queries and expected answers. + +Fixtures come from MIT-licensed Hermes code, not private maintainer production trajectories. Long logs and build outputs +use deterministic original head/matching-line views, not every byte of the upstream workspace; no noise or renamed identifiers +are added. Cohorts are fixed before collection: evidence-dependent tasks with at least 2,000 source-snapshot characters +enter the primary cohort, with the original payment source pinned separately. + +| Cohort | Tasks | +| --- | --- | +| Primary (4) | `payment`, `checkout_504`, `server_error`, `build_token` | +| Short/query-only controls (10) | `backup`, `refactor`, `three_files`, `active_config`, `dependency_version`, `retry_backoff`, `meeting_notes`, `targeted_edit`, `package_versions`, `checksum_anomaly` | + +### Methods + +There are **13 methods**: Full context, plus S and H each with Jev Top 0–5. Top 0 means summary only. + +| Method | Input construction | +| --- | --- | +| Full context | Original system/task and public history before the final collection report, including tool calls/results | +| Generic summary S | Qwen handoff summary of all evidence units and the original task, using the prompt in [`cases.py`](./cases.py) | +| Hermes compaction summary H (small-budget setting) | The same Qwen model through Hermes' native summary-generation pipeline, summarizing evidence history with the original task as focus | +| S/H + Jev Top N | The entire corresponding summary body, followed by the top N whole original evidence units | + +Each S/H stays byte-identical across its own Top 0–5 conditions. Both mechanisms share exactly the same Jev ranking and +selected originals for each task/generation/N. S receives structured evidence units; H receives public history including +tool calls/results, so their differences extend beyond the prompt. Compressed arms use the same +`Recorded evidence:` / `Summary:` / `Original evidence:` framing, preserve the original system/task and append the same exam. +Internal metadata, thinking and final collection reports are excluded. Exams and gold never enter collection, summary or decision requests. + +H calls `agent.context_compressor.ContextCompressor._generate_summary` with `tail_mode=legacy`, reusing native +serialization, structured checkpoint templates, redaction and provenance validation. The experiment overrides the summary +budget and adds a total word limit, retaining Historical Task Snapshot/no-user sentinel and Critical Context while allowing +other sections to be omitted. Hermes' helper removes the control prefix without rewriting facts; lean-mode extra session +logs and appendices are not used. H is **Hermes compaction summary under a small-budget setting**: it invokes generation +directly rather than full `compress()`. Automatic triggering, head/tail selection and post-compression conversation repair +are outside this comparison. H-only inputs must actually shorten Full; original-history fallback is not accepted. + +### How Jev selects originals + +Evidence units are **chunks of tool-result text**, with an ID, file path extracted from tool arguments, source positions +and body text. Long results are split into related events or text segments. Top N does not mean N complete tool calls: +units do not carry the full tool name and all call arguments. See `event_units` in [`cases.py`](./cases.py) and +`decision_input` in [`run.py`](./run.py). + +Jev receives the original task and `evidence_units` (`id`, `file`, `text`, **including tool-result bodies**). It answers a +binary probability question for each unit: does it contain material information needed to continue the task correctly? +Ranking is not conditioned on the summary or exam. Requests contain at most ten units, an experiment batch size rather +than a fixed service limit. The three ranking generations saved 54 Jev requests: thirteen tasks need one request each per +generation, while the 41-unit `server_error` needs five (10/10/10/10/1). Completed rankings are reused across both summaries +and all N values; Top N does not determine request count. + +Select a strict descending-probability prefix, break ties by ID, then restore source order. There is no threshold, skipping +or partial-unit truncation. The actual count is `min(N, available units)`, saved as `actual_n`. The original payment source +has ten units (maximum Top 10); this study sweeps Top 1–5 throughout. + +### Budgets, repetitions and scoring + +- Three independently requested summary/ranking generations per task, two answers per context: + **14 × 3 × 13 = 546 contexts, 1,092 answers**. Each method has 24 primary and 60 control answers; repetitions are not independent tasks. +- Summary-body allowance: `max(64, min(256, int(0.2 * evidence_tokens)))`. S allows up to three total length-check attempts, + with word target `max(12, int(allowance * 0.4 * 0.8**attempt))`. +- H also allows three total length-check attempts. Requested budget: + `max(12, int(allowance * 0.8**attempt))`; word limit: `max(18, int(requested_budget * 0.35))`. + Length adjustment never uses answer feedback. +- Before answering, compute the complete-input ceiling from protected prompts, the five largest whole units, summary + allowance and 96 reserve/framing tokens; enlarge it first if serialized S needs more space. This is an incremental + comparison, **not equal-length or a universal 1,000/2,000-token ceiling**, and not every Full exceeds its ceiling. + See [`suite_run.py`](./suite_run.py). + +| Case | Evidence units | Summary allowance | Complete-input ceiling | +| --- | ---: | ---: | ---: | +| payment | 10 | 256 | 4,496 | +| checkout_504 | 5 | 256 | 5,246 | +| server_error | 41 | 256 | 3,185 | +| build_token | 8 | 256 | 3,859 | + +Answers must return `{"facts": {...}}`. Predefined fields require recursive exact JSON types/values: integers differ from +booleans and floats; strings and ordered arrays must match exactly. Only payment `no_retry_http_statuses` ignores array order. +Missing/malformed fields and failed planned conditions score zero and remain in the denominator. No answer aliases, +rewrites or retries are added after scoring; **no LLM judge is used**. Average fields within answers, answers within +generations, generations within tasks, then tasks equally. Primary/control cohorts are separate. All-fields-correct rates +are also saved. See [`suite.py`](./suite.py). + +Charted answer input uses actual Qwen API `prompt_tokens`, including system/task, history and exam, excluding summary +and Jev requests. Budget checks use a local tokenizer proxy; these are distinct measures. + +## Results and conclusions + +### Four primary tasks + +![Answer quality and downstream input across two summary strategies](../../docs/figure/compaction-topn-results.svg) + +| Method | Fact score | Qwen answer input tokens | +| --- | ---: | ---: | +| Full context | 100.0% | 4,014 | +| S | 82.0% | 2,480 | +| S + Jev Top 1 | 83.0% | 2,753 | +| S + Jev Top 2 | 94.5% | 2,974 | +| S + Jev Top 3 | 99.5% | 3,241 | +| S + Jev Top 4 | 98.1% | 3,496 | +| S + Jev Top 5 | 97.4% | 3,660 | +| H (Hermes, small budget) | 79.4% | 2,432 | +| H + Jev Top 1 | 85.3% | 2,705 | +| H + Jev Top 2 | 92.9% | 2,927 | +| H + Jev Top 3 | 97.9% | 3,193 | +| H + Jev Top 4 | 96.9% | 3,448 | +| H + Jev Top 5 | 96.9% | 3,613 | + +On these four tasks, Top 3 raises S from **82.0% to 99.5%** and H from **79.4% to 97.9%**, reducing mean answer input by +**19.3% / 20.4%** relative to Full. A few originals can therefore supplement exact evidence lost by either summary. +Top 4–5 do not improve the point estimates; retaining more original text does not guarantee greater accuracy. + +Every condition fits its input ceiling; all 42 H-only contexts shorten Full. However, at Top 3–5 only **9/12 primary contexts +per method** are shorter than their paired Full. Across H fusion arms, 172/210 contexts shorten Full (51/60 primary, +121/150 controls); the rest remain scored. **Mean savings do not imply per-case savings.** + +This is an information-retention diagnostic on a small synthetic panel. Payment was used during earlier development, +not held out independently. There is no equal-length longer-summary or alternative-selector control, so the gains cannot +all be attributed to Jev ranking. The results establish neither a generally optimal N, statistical equivalence to Full, +nor improved agent task-completion rates. + +
+Ten short/query-only controls + +| Method | Fact score | Qwen answer input tokens | +| --- | ---: | ---: | +| Full context | 100.0% | 2,727 | +| S | 67.1% | 2,315 | +| S + Jev Top 1 | 84.7% | 2,493 | +| S + Jev Top 2 | 87.5% | 2,573 | +| S + Jev Top 3 | 92.9% | 2,687 | +| S + Jev Top 4 | 92.9% | 2,722 | +| S + Jev Top 5 | 92.5% | 2,722 | +| H (Hermes, small budget) | 73.2% | 2,308 | +| H + Jev Top 1 | 80.1% | 2,486 | +| H + Jev Top 2 | 92.5% | 2,566 | +| H + Jev Top 3 | 99.2% | 2,680 | +| H + Jev Top 4 | 99.2% | 2,715 | +| H + Jev Top 5 | 98.8% | 2,715 | + +These are not ten additional long-context tasks. Top N saturates when fewer source units are available. + +
+ +## Reproduction + +### Models and runtime + +| Component | Version/model | +| --- | --- | +| Hermes Agent `AIAgent` | commit `30463589eb54fa0f48724594cc0e98e6b995729b` | +| Collection, S/H generation, answering model | `qwen3.8-max` | +| Jev (through AgentScope/TypeSafe) | Resolved response version `jev-1.13.0` | +| Experiment implementation snapshot | ReMe commit `50c2c806446c7ae6acc66217afd6e5edc2114046` | + +Qwen requests/responses identify only a model alias, not an immutable provider weight revision. Service updates and +nondeterminism may change results even at temperature zero. The run used the public China (Beijing) DashScope +OpenAI-compatible Chat Completions endpoint; use the same `LLM_BASE_URL` service for the closest rerun. + +
+Dependency versions and request parameters + +Two separate Python environments were used. These versions were **verified from the retained environments after the +run**, not from a pre-run lockfile; they are a reproduction reference rather than proof of a fully locked installation. +The local run used Python **3.13.3**, macOS arm64. No local GPU performs inference. + +| Package | ReMe evaluation environment | Hermes collection/summary environment | +| --- | --- | --- | +| `openai` | 2.53.0 | 3.31.0 | +| `agentscope` | 2.0.10 | — | +| `typesafe-sdk` | 0.7.5 | — | +| `httpx` | 0.28.1 | 0.28.1 | +| `httpx2` | — | 2.13.1 | +| `python-dotenv` | 1.2.2 | 1.2.4 | +| `pydantic` | 2.13.4 | 2.14.1 | +| `tokenizers` | 0.23.3 | 0.23.3 | + +`tokenizers` was imported from a separate package directory on `PYTHONPATH`; it was not installed in either venv. +A new setup may install that exact version into each venv. The local artifact bundle includes both environments' +distribution metadata, recorded after the experiment. It does not contain credentials or virtualenv binaries. + +Token accounting uses `Qwen/Qwen3.5-9B`'s **`tokenizer.json`**, revision +`c202236235762e1c871ad0ccb60c8ee5ba337b9a`, with SHA-256 +`5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42`. +This is a **budget proxy**, not a claim that Qwen 3.8 uses that tokenizer. Reported downstream inputs use actual API usage. + +#### Request parameters + +| Stage | Sampling/output parameters | Execution limits | +| --- | --- | --- | +| Hermes collection | `max_tokens=1024`; `reasoning_effort=none`, `enable_thinking=false`; no explicit temperature or seed | 12 agent iterations, 240-second run budget; four workers; 360-second subprocess timeout | +| S generation | `temperature=0`, `max_tokens=4096`; thinking disabled; no seed | OpenAI client: 90-second timeout, `max_retries=1` | +| H generation | `temperature=0`; no explicit API `max_tokens`; thinking disabled; no seed | Same 90-second client timeout / one SDK retry; four workers; 600-second subprocess timeout | +| Jev ranking | Binary probability question per evidence unit; batch size 10 | Adapter timeout 45 seconds; `max_retries=2`, retry delay 0.5 seconds | +| Answering | `temperature=0`, `max_tokens=2400`, `response_format={"type":"json_object"}`; thinking disabled; no seed | 90-second client timeout, `max_retries=1`; four concurrent conditions, two sequential answers each | + +For direct Qwen calls, thinking is disabled with `reasoning_effort="none"` and +`extra_body={"enable_thinking":false}`. The collection wire request carries `enable_thinking=false` at top level. +H's API output limit is intentionally not its summary-body allowance: acceptance uses the local body-token check. +Captured request bodies are authoritative for the recorded run. + +Collection exposes only allowlisted `read_file`; source workspaces are read-only. Automatic compression and micro +compaction are disabled during collection. Memory/context files and background review are skipped. Each worker has an +isolated `HERMES_HOME`, workspace and session DB. See [`hermes_worker.py`](./hermes_worker.py). + +
+ +### Fresh collection and evaluation + +Use ReMe's development environment and a separate Hermes Python environment for collection/H generation; both must +import `tokenizers`. Set `LLM_API_KEY`, `LLM_BASE_URL`, `DECISION_MODEL_API_KEY`, optionally `DECISION_MODEL_BASE_URL`, +in `.env`; request `DECISION_MODEL_NAME=jev-1.13.0` when the service supports it. Every command requires explicit `--live` +and new output directories outside Git; existing runs are never overwritten. ```bash # Freeze all task/answer definitions, then collect 13 new read-only trajectories. @@ -55,21 +266,90 @@ python -m benchmark.compaction.suite_fusion --live \ --output /path/outside/repository/dual-summary-results ``` -The task definitions use local Hermes commit `30463589eb54fa0f48724594cc0e98e6b995729b` (MIT). The recorded study uses -`qwen3.8-max`, three summary/ranking generations and two answers per context: **546 contexts, 1,092 answers across thirteen methods**. -Provider requests are bounded to four at a time. Raw files, sessions, wire bodies, code snapshots and per-field grades stay -outside git; existing runs are never overwritten. +This reproduces the protocol; new agent calls may produce different trajectories. `suite_fusion` re-answers only +H + Top 1–5, retaining the other eight arms, rather than re-answering all thirteen. To reuse the recorded H generations, +also pass `--native-evaluation /path/to/pre-amendment/evaluation`. The runner checks source hashes and permits only the +held-out `exam` to differ; the summary generator never receives that field. -S and H each remain byte-identical across their five fusion variants; both use the same Jev selections. -Top N means a strict descending-score prefix, without a probability threshold, skipping or truncation; selected units are rendered in source order. When a task has fewer -than N units, all available units are retained and `actual_n` records the count. The original payment source has ten -units (maximum Top 10); this study sweeps Top 1–5 with task-specific unit counts. +### Frozen offline verification -H reuses Hermes' context-compaction summary-generation pipeline with a small-budget setting: legacy template, -S-matched body budget, final word constraint, native provenance validation, and body-only downstream framing. The experiment invokes summary generation directly, bypassing -automatic triggering and head/tail selection, and **must actually shorten input**. -See [the exact simulation settings and preflight disclosures](./STUDY.md#hermes-compaction-summary-small-budget-setting). +Public Git contains the experiment code, fixture definitions, rules and curated results. The original payment trajectory, +frozen corpus and wire records are **not committed**. Exact replay requires the local artifact bundle; it is not currently +a public one-command reproduction. Do not substitute another payment trajectory: its expected SHA-256 is +`0f37e6510ffafd920d088452d8719026b00fd4e74f87f7dd1a42a85c8287e6f1`. -For the recorded format amendment, native summaries used the pre-amendment source files: pass -`--native-evaluation /path/to/pre-amendment/evaluation` when extending that run. The extension verifies its source hash -and permits only the held-out `exam` field to differ; the summary generator never receives that field. +The prepared bundle is outside Git: +`/Users/yuli/workspace/trajectory-runs/compaction-dual-summary-reproduction-20261016.tar.gz`. +Its SHA-256 is `40ee93c9117dfe59cf0b4dc8887ba4b2c62fa16f3bc97f2dac6d536343f942a3`. +It contains 3,187 files, a relative-path SHA-256 manifest, environment metadata and `verify.py`. +With ReMe's development environment installed, from the ReMe checkout: + +```bash +# Extract outside the repository. Substitute the directory holding the archive. +export REPRO_DIR=/path/outside/repository +shasum -a 256 "$REPRO_DIR/compaction-dual-summary-reproduction-20261016.tar.gz" +tar -xzf "$REPRO_DIR/compaction-dual-summary-reproduction-20261016.tar.gz" -C "$REPRO_DIR" +PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}" python \ + "$REPRO_DIR/compaction-dual-summary-reproduction-20261016/verify.py" +``` + +Expected result: **3,187 artifact hashes, 546 contexts and 1,092 grades verified; zero API calls**. +The bundle's `corpus/`, `evaluation/`, `native/`, `native-source/`, `combined/` and `final/` directories correspond to +the run stages above. Pass `--native-evaluation /native-source` when reusing its H generations. + +## Appendix + + + +
+Queries and expected answers + +Each initial query supplies the collection goal; its follow-up asks for the JSON fields below. One task and its follow-up +are one sample. These are the original English queries; the Chinese version paraphrases them. Actual read-only collection +adaptations and field descriptions are defined in [`suite.py`](./suite.py) and frozen requests. Gold keys, types and strings +are preserved exactly. + +| ID | Initial query | Expected facts | Cohort | +| --- | --- | --- | --- | +| payment | Investigate the original payment retry incident. | `{"incident_id": "PAY-4827", "invoice_id": "INV-731", "successful_charge_id": "CH-991", "amount_minor": 12900, "attempt_keys": ["pay-a", "pay-b"], "first_attempt_committed": true, "max_attempts": 3, "no_retry_http_statuses": [402, 422], "key_generation_line": 5, "regression_attempt_counts": {"timeout_then_success": 2, "validation_422": 1, "three_timeouts": 3}}` | Primary | +| backup | A while back I told you one of my repos needed nightly backups — find which repo that was in our past conversations (do not guess). Then: write backup.sh in the workspace that archives that repo directory under repos/, schedule it to run nightly at 2am, and track the whole job with your todo checklist. Report back which repo it was and what you set up. | `{"repo": "sparks-data", "ignored_repo": "toybox", "data_text": "important"}` | Control: short_fixture | +| refactor | In src/, every requests call uses timeout=30. Bump them all to timeout=45, add a CHANGELOG.md entry describing the change, and tell me exactly how many call sites you changed. | `{"old_timeout": 30, "call_sites": 3, "functions": ["fetch_a", "fetch_b", "fetch_c"], "methods": ["get", "post", "head"], "old_version": "1.0"}` | Control: short_fixture | +| three_files | This is a multi-step job — track it with your todo checklist tool and keep it updated as you go, marking each step completed when verified. Steps: (1) create greet.py containing a greet(name) function that returns `'hello '`; (2) create data/rows.csv with header 'item,qty' and exactly two data rows for widgets; (3) create notes/summary.md stating how many files you created. Finish only when all three are done and checked off. | `{"file_count": 3, "header": "item,qty", "data_rows": 2}` | Control: query_only | +| active_config | Search this workspace for wherever the retry_limit setting is configured and tell me its current value. | `{"retry_limit": 7, "backoff": 2, "active_file": "conf/app.ini", "old_file": "conf/old.ini"}` | Control: short_fixture | +| checkout_504 | Investigate the checkout 504 incident: distinguish the failed rollback, root cause and final database fix. | `{"setting": "statement_timeout", "value_seconds": 45, "index_mode": "CONCURRENTLY", "root_cause": "CREATE INDEX without CONCURRENTLY", "rolled_back_version": "v2.30.2", "rollback_fixed": false, "deployed_version": "v2.31.0", "initial_error_percent": 8}` | Primary | +| dependency_version | What exact left-pad version is pinned, and which file declares the test script? Record the package scripts for a handoff. | `{"version": "1.3.0", "declaration": "package.json", "test": "vitest run", "build": "tsc -p .", "package_version": "4.2.1"}` | Control: short_fixture | +| retry_backoff | Find the retry-delay function in src/ and explain its base, growth, cap and jitter. | `{"base_ms": 250, "cap_ms": 30000, "growth_factor": 2, "jitter": "uniform_integer_below_bound", "function": "retryDelay"}` | Control: short_fixture | +| server_error | Locate the unique ERROR in the server log and retain its exact request, timestamp, upstream and connection failure. | `{"request_id": "req-7f3d9", "timestamp": "2026-08-08T23:41:17Z", "status": 502, "upstream": "payments", "error": "ECONNREFUSED", "address": "10.0.4.17:8443"}` | Primary | +| meeting_notes | Read the meeting notes and report the third bullet exactly, retaining the other agreed actions. | `{"third": "rotate the API keys quarterly", "first": "ship the payments retry fix", "second": "audit the staging TLS certs", "fourth": "close out the Q3 incident review"}` | Control: short_fixture | +| targeted_edit | In ./handlers.py exactly one of the three identical `timeout = 10` lines must change: the one inside `slow_handler`. Change it to `timeout = 60` using the patch tool. Then run `python3 check_handlers.py` from . and reply with its output. | `{"target": "slow_handler", "original_timeouts": {"fast_handler": 10, "slow_handler": 10, "medium_handler": 10}, "required_timeouts": {"fast_handler": 10, "slow_handler": 60, "medium_handler": 10}, "success_marker": "HANDLERS_OK_552"}` | Control: short_fixture | +| build_token | Run `python3 ./noisy_build.py` (it prints a lot). Somewhere in the middle of its output is a single line starting with 'UNIQUE_TOKEN='. Reply with the full token value. | `{"token": "tok_9f31c_middle", "step": 2000, "total_steps": 4000, "prefix": "UNIQUE_TOKEN="}` | Primary | +| package_versions | The project . has three package dirs: pkg_a, pkg_b, pkg_c, each containing version.txt. Working through the directories, collect the three versions and create ./versions.txt containing them comma-separated in order (a,b,c). Reply DONE plus the joined string. | `{"versions": {"pkg_a": "1.4.2", "pkg_b": "0.9.7", "pkg_c": "3.2.1"}, "joined": "1.4.2,0.9.7,3.2.1"}` | Control: short_fixture | +| checksum_anomaly | The file ./records.log contains exactly one line starting with 'ANOMALY:'. Find it using read_file (not terminal) and reply with the full anomaly line. | `{"line": "ANOMALY: checksum drift detected in shard 7 (code X99Q)", "code": "X99Q", "shard": 7, "line_number": 4201}` | Control: short_fixture | + +
+ +
+Protocol amendments, audit and historical artifacts + +The initial seven-arm, 588-answer run exposed two ambiguous format instructions: payment test counts were returned as an +array, and meeting bullets included list markers. Gold/grader stayed unchanged; only the payment named JSON mapping and +marker-free meeting text were clarified. Both tasks were re-answered across all seven arms with frozen S/rankings; the +initial 84 answers remain as excluded preflight records. The other twelve tasks retained 504 answers. H-only then added +84 answers, and H fusion added 420, forming the final results; all 42 H summaries, rankings and 672 prior answers were reused. +Cohorts, evidence and gold values were not adjusted based on outcomes. + +The first forced H generation included a long control prefix; 40/42 inputs exceeded Full and remain integration preflight +records only. Body framing and word constraints were fixed before fresh scored answers, without changing S, gold, cohorts +or Jev rankings. H used pre-amendment source files; only the `exam` field, excluded from generation, differs. +The early payment `budget.json` records 4,459; the final question adds 37 tokens, making final `results.json`'s 4,496 the +scored ceiling. Frozen artifacts are preserved, not rewritten. + +The offline audit checks source hashes, frozen S/H, identical nested Top-N originals, captured inputs, every grade, +input ceilings, unchanged prior results and configured-key exclusion. All 546 conditions are valid. Raw files, sessions, +API requests/responses, code snapshots and per-field grades remain outside Git. Historical directories under +`/Users/yuli/workspace/trajectory-runs/`: +`hermes-fixture14-20261016/` (corpus), `compaction-fixture14-20261016-v2/` (S/rankings), +`compaction-fixture14-hermes-adapted-20261016/` (H), `compaction-fixture14-final-20261016/` (original eight arms), +`compaction-fixture14-dual-summary-20261016/` (final thirteen arms). + +
diff --git a/benchmark/compaction/README_ZH.md b/benchmark/compaction/README_ZH.md index 9abcc66e..d8203316 100644 --- a/benchmark/compaction/README_ZH.md +++ b/benchmark/compaction/README_ZH.md @@ -1,23 +1,217 @@ -# 上下文压缩：固定摘要与原文证据选择 +# 上下文压缩实验：两种摘要机制下的 Jev 增益 [English](./README.md) | **简体中文** -本实验使用 **14 个预设 query/answer 的合成任务模板**，按固定规则评分：4 个主任务，另有 10 个短任务或仅依赖问题的对照。 -参见[任务、标准答案和固定规则](./STUDY_ZH.md)与[实验结果](./RESULTS_ZH.md)。 +摘要可能遗漏后续回答需要的标识符、配置值或日志细节。本实验检验：在固定摘要后追加少量 Jev 选出的原文证据， +能否在两种摘要机制下提高信息保留率，并保持比完整历史更短的平均输入。 -Hermes 通过实际 `read_file` 调用采集轨迹。随后比较 Full context、固定的通用 Qwen 摘要 S，以及 -**Hermes 上下文压缩摘要 H（小预算版）**；S、H 分别追加 Jev Top 0～5 原文。评分时直接调用 Qwen， -没有恢复 Hermes 的工具循环或从归档中取回原文。生产功能 `compact_context` 与本实验独立。 +这是 **2026-10-16 的合成任务对照实验**。Hermes 负责采集轨迹，评分阶段直接调用 Qwen； +不继续 agent 的工具循环，不从归档恢复原文，也不衡量实际任务完成率。生产功能 `compact_context` 与本实验独立。 -模型、框架与运行环境版本、请求参数、产物哈希和离线复核要求见[实验设置与复现说明](./SETTINGS_ZH.md)。 +## 实验设计 -## 运行 +### 数据与分组 -使用 ReMe 开发环境并安装可选依赖 `tokenizers`。轨迹采集和 Hermes 摘要生成需要独立的 Hermes 环境， -该环境也必须能导入 `tokenizers`。凭证从 ReMe 的 `.env` 读取： -`LLM_API_KEY`、`LLM_BASE_URL`、`DECISION_MODEL_API_KEY`，以及可选的 -`DECISION_MODEL_NAME`、`DECISION_MODEL_BASE_URL`。 -所有命令均需显式传入 `--live`；输出必须使用仓库外的新目录。 +共 **14 个预设 query/answer 任务模板**：复用 1 条原始支付轨迹，另在 Hermes 的合成任务素材上实际采集 13 条只读轨迹。 +原本要求修改文件或创建定时任务的 query，采集时改写为调查证据并提出行动建议。 +来源工作区不可变，仅开放白名单内的 `read_file`。完整问题和标准答案见文末附录。 + +任务素材来自 Hermes 的 MIT 许可代码，不是维护者私有的生产轨迹。 +长日志和构建输出采用确定性的原文开头与匹配行附近视图，不包含上游工作区的全部字节，也不添加噪声或替换标识符。 +分组在采集前按结构规则固定：来源快照至少 2000 字符且事实依赖证据的任务进入主组，支付来源单独固定。 + +| 分组 | 任务 | +| --- | --- | +| 主任务（4 个） | `payment`、`checkout_504`、`server_error`、`build_token` | +| 短来源／仅依赖问题的对照（10 个） | `backup`、`refactor`、`three_files`、`active_config`、`dependency_version`、`retry_backoff`、`meeting_notes`、`targeted_edit`、`package_versions`、`checksum_anomaly` | + +### 对照方法 + +共 **13 组**：Full context，以及 S、H 各自搭配 Jev Top 0～5。Top 0 表示纯摘要。 + +| 方法 | 输入构造 | +| --- | --- | +| Full context | 原始 system、任务及采集最终报告之前的公开历史，包含工具调用与结果 | +| 通用摘要 S | Qwen 对全部证据块和原任务生成交接摘要，使用 [`cases.py`](./cases.py) 的提示词 | +| Hermes 上下文压缩摘要 H（小预算版） | 同一 Qwen 模型通过 Hermes 原生摘要生成流程，对证据历史生成摘要，以原任务为关注主题 | +| S／H + Jev Top N | 保留对应摘要的全部正文，再追加排名前 N 的完整原文证据块 | + +S、H 各自在 Top 0～5 中逐字不变；同一任务／轮次／N 在两种机制下使用完全相同的 Jev 排名与原文块。 +S 输入的是整理后的证据块，H 输入的是包含工具调用与结果的公开历史，因此两者不只是提示词不同。 +压缩组使用统一的 `Recorded evidence:`／`Summary:`／`Original evidence:` 封装，保留原始 system 和任务，最后追加同一评测问题。 +内部元数据、thinking 和采集结束时的最终报告不进入对照；评测问题与标准答案不进入采集、摘要或决策请求。 + +H 实际调用 `agent.context_compressor.ContextCompressor._generate_summary`，采用 `tail_mode=legacy`， +复用原生序列化、结构化检查点模板、敏感信息遮蔽和来源校验。实验覆盖摘要预算并附加总字数约束， +保留 Historical Task Snapshot／无用户消息标记和 Critical Context，允许省略其他章节； +用 Hermes 辅助函数移除控制前缀，不改写摘要事实，也不使用 lean 模式的额外会话日志与附录。 +它是 **Hermes 上下文压缩摘要的小预算设置**：直接调用生成器，不调用完整的 `compress()`， +自动触发、首尾窗口选择和压缩后的会话修复均不在本次比较范围内。纯 H 必须实际缩短输入，不接受原文回退。 + +### Jev 如何选择原文 + +证据单元是**工具结果正文的块**，记录块 ID、工具参数中提取的文件路径、原始位置和正文； +长结果按相关事件或文本片段分块。Top N 不是 N 次完整工具调用，单元不携带完整工具名和全部调用参数。 +实现见 [`cases.py`](./cases.py) 的 `event_units` 和 [`run.py`](./run.py) 的 `decision_input`。 + +Jev 接收原任务与 `evidence_units`（每块的 `id`、`file`、`text`，**包含工具结果正文**）， +对每块回答一个“是否包含继续原任务所需的重要信息”的二元概率问题。排名不以摘要或评测问题为条件。 +每次请求最多放 10 块，这是实验代码的批大小，并非服务固定限制。 +本次三轮排名共保存 54 次 Jev 请求：13 个任务每轮各 1 次，41 块的 `server_error` 每轮 5 次（10/10/10/10/1）。 +完整排名生成后在两种摘要和全部 N 中复用，Top N 不决定请求次数。 + +按概率降序严格取前 N 块，同分按 ID 排序，再恢复来源顺序。不设阈值、不跳过候选、不截断块。 +实际数量为 `min(N, available units)`，记录为 `actual_n`。原始支付轨迹有 10 块，最多 Top 10；本次统一测试 Top 1～5。 + +### 预算、重复与评分 + +- 每个任务独立请求 3 轮摘要／排名，每个上下文回答 2 次，共 **14 × 3 × 13 = 546 个上下文、1092 次回答**。 + 主任务每种方法有 24 次回答，对照有 60 次；重复回答不等于独立任务。 +- 摘要正文上限为 `max(64, min(256, int(0.2 * evidence_tokens)))`。S 最多 3 次长度检查尝试， + 字数目标为 `max(12, int(allowance * 0.4 * 0.8**attempt))`。 +- H 同样最多 3 次长度检查尝试，请求预算为 `max(12, int(allowance * 0.8**attempt))`， + 字数上限为 `max(18, int(requested_budget * 0.35))`。长度调整不使用回答反馈。 +- 回答前按受保护提示、最大的五个完整块、摘要预算及 96 tokens 预留／封装空间计算完整输入上限； + 若序列化后的 S 需要更多空间，则先提高上限。它是逐步增加原文的比较，**不是等长比较，也不是统一的 1000／2000-token 上限**， + 不保证每个 Full 都超过上限。计算见 [`suite_run.py`](./suite_run.py)。 + +| 任务 | 证据块数 | 摘要正文上限 | 完整输入上限 | +| --- | ---: | ---: | ---: | +| payment | 10 | 256 | 4,496 | +| checkout_504 | 5 | 256 | 5,246 | +| server_error | 41 | 256 | 3,185 | +| build_token | 8 | 256 | 3,859 | + +回答必须返回 `{"facts": {...}}`，按预设字段递归检查精确 JSON 类型和值：整数不同于布尔值、浮点数， +字符串和有序数组须精确一致；仅支付任务的 `no_retry_http_statuses` 忽略数组顺序。 +缺失、格式错误及计划中的失败条件记零分并保留在分母中。评分后不添加答案别名、改写或重试，**不使用 LLM judge**。 +先平均回答内字段，再平均同一轮的回答、同一任务的轮次，最后对任务等权平均；主任务与对照分开报告。 +另保存全部字段均正确的比例。规则见 [`suite.py`](./suite.py)。 + +图表的回答输入为 Qwen API 实际返回的 `prompt_tokens`，包含 system／任务、历史和评测问题， +不包含摘要生成和 Jev 排名请求。预算检查使用本地 tokenizer 代理，二者不能混用。 + +## 结果与结论 + +### 四个主任务 + +![两种摘要机制的字段正确率与回答输入量](../../docs/figure/compaction-topn-results.svg) + +| 方法 | 字段正确率 | Qwen 回答输入 tokens | +| --- | ---: | ---: | +| Full context | 100.0% | 4,014 | +| S | 82.0% | 2,480 | +| S + Jev Top 1 | 83.0% | 2,753 | +| S + Jev Top 2 | 94.5% | 2,974 | +| S + Jev Top 3 | 99.5% | 3,241 | +| S + Jev Top 4 | 98.1% | 3,496 | +| S + Jev Top 5 | 97.4% | 3,660 | +| H（Hermes，小预算版） | 79.4% | 2,432 | +| H + Jev Top 1 | 85.3% | 2,705 | +| H + Jev Top 2 | 92.9% | 2,927 | +| H + Jev Top 3 | 97.9% | 3,193 | +| H + Jev Top 4 | 96.9% | 3,448 | +| H + Jev Top 5 | 96.9% | 3,613 | + +在这四个任务中，Top 3 将 S 从 **82.0% 提升至 99.5%**，将 H 从 **79.4% 提升至 97.9%**； +平均回答输入相对 Full 分别减少 **19.3%／20.4%**。因此，少量原文能补充两种摘要遗漏的精确证据； +Top 4～5 的点估计没有进一步提高，追加更多原文不保证更准确。 + +所有条件满足各自输入上限，42 个纯 H 上下文全部短于 Full。但 Top 3～5 每种方法只有 **9/12 个主任务上下文** +短于配对 Full。全部 H 融合组有 172/210 个上下文缩短（主任务 51/60、对照 121/150）；其余条件仍保留评分。 +**平均缩短不代表每个样例都缩短。** + +本结果是少量合成模板上的信息保留率诊断。支付任务已用于早期开发，不是独立留出验证； +没有等长的更长摘要或其他选择器对照，不能把全部提升归因于 Jev 排名，也不能据此宣称通用最优 N、 +与 Full 统计等价或实际 agent 任务成功率提高。 + +
+十个短来源／仅依赖问题的对照 + +| 方法 | 字段正确率 | Qwen 回答输入 tokens | +| --- | ---: | ---: | +| Full context | 100.0% | 2,727 | +| S | 67.1% | 2,315 | +| S + Jev Top 1 | 84.7% | 2,493 | +| S + Jev Top 2 | 87.5% | 2,573 | +| S + Jev Top 3 | 92.9% | 2,687 | +| S + Jev Top 4 | 92.9% | 2,722 | +| S + Jev Top 5 | 92.5% | 2,722 | +| H（Hermes，小预算版） | 73.2% | 2,308 | +| H + Jev Top 1 | 80.1% | 2,486 | +| H + Jev Top 2 | 92.5% | 2,566 | +| H + Jev Top 3 | 99.2% | 2,680 | +| H + Jev Top 4 | 99.2% | 2,715 | +| H + Jev Top 5 | 98.8% | 2,715 | + +这些不是额外十个长上下文任务。可用块较少时，Top N 会达到数量上限。 + +
+ +## 复现 + +### 模型与运行环境 + +| 组件 | 版本／模型 | +| --- | --- | +| Hermes Agent `AIAgent` | commit `30463589eb54fa0f48724594cc0e98e6b995729b` | +| 采集、S／H 生成、回答模型 | `qwen3.8-max` | +| Jev（经 AgentScope／TypeSafe） | 响应解析版本 `jev-1.13.0` | +| 实验实现快照 | ReMe commit `50c2c806446c7ae6acc66217afd6e5edc2114046` | + +Qwen 请求和响应只提供模型别名，没有不可变的服务端权重版本；即使温度为 0，服务更新与非确定性也可能改变结果。 +本次使用中国北京地域的公开 DashScope OpenAI 兼容 Chat Completions 接口；接近原运行需使用相同的 `LLM_BASE_URL` 服务。 + +
+依赖版本与请求参数 + +使用两个独立 Python 环境。下表版本是**实验结束后从保留环境中核实的版本**，不是运行前固定的锁文件； +可作为复现参考，但不能证明环境已被完整锁定。本地使用 Python **3.13.3**、macOS arm64；推理由外部服务执行，不使用本地 GPU。 + +| 包 | ReMe 评测环境 | Hermes 采集／摘要环境 | +| --- | --- | --- | +| `openai` | 2.53.0 | 3.31.0 | +| `agentscope` | 2.0.10 | — | +| `typesafe-sdk` | 0.7.5 | — | +| `httpx` | 0.28.1 | 0.28.1 | +| `httpx2` | — | 2.13.1 | +| `python-dotenv` | 1.2.2 | 1.2.4 | +| `pydantic` | 2.13.4 | 2.14.1 | +| `tokenizers` | 0.23.3 | 0.23.3 | + +`tokenizers` 来自 `PYTHONPATH` 中的独立包目录，未安装进这两个 venv。新环境可分别安装该精确版本。 +本地产物包包含实验结束后记录的两个环境的发行包元数据，不包含凭证或虚拟环境二进制文件。 + +预算计数使用 `Qwen/Qwen3.5-9B` 的 **`tokenizer.json`**，revision 为 +`c202236235762e1c871ad0ccb60c8ee5ba337b9a`，SHA-256 为 +`5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42`。 +这是**预算估算代理**，不表示 Qwen 3.8 使用该 tokenizer；下游输入的报告值来自 API 的实际 usage。 + +#### 请求参数 + +| 阶段 | 采样／输出参数 | 执行限制 | +| --- | --- | --- | +| Hermes 采集 | `max_tokens=1024`；`reasoning_effort=none`、`enable_thinking=false`；未显式设置温度或 seed | 最多 12 次 agent 迭代，运行预算 240 秒；4 个 worker；子进程超时 360 秒 | +| S 生成 | `temperature=0`、`max_tokens=4096`；关闭 thinking；无 seed | OpenAI 客户端超时 90 秒，`max_retries=1` | +| H 生成 | `temperature=0`；未显式设置 API `max_tokens`；关闭 thinking；无 seed | 同样为 90 秒客户端超时、1 次 SDK 重试；4 个 worker；子进程超时 600 秒 | +| Jev 排名 | 每个证据块一个二元概率问题，每批 10 块 | 适配器超时 45 秒，`max_retries=2`，重试间隔 0.5 秒 | +| 回答 | `temperature=0`、`max_tokens=2400`、`response_format={"type":"json_object"}`；关闭 thinking；无 seed | 客户端超时 90 秒，`max_retries=1`；4 种条件并发，每种依次回答 2 次 | + +直接 Qwen 请求通过 `reasoning_effort="none"` 和 `extra_body={"enable_thinking":false}` 关闭 thinking。 +采集阶段的实际网络请求在顶层携带 `enable_thinking=false`。H 的 API 输出限制与摘要正文上限不同： +是否接受摘要由本地正文 token 检查决定。实际运行的请求参数以保存的请求正文为准。 + +采集仅开放白名单内的 `read_file`，来源工作区只读；关闭自动压缩与 micro compaction，跳过记忆文件、 +上下文文件和后台 review。每个 worker 使用独立 `HERMES_HOME`、工作区和会话数据库。 +实现见 [`hermes_worker.py`](./hermes_worker.py)。 + +
+ +### 重新采集并运行 + +使用 ReMe 开发环境；采集和 H 生成另需 Hermes Python 环境，两个环境都要能导入 `tokenizers`。 +`.env` 需配置 `LLM_API_KEY`、`LLM_BASE_URL`、`DECISION_MODEL_API_KEY`，可选 `DECISION_MODEL_BASE_URL`； +服务支持时设置 `DECISION_MODEL_NAME=jev-1.13.0`。全部命令需显式 `--live`，使用仓库外的新输出目录，不覆盖已有运行。 ```bash # 固定全部任务与标准答案，再采集 13 条新的只读轨迹。 @@ -55,18 +249,85 @@ python -m benchmark.compaction.suite_fusion --live \ --output /path/outside/repository/dual-summary-results ``` -任务定义来自 Hermes commit `30463589eb54fa0f48724594cc0e98e6b995729b`（MIT 许可）。本次使用 -`qwen3.8-max`，每个任务请求 3 轮摘要／排名，每个上下文回答 2 次：13 种方法共 **546 个上下文、1092 次回答**。 -模型请求最多同时运行 4 个。原始文件、会话、API 请求／响应、代码快照和逐字段评分均保留在 Git 仓库外，已有运行不会被覆盖。 +这复现的是实验协议，新 agent 调用可能产生不同轨迹。`suite_fusion` 只重新回答 H + Top 1～5，保留其他八组， +不会重新回答全部十三组。复用本次已有 H 时，额外传入 `--native-evaluation /path/to/pre-amendment/evaluation`； +程序校验来源哈希，仅允许评分用的 `exam` 不同，摘要生成器不接收该字段。 -S、H 在各自五种融合条件中逐字不变，两种机制使用相同的 Jev 选择结果。Top N 是按概率严格降序选出的前 N 块， -不设置概率阈值，不跳过候选块，也不截断块；入选块按来源顺序排列。若可用块少于 N，则全部保留， -并用 `actual_n` 记录实际数量。原支付轨迹有 10 块，最多 Top 10；本实验测试 Top 1～5，各任务的块数不同。 +### 冻结结果离线复核 -H 复用 Hermes 的上下文压缩摘要生成流程，并采用小预算设置：`legacy` 模板、与 S 相同的正文预算上限、 -额外字数约束、原生来源校验，以及统一的摘要正文封装。实验直接生成摘要，绕过自动触发和首尾窗口选择， -且**必须实际缩短输入**。具体设置与预检记录说明见[实验方案](./STUDY_ZH.md)。 +公开 Git 包含实验代码、任务定义、规则与整理后的结果，**不提交**原始支付轨迹、冻结语料及实际 API 记录。 +精确复核需要本地产物包，目前不属于公开的一条命令即可复现的实验。不能替换为另一条支付轨迹，预期 SHA-256 为 +`0f37e6510ffafd920d088452d8719026b00fd4e74f87f7dd1a42a85c8287e6f1`。 -本次曾修正评测问题的格式说明，而 Hermes 摘要使用修正前的来源文件。因此扩展该次运行时，需额外传入 -`--native-evaluation /path/to/pre-amendment/evaluation`。扩展程序会校验来源哈希，且只允许保留用于评分的 -`exam` 字段不同；摘要生成器不会接收这个字段。 +产物包位于 Git 仓库外： +`/Users/yuli/workspace/trajectory-runs/compaction-dual-summary-reproduction-20261016.tar.gz`。 +其 SHA-256 为 `40ee93c9117dfe59cf0b4dc8887ba4b2c62fa16f3bc97f2dac6d536343f942a3`。 +其中含 3187 个文件、相对路径 SHA-256 清单、环境元数据和 `verify.py`。 +安装 ReMe 开发环境后，在 ReMe 仓库中运行： + +```bash +# 在仓库外解压，替换为实际存放归档文件的目录。 +export REPRO_DIR=/path/outside/repository +shasum -a 256 "$REPRO_DIR/compaction-dual-summary-reproduction-20261016.tar.gz" +tar -xzf "$REPRO_DIR/compaction-dual-summary-reproduction-20261016.tar.gz" -C "$REPRO_DIR" +PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}" python \ + "$REPRO_DIR/compaction-dual-summary-reproduction-20261016/verify.py" +``` + +预期结果：**核验 3187 个产物哈希、546 个上下文和 1092 次评分；零 API 调用**。 +产物包内的 `corpus/`、`evaluation/`、`native/`、`native-source/`、`combined/`、`final/` 对应 +上述运行阶段。复用其中 H 摘要时传入 `--native-evaluation /native-source`。 + +## 附录 + + + +
+完整 query 与标准答案 + +初始 query 提供采集目标，后续问题要求返回下表的 JSON 字段；一个任务加一个后续问题仍是一个样本。 +中文 query 为释义，英文原文见[英文版](./README.md#queries-and-expected-answers)；实际只读改写及字段说明以 +[`suite.py`](./suite.py) 和冻结请求为准。标准答案的键、类型和英文字符串保持原样。 + +| ID | 初始 query 的中文释义 | 标准事实 | 分组 | +| --- | --- | --- | --- | +| payment | 调查原始支付重试事件。 | `{"incident_id": "PAY-4827", "invoice_id": "INV-731", "successful_charge_id": "CH-991", "amount_minor": 12900, "attempt_keys": ["pay-a", "pay-b"], "first_attempt_committed": true, "max_attempts": 3, "no_retry_http_statuses": [402, 422], "key_generation_line": 5, "regression_attempt_counts": {"timeout_then_success": 2, "validation_422": 1, "three_timeouts": 3}}` | 主任务 | +| backup | 从历史对话确认哪个仓库需要每晚备份，不要猜测；在工作区编写 backup.sh，将 repos/ 下该仓库归档，安排每天凌晨 2 点运行，用待办清单跟踪，并报告仓库与设置。 | `{"repo": "sparks-data", "ignored_repo": "toybox", "data_text": "important"}` | 对照：短来源 | +| refactor | 把 src/ 中所有 requests 调用的 timeout=30 改为 timeout=45，在 CHANGELOG.md 记录变更，并报告实际修改的调用点数量。 | `{"old_timeout": 30, "call_sites": 3, "functions": ["fetch_a", "fetch_b", "fetch_c"], "methods": ["get", "post", "head"], "old_version": "1.0"}` | 对照：短来源 | +| three_files | 用待办清单跟踪并核验：创建含 greet(name)、返回 `'hello '` 的 greet.py；创建表头为 item,qty、含两行 widgets 数据的 data/rows.csv；创建说明新建文件数量的 notes/summary.md。完成并检查三项后再结束。 | `{"file_count": 3, "header": "item,qty", "data_rows": 2}` | 对照：仅依赖问题 | +| active_config | 在工作区查找 retry_limit 配置，并报告当前值。 | `{"retry_limit": 7, "backoff": 2, "active_file": "conf/app.ini", "old_file": "conf/old.ini"}` | 对照：短来源 | +| checkout_504 | 调查 checkout 504 事件，区分未奏效的回滚、根因及最终数据库修复。 | `{"setting": "statement_timeout", "value_seconds": 45, "index_mode": "CONCURRENTLY", "root_cause": "CREATE INDEX without CONCURRENTLY", "rolled_back_version": "v2.30.2", "rollback_fixed": false, "deployed_version": "v2.31.0", "initial_error_percent": 8}` | 主任务 | +| dependency_version | 确认固定的 left-pad 精确版本，以及哪个文件声明测试脚本；为任务交接记录 package 脚本。 | `{"version": "1.3.0", "declaration": "package.json", "test": "vitest run", "build": "tsc -p .", "package_version": "4.2.1"}` | 对照：短来源 | +| retry_backoff | 查找 src/ 中的重试延迟函数，解释基数、增长方式、上限与随机抖动。 | `{"base_ms": 250, "cap_ms": 30000, "growth_factor": 2, "jitter": "uniform_integer_below_bound", "function": "retryDelay"}` | 对照：短来源 | +| server_error | 定位服务器日志中唯一的 ERROR，保留精确请求标识、时间戳、上游和连接失败信息。 | `{"request_id": "req-7f3d9", "timestamp": "2026-08-08T23:41:17Z", "status": 502, "upstream": "payments", "error": "ECONNREFUSED", "address": "10.0.4.17:8443"}` | 主任务 | +| meeting_notes | 读取会议记录，准确报告第三条要点，并保留其他已商定行动。 | `{"third": "rotate the API keys quarterly", "first": "ship the payments retry fix", "second": "audit the staging TLS certs", "fourth": "close out the Q3 incident review"}` | 对照：短来源 | +| targeted_edit | 在 ./handlers.py 的三处 timeout = 10 中，仅将 slow_handler 内的一处用 patch 工具改为 timeout = 60；随后在当前目录运行 python3 check_handlers.py 并回复输出。 | `{"target": "slow_handler", "original_timeouts": {"fast_handler": 10, "slow_handler": 10, "medium_handler": 10}, "required_timeouts": {"fast_handler": 10, "slow_handler": 60, "medium_handler": 10}, "success_marker": "HANDLERS_OK_552"}` | 对照：短来源 | +| build_token | 运行输出很长的 python3 ./noisy_build.py，找出中间唯一以 UNIQUE_TOKEN= 开头的行，回复完整 token 值。 | `{"token": "tok_9f31c_middle", "step": 2000, "total_steps": 4000, "prefix": "UNIQUE_TOKEN="}` | 主任务 | +| package_versions | 依次读取 pkg_a、pkg_b、pkg_c 中的 version.txt，将 a,b,c 顺序的版本以逗号连接写入 ./versions.txt，回复 DONE 和连接后的字符串。 | `{"versions": {"pkg_a": "1.4.2", "pkg_b": "0.9.7", "pkg_c": "3.2.1"}, "joined": "1.4.2,0.9.7,3.2.1"}` | 对照：短来源 | +| checksum_anomaly | 用 read_file 而非 terminal 查找 ./records.log 中唯一以 ANOMALY: 开头的行，并回复该行全文。 | `{"line": "ANOMALY: checksum drift detected in shard 7 (code X99Q)", "code": "X99Q", "shard": 7, "line_number": 4201}` | 对照：短来源 | + +
+ +
+协议修正、审计与历史产物 + +初始七组的 588 次回答暴露两处格式说明歧义：支付测试次数返回为数组，会议要点带列表标记。 +标准答案和评分器不变，仅明确支付次数须用指定键的 JSON 映射、会议要点不带列表标记。 +两任务的七种方法使用原有 S／排名统一重答，初始 84 次作为预检排除；其他 12 个任务保留 504 次回答。 +随后纯 H 新增 84 次，H 融合新增 420 次，组成当前最终结果；全部 42 份 H、排名及已有 672 次回答被复用。 +分组、证据、答案值均未因结果调整。 + +H 首次强制生成包含长控制前缀，40/42 个输入超过 Full，仅作为集成预检保留。 +正文封装和字数约束在新的评分回答前固定，不改动 S、标准答案、分组或 Jev 排名。 +H 使用格式修正前的来源文件，两版本只有不发送给生成器的 `exam` 字段不同。 +支付任务早期 `budget.json` 为 4459；最终问题增加 37 tokens，最终 `results.json` 的 4496 是评分上限。 +冻结文件保留原貌，不回写修改。 + +离线审计核验来源哈希、固定 S／H、相同且嵌套的 Top-N 原文、实际输入、全部评分、输入上限、 +已有结果未变与配置密钥未泄露；全部 546 个条件有效。原始文件、会话、API 请求／响应、代码快照和逐字段评分不进入 Git。 +历史目录位于 `/Users/yuli/workspace/trajectory-runs/`： +`hermes-fixture14-20261016/`（语料）、`compaction-fixture14-20261016-v2/`（S／排名）、 +`compaction-fixture14-hermes-adapted-20261016/`（H）、`compaction-fixture14-final-20261016/`（原八组）， +`compaction-fixture14-dual-summary-20261016/`（最终十三组）。 + +
diff --git a/benchmark/compaction/RESULTS.md b/benchmark/compaction/RESULTS.md deleted file mode 100644 index cdaea44a..00000000 --- a/benchmark/compaction/RESULTS.md +++ /dev/null @@ -1,97 +0,0 @@ -# Jev across two summary strategies - -**English** | [简体中文](./RESULTS_ZH.md) - -2026-10-16 · Qwen `qwen3.8-max` · Jev `jev-1.13.0`. - -One **13-method experiment**: Full context, generic summary **S + Top 0–5**, and Hermes compaction summary (small-budget setting) -**H + Top 0–5**. Top 0 means summary only. Both curves share the same original evidence and frozen Jev rankings; -each summary remains byte-identical across its six conditions. - -## Four primary tasks - -| Method | Fact score | Qwen answer input tokens | -| --- | ---: | ---: | -| Full context | 100.0% | 4,014 | -| S | 82.0% | 2,480 | -| S + Jev Top 1 | 83.0% | 2,753 | -| S + Jev Top 2 | 94.5% | 2,974 | -| S + Jev Top 3 | 99.5% | 3,241 | -| S + Jev Top 4 | 98.1% | 3,496 | -| S + Jev Top 5 | 97.4% | 3,660 | -| H (Hermes, small budget) | 79.4% | 2,432 | -| H + Jev Top 1 | 85.3% | 2,705 | -| H + Jev Top 2 | 92.9% | 2,927 | -| H + Jev Top 3 | 97.9% | 3,193 | -| H + Jev Top 4 | 96.9% | 3,448 | -| H + Jev Top 5 | 96.9% | 3,613 | - -![Answer quality and downstream input across two summary strategies](../../docs/figure/compaction-topn-results.svg) - -Both summary mechanisms benefit on this panel: S rises from **82.0% to 99.5%** at Top 3; H rises from **79.4% to 97.9%**. -Mean downstream inputs at Top 3 are **3,241 / 3,193**, versus Full's **4,014**: reductions of **19.3% / 20.4%**. -Top 4–5 do not improve these point estimates. This is an exploratory result on four synthetic task templates, -not a general optimal-N claim, equivalence test or proof that Jev beats other equal-budget selectors. - -Answer input is actual downstream Qwen API usage, including the original system/task, compressed history and held-out -question. It does not include summary-generation or Jev-ranking requests. The figure reports only exact-field score -and downstream context length. - -## Setup and boundaries - -[Queries, fixed answers and scoring rules](./STUDY.md) · [Experiment settings and reproducibility](./SETTINGS.md) · [Run commands](./README.md). - -The panel contains **14 task templates**, one existing payment trajectory and thirteen real read-only Hermes executions -over synthetic upstream fixtures. Four primary tasks were selected by source length before collection; ten short/query-only -controls remain separate. Three summary/ranking pairs and two answers per condition give **546 contexts / 1,092 answers**. -The extension reused all 42 H summaries, rankings and 672 previous answers; it added only 420 answering calls. -No new summary or Jev calls were needed. All 1,092 grades were replayed against captured requests/responses. - -Scoring uses predefined, recursive type-strict JSON field rules, with no LLM judge. Questions/gold/final collection reports -are excluded from summary and decision inputs. Answering calls Qwen directly, without Hermes tool continuation or recovery. -S and H share a 64–256-token maximum summary-body allowance, but use different prompts and actual lengths. -H reuses Hermes' context-compaction summary-generation pipeline with `tail_mode=legacy`, a smaller summary budget -and additional length constraints. It invokes the generator directly, bypassing automatic triggering and head/tail selection. All 42 H-only contexts strictly shorten Full, without original-history fallback. - -Top N adds complete source units without truncation, saturating at the available count. The payment source has ten units -(maximum Top 10); this experiment sweeps Top 1–5. Existing per-task input ceilings are preserved, not tuned after answers. -The comparison is incremental, **not equal-length**. All conditions fit their ceilings, but only **9/12 primary contexts -per method at Top 3–5** are shorter than their paired Full. Across the new H fusion arms, 172/210 contexts shorten Full; -the other 38 comprise 9 primary and 29 control contexts. All remain scored. Average savings do not imply per-case savings. - -
-Ten short/query-only controls - -| Method | Fact score | Qwen answer input tokens | -| --- | ---: | ---: | -| Full context | 100.0% | 2,727 | -| S | 67.1% | 2,315 | -| S + Jev Top 1 | 84.7% | 2,493 | -| S + Jev Top 2 | 87.5% | 2,573 | -| S + Jev Top 3 | 92.9% | 2,687 | -| S + Jev Top 4 | 92.9% | 2,722 | -| S + Jev Top 5 | 92.5% | 2,722 | -| H (Hermes, small budget) | 73.2% | 2,308 | -| H + Jev Top 1 | 80.1% | 2,486 | -| H + Jev Top 2 | 92.5% | 2,566 | -| H + Jev Top 3 | 99.2% | 2,680 | -| H + Jev Top 4 | 99.2% | 2,715 | -| H + Jev Top 5 | 98.8% | 2,715 | - -These controls are not ten additional long-context tasks. Their small source-unit counts can make Top N saturate. - -
- -
-Audit and artifact locations - -The audit verifies source hashes, frozen S/H, identical nested Top-N originals, exact captured inputs, all 1,092 grades, -actual token ceilings, unchanged prior results and configured-key exclusion. All 546 conditions are valid. -Prior answer-format amendments and native calibration disclosures are retained in [the protocol](./STUDY.md). - -Artifacts under `/Users/yuli/workspace/trajectory-runs/` remain outside Git: -`hermes-fixture14-20261016/` (corpus), `compaction-fixture14-20261016-v2/` (S/rankings), -`compaction-fixture14-hermes-adapted-20261016/` (H), `compaction-fixture14-final-20261016/` (original eight arms), -and `compaction-fixture14-dual-summary-20261016/` (current 13-arm results, usage and audit). - -
diff --git a/benchmark/compaction/RESULTS_ZH.md b/benchmark/compaction/RESULTS_ZH.md deleted file mode 100644 index 61e499d5..00000000 --- a/benchmark/compaction/RESULTS_ZH.md +++ /dev/null @@ -1,94 +0,0 @@ -# 两种摘要机制下的 Jev 增益 - -[English](./RESULTS.md) | **简体中文** - -2026-10-16 · Qwen `qwen3.8-max` · Jev `jev-1.13.0`。 - -一个 **13 组对照实验**：Full context、通用摘要 **S + Top 0～5**、Hermes 上下文压缩摘要（小预算版） -**H + Top 0～5**。Top 0 表示纯摘要。两条曲线使用相同的原始证据与固定 Jev 排名；每轮摘要在其六种条件中逐字不变。 - -## 四个主任务 - -| 方法 | 字段正确率 | Qwen 回答输入 tokens | -| --- | ---: | ---: | -| Full context | 100.0% | 4,014 | -| S | 82.0% | 2,480 | -| S + Jev Top 1 | 83.0% | 2,753 | -| S + Jev Top 2 | 94.5% | 2,974 | -| S + Jev Top 3 | 99.5% | 3,241 | -| S + Jev Top 4 | 98.1% | 3,496 | -| S + Jev Top 5 | 97.4% | 3,660 | -| H（Hermes，小预算版） | 79.4% | 2,432 | -| H + Jev Top 1 | 85.3% | 2,705 | -| H + Jev Top 2 | 92.9% | 2,927 | -| H + Jev Top 3 | 97.9% | 3,193 | -| H + Jev Top 4 | 96.9% | 3,448 | -| H + Jev Top 5 | 96.9% | 3,613 | - -![两种摘要机制的字段正确率与回答输入量](../../docs/figure/compaction-topn-results.svg) - -本组任务中两种摘要均能从追加原文中获益：Top 3 将 S 从 **82.0% 提升至 99.5%**，将 H 从 -**79.4% 提升至 97.9%**。Top 3 的平均回答输入分别为 **3241／3193 tokens**，Full 为 **4014 tokens**， -对应减少 **19.3%／20.4%**。Top 4～5 未提高这些点估计。这是四个合成任务模板上的探索结果， -不代表通用的最优 N、与 Full 统计等价，也不证明 Jev 优于其他等预算选择器。 - -回答输入使用下游 Qwen API 的实际 usage，包含原始 system／任务、压缩历史与评测问题， -不包含摘要生成或 Jev 排名请求。图中仅报告严格字段正确率和下游上下文长度。 - -## 设置与适用边界 - -[任务、标准答案与评分](./STUDY_ZH.md) · [实验设置与复现](./SETTINGS_ZH.md) · [运行命令](./README_ZH.md)。 - -共 **14 个任务模板**：1 条已有支付轨迹，加上基于上游合成任务只读改写、实际运行 Hermes 采集的 13 条轨迹。 -4 个主任务在采集前按来源长度确定；10 个短任务／仅依赖问题的对照单独报告。每个任务 3 轮摘要／排名， -每种条件回答 2 次，共 **546 个上下文、1092 次回答**。扩展实验复用全部 42 份 H 摘要、排名及已有 672 次回答， -只新增 420 次回答，没有新增摘要或 Jev 调用。全部 1092 次评分均已对照保存的请求／响应离线重算。 - -评分使用预设的递归、类型严格的 JSON 字段规则，没有 LLM judge。评测问题、标准答案和采集结束时的最终报告 -均不进入摘要或决策输入。回答阶段直接调用 Qwen，不继续 Hermes 工具循环，也不恢复原文。 -S、H 的摘要正文上限均为 64～256 tokens，但提示词及实际长度不同。 -H 复用 Hermes 的摘要生成流程，采用 `tail_mode=legacy`、较小的摘要预算及附加长度约束； -直接调用生成器，绕过自动触发和首尾窗口选择。全部 42 个纯 H 上下文严格短于 Full，未接受原文回退。 - -Top N 追加完整证据块，不截断；可用块不足 N 时保留全部。支付来源有 10 块，最多 Top 10；本实验测试 Top 1～5。 -沿用原有按任务预设的完整输入上限，不按回答结果调整。本比较逐步追加原文，**并非等长比较**。 -全部条件满足其上限，但 Top 3～5 每种方法只有 **9/12 个主任务上下文**短于配对 Full。 -新增 H 融合组有 172/210 个上下文实现缩短，其余 38 个包括 9 个主任务上下文和 29 个对照上下文。 -全部保留评分，平均节省不代表每个样例都节省。 - -
-十个短任务／仅依赖问题的对照 - -| 方法 | 字段正确率 | Qwen 回答输入 tokens | -| --- | ---: | ---: | -| Full context | 100.0% | 2,727 | -| S | 67.1% | 2,315 | -| S + Jev Top 1 | 84.7% | 2,493 | -| S + Jev Top 2 | 87.5% | 2,573 | -| S + Jev Top 3 | 92.9% | 2,687 | -| S + Jev Top 4 | 92.9% | 2,722 | -| S + Jev Top 5 | 92.5% | 2,722 | -| H（Hermes，小预算版） | 73.2% | 2,308 | -| H + Jev Top 1 | 80.1% | 2,486 | -| H + Jev Top 2 | 92.5% | 2,566 | -| H + Jev Top 3 | 99.2% | 2,680 | -| H + Jev Top 4 | 99.2% | 2,715 | -| H + Jev Top 5 | 98.8% | 2,715 | - -这些对照不能视为额外十个长上下文任务。来源块数较少时，Top N 会达到可用数量上限。 - -
- -
-审计与产物位置 - -审计校验来源哈希、固定 S／H、完全一致且嵌套的 Top-N 原文、实际请求输入、全部 1092 次评分、 -实际 token 上限、原有结果未变和配置密钥未泄露。全部 546 种条件有效。 -早期回答格式修正和 Hermes 长度预检的记录说明保留在[实验方案](./STUDY_ZH.md)。 - -以下产物位于 `/Users/yuli/workspace/trajectory-runs/`，不进入 Git： -`hermes-fixture14-20261016/`（语料）、`compaction-fixture14-20261016-v2/`（S／排名）、 -`compaction-fixture14-hermes-adapted-20261016/`（H）、`compaction-fixture14-final-20261016/`（原八组结果）， -以及 `compaction-fixture14-dual-summary-20261016/`（当前 13 组结果、usage 与审计）。 - -
diff --git a/benchmark/compaction/SETTINGS.md b/benchmark/compaction/SETTINGS.md deleted file mode 100644 index 6ebf0134..00000000 --- a/benchmark/compaction/SETTINGS.md +++ /dev/null @@ -1,158 +0,0 @@ -# Experiment settings and reproducibility - -**English** | [简体中文](./SETTINGS_ZH.md) - -This describes the **13-method comparison** reported in [RESULTS.md](./RESULTS.md): Full context, generic summary S -and Hermes compaction summary H, each with Jev Top 0–5. Top 0 means summary only. -[STUDY.md](./STUDY.md) defines the queries, fixed answers, scoring rules and protocol amendments. - -## Models and agent framework - -| Role | Recorded configuration | -| --- | --- | -| Trajectory collection | Hermes Agent `AIAgent`, commit `30463589eb54fa0f48724594cc0e98e6b995729b` | -| Collection model | `qwen3.8-max`, custom OpenAI-compatible Chat Completions endpoint | -| Generic summary S | Direct `qwen3.8-max` call; handoff prompt in [`cases.py`](./cases.py) | -| Hermes summary H | Same `qwen3.8-max`; Hermes `ContextCompressor._generate_summary`, `tail_mode=legacy` | -| Decision/ranking | Jev; recorded resolved response model **`jev-1.13.0`**, through AgentScope/TypeSafe | -| Answering | Direct `qwen3.8-max`, no resumed agent/tool loop | -| Grading | Deterministic, recursive type-strict JSON field checks; no LLM judge | -| Experiment implementation | ReMe commit `50c2c806446c7ae6acc66217afd6e5edc2114046` | - -H is **Hermes compaction summary under a small-budget setting**. It reuses Hermes' serialization, structured checkpoint -prompt, redaction and provenance validation. The experiment overrides the summary budget and appends a total word limit, -then removes the native control prefix using Hermes' helper. It invokes summary generation directly rather than -`compress()`: automatic triggering, head/tail-window selection and post-compression conversation repair are outside -this comparison. This is more specific than calling H a modified version of all Hermes context management. - -The Qwen alias is what the requests and captured answering responses identify; an immutable provider weight revision -was not supplied. Service-side updates and nondeterminism can change new-run results even at temperature zero. -Use the same `LLM_BASE_URL` service for the closest rerun; this endpoint and API keys are supplied privately in `.env`. -The endpoint was verified as the public China (Beijing) DashScope endpoint; API keys remain private. - -## Runtime dependencies - -Two separate Python environments were used. These versions were **verified from the retained environments after the -run**, not from a pre-run lockfile; they are a reproduction reference rather than proof of a fully locked installation. -The local run used Python **3.13.3**, macOS arm64. No local GPU performs inference. - -| Package | ReMe evaluation environment | Hermes collection/summary environment | -| --- | --- | --- | -| `openai` | 2.53.0 | 3.31.0 | -| `agentscope` | 2.0.10 | — | -| `typesafe-sdk` | 0.7.5 | — | -| `httpx` | 0.28.1 | 0.28.1 | -| `httpx2` | — | 2.13.1 | -| `python-dotenv` | 1.2.2 | 1.2.4 | -| `pydantic` | 2.13.4 | 2.14.1 | -| `tokenizers` | 0.23.3 | 0.23.3 | - -`tokenizers` was imported from a separate package directory on `PYTHONPATH`; it was not installed in either venv. -A new setup may install that exact version into each venv. The local artifact bundle includes both environments' -distribution metadata, recorded after the experiment. It does not contain credentials or virtualenv binaries. - -Token accounting uses `Qwen/Qwen3.5-9B`'s **`tokenizer.json`**, revision -`c202236235762e1c871ad0ccb60c8ee5ba337b9a`, with SHA-256 -`5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42`. -This is a **budget proxy**, not a claim that Qwen 3.8 uses that tokenizer. Reported downstream inputs use actual API usage. - -## Request parameters - -| Stage | Sampling/output parameters | Execution limits | -| --- | --- | --- | -| Hermes collection | `max_tokens=1024`; `reasoning_effort=none`, `enable_thinking=false`; no explicit temperature or seed | 12 agent iterations, 240-second run budget; four workers; 360-second subprocess timeout | -| S generation | `temperature=0`, `max_tokens=4096`; thinking disabled; no seed | OpenAI client: 90-second timeout, `max_retries=1` | -| H generation | `temperature=0`; no explicit API `max_tokens`; thinking disabled; no seed | Same 90-second client timeout / one SDK retry; four workers; 600-second subprocess timeout | -| Jev ranking | Binary probability question per evidence unit; batch size 10 | Adapter timeout 45 seconds; `max_retries=2`, retry delay 0.5 seconds | -| Answering | `temperature=0`, `max_tokens=2400`, `response_format={"type":"json_object"}`; thinking disabled; no seed | 90-second client timeout, `max_retries=1`; four concurrent conditions, two sequential answers each | - -For direct Qwen calls, thinking is disabled with `reasoning_effort="none"` and -`extra_body={"enable_thinking":false}`. The collection wire request carries `enable_thinking=false` at top level. -H's API output limit is intentionally not its summary-body allowance: acceptance uses the local body-token check. -Captured request bodies are authoritative for the recorded run. - -Collection exposes only allowlisted `read_file`; source workspaces are read-only. Automatic compression and micro -compaction are disabled during collection. Memory/context files and background review are skipped. Each worker has an -isolated `HERMES_HOME`, workspace and session DB. See [`hermes_worker.py`](./hermes_worker.py). - -## Data, pairing and budgets - -- **14 task templates**: one reused synthetic payment trajectory plus 13 actual Hermes executions over read-only - adaptations of upstream fixtures. Long sources use deterministic original head/matching-line views, not added noise. -- Four primary tasks: `payment`, `checkout_504`, `server_error`, `build_token`. Ten short/query-only controls are separate. - Assignment is fixed from source length before collection; it is not based on scores. -- Three independently requested summary/ranking pairs per task, two answering calls per condition: - **14 × 3 × 13 = 546 contexts; 1,092 answers**. Repeated answers are not independent task samples. -- S/H remain byte-identical across their own Top 0–5 conditions. Both mechanisms use identical Jev scores and source units. - Ranking is task-based, not conditioned on the summary. Select a strict descending-score prefix, break ties by unit ID, - then render in source order. No threshold, skipping or partial-unit truncation. Actual N is `min(N, available units)`. -- Summary-body allowance: `max(64, min(256, int(0.2 * evidence_tokens)))`. S allows up to three total length-check attempts, - with word targets `max(12, int(allowance * 0.4 * 0.8**attempt))`. -- H allows up to three total length-check attempts. Its requested budget is - `max(12, int(allowance * 0.8**attempt))`; appended word limit is `max(18, int(requested_budget * 0.35))`. - Native Historical Task Snapshot/no-user sentinel and Critical Context are retained; other sections may be omitted. -- The complete-input ceiling is computed before answering from protected prompts, the five largest whole evidence units, - summary allowance and 96 reserve/framing tokens, then enlarged if serialized S needs more space. It is not one universal - 1,000/2,000-token limit or an equal-length comparison. See [`suite_run.py`](./suite_run.py) for the calculation. -- Original system/task messages are preserved. Final collection reports and internal metadata are excluded. Exams/gold - never enter collection, summary or decision requests. See [`suite.py`](./suite.py) for exact follow-up construction. - -Recorded primary-case budgets below use **final scored rows**, after the format clarification: - -| Case | Evidence units | Summary allowance | Complete-input ceiling | -| --- | ---: | ---: | ---: | -| payment | 10 | 256 | 4,496 | -| checkout_504 | 5 | 256 | 5,246 | -| server_error | 41 | 256 | 3,185 | -| build_token | 8 | 256 | 3,859 | - -The retained pre-amendment payment `budget.json` says 4,459; its final question requires 37 more tokens. The final -`results.json` rows record 4,496 and govern the scored comparison. Frozen artifacts are preserved rather than rewritten. -Every condition fits its recorded ceiling. All 42 H-only contexts shorten Full; at Top 3–5, each mechanism shortens only -9/12 primary contexts per method. All cases remain scored; mean savings do not imply every case saves tokens. - -## Scoring - -Each answer returns `{"facts": {...}}`. Field checks require exact recursive types/values, with only payment HTTP-code -order ignored. Missing/malformed/failed planned conditions receive zero. Average fields within answers, answers within -generations, generations within tasks, then tasks equally. Report primary/control cohorts separately. - -Answer input is the answering API's actual `prompt_tokens`, including system/task messages, compressed history and -the follow-up question. Summary-generation and Jev-ranking requests are separate and are not added to this metric. - -## Reproduction levels and artifact availability - -1. **Frozen replay** verifies exact artifact hashes, model requests/responses and all 1,092 grades, with zero API calls. -2. **Frozen-input extension/rerun** uses `suite_fusion` to re-answer H + Top 1–5 with saved trajectories, summaries - and rankings, retaining the other eight arms. Scores can vary; this command does not re-answer all thirteen arms. -3. **Fresh collection/rerun** uses the pinned Hermes fixtures and the commands in [README.md](./README.md). New agent - calls can produce different trajectories; this reproduces the protocol rather than identical results. - -Public Git contains the experiment code, fixture definitions, rules and curated results. The original payment trajectory, -frozen corpus and wire records are **not committed**. Exact replay requires the local artifact bundle; it is not currently -a public one-command reproduction. Do not substitute another payment trajectory: its expected SHA-256 is -`0f37e6510ffafd920d088452d8719026b00fd4e74f87f7dd1a42a85c8287e6f1`. - -The prepared bundle is outside Git: -`/Users/yuli/workspace/trajectory-runs/compaction-dual-summary-reproduction-20261016.tar.gz`. -Its SHA-256 is `40ee93c9117dfe59cf0b4dc8887ba4b2c62fa16f3bc97f2dac6d536343f942a3`. -It contains 3,187 files, a relative-path SHA-256 manifest, environment metadata and `verify.py`. -With ReMe's development environment installed, from the ReMe checkout: - -```bash -# Extract outside the repository. Substitute the directory holding the archive. -export REPRO_DIR=/path/outside/repository -shasum -a 256 "$REPRO_DIR/compaction-dual-summary-reproduction-20261016.tar.gz" -tar -xzf "$REPRO_DIR/compaction-dual-summary-reproduction-20261016.tar.gz" -C "$REPRO_DIR" -PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}" python \ - "$REPRO_DIR/compaction-dual-summary-reproduction-20261016/verify.py" -``` - -Expected result: **3,187 artifact hashes, 546 contexts and 1,092 grades verified; zero API calls**. -The bundle's `corpus/`, `evaluation/`, `native/`, `native-source/`, `combined/` and `final/` directories correspond to -[README.md's run stages](./README.md). Pass `--native-evaluation /native-source` when reusing its H generations. - -Fresh runs require `.env` entries `LLM_API_KEY`, `LLM_BASE_URL`, `DECISION_MODEL_API_KEY`, optional -`DECISION_MODEL_BASE_URL`, and `DECISION_MODEL_NAME=jev-1.13.0` to request the recorded Jev version when available. -Use isolated Python environments, the versions above, the pinned Hermes checkout and tokenizer, and new output directories. -Do not commit `.env`, raw records or the bundle. No fresh model inference is needed for frozen replay. diff --git a/benchmark/compaction/SETTINGS_ZH.md b/benchmark/compaction/SETTINGS_ZH.md deleted file mode 100644 index 19365813..00000000 --- a/benchmark/compaction/SETTINGS_ZH.md +++ /dev/null @@ -1,149 +0,0 @@ -# 实验设置与复现说明 - -[English](./SETTINGS.md) | **简体中文** - -本文说明[结果页](./RESULTS_ZH.md)中的 **13 组对照**：Full context、通用摘要 S，以及 Hermes 上下文压缩摘要 H； -S、H 分别搭配 Jev Top 0～5。Top 0 表示纯摘要。[实验方案](./STUDY_ZH.md)列出问题、固定答案、评分规则及协议修正。 - -## 模型与 agent 框架 - -| 角色 | 实际记录的设置 | -| --- | --- | -| 轨迹采集 | Hermes Agent `AIAgent`，commit `30463589eb54fa0f48724594cc0e98e6b995729b` | -| 采集模型 | `qwen3.8-max`，自定义 OpenAI 兼容 Chat Completions 接口 | -| 通用摘要 S | 直接调用 `qwen3.8-max`，使用 [`cases.py`](./cases.py) 中的任务交接提示词 | -| Hermes 摘要 H | 同样使用 `qwen3.8-max`，调用 Hermes `ContextCompressor._generate_summary`，`tail_mode=legacy` | -| 决策／排名 | Jev，响应中实际解析出的版本为 **`jev-1.13.0`**，经 AgentScope／TypeSafe 调用 | -| 回答 | 直接调用 `qwen3.8-max`，不恢复 agent／工具循环 | -| 评分 | 确定性的递归、类型严格 JSON 字段规则，不使用 LLM judge | -| 实验实现 | ReMe commit `50c2c806446c7ae6acc66217afd6e5edc2114046` | - -H 是**小预算设置下的 Hermes 上下文压缩摘要**，复用 Hermes 的序列化、结构化检查点提示词、敏感信息遮蔽和来源校验。 -实验覆盖摘要预算并附加总字数上限，再用 Hermes 的辅助函数移除控制前缀。它直接生成摘要，不调用完整的 `compress()`： -自动触发、首尾窗口选择和压缩后的会话修复不在本次比较范围内。因此不能笼统称为整个 Hermes 上下文管理的修改版本。 - -请求与保存的回答响应仅标识 Qwen 别名，没有提供不可变的服务端权重版本。即使温度为 0，服务端更新和非确定性 -仍可能改变新运行的结果。尽量使用同一 `LLM_BASE_URL` 服务；接口及密钥通过私有 `.env` 提供。 -本次接口已核实为中国北京地域的公开 DashScope 接口，API Key 不公开。 - -## 运行环境依赖 - -使用两个独立 Python 环境。下表版本是**实验结束后从保留环境中核实的版本**，不是运行前固定的锁文件； -可作为复现参考，但不能证明环境已被完整锁定。本地使用 Python **3.13.3**、macOS arm64；推理由外部服务执行，不使用本地 GPU。 - -| 包 | ReMe 评测环境 | Hermes 采集／摘要环境 | -| --- | --- | --- | -| `openai` | 2.53.0 | 3.31.0 | -| `agentscope` | 2.0.10 | — | -| `typesafe-sdk` | 0.7.5 | — | -| `httpx` | 0.28.1 | 0.28.1 | -| `httpx2` | — | 2.13.1 | -| `python-dotenv` | 1.2.2 | 1.2.4 | -| `pydantic` | 2.13.4 | 2.14.1 | -| `tokenizers` | 0.23.3 | 0.23.3 | - -`tokenizers` 来自 `PYTHONPATH` 中的独立包目录，未安装进这两个 venv。新环境可分别安装该精确版本。 -本地产物包包含实验结束后记录的两个环境的发行包元数据，不包含凭证或虚拟环境二进制文件。 - -预算计数使用 `Qwen/Qwen3.5-9B` 的 **`tokenizer.json`**，revision 为 -`c202236235762e1c871ad0ccb60c8ee5ba337b9a`，SHA-256 为 -`5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42`。 -这是**预算估算代理**，不表示 Qwen 3.8 使用该 tokenizer；下游输入的报告值来自 API 的实际 usage。 - -## 请求参数 - -| 阶段 | 采样／输出参数 | 执行限制 | -| --- | --- | --- | -| Hermes 采集 | `max_tokens=1024`；`reasoning_effort=none`、`enable_thinking=false`；未显式设置温度或 seed | 最多 12 次 agent 迭代，运行预算 240 秒；4 个 worker；子进程超时 360 秒 | -| S 生成 | `temperature=0`、`max_tokens=4096`；关闭 thinking；无 seed | OpenAI 客户端超时 90 秒，`max_retries=1` | -| H 生成 | `temperature=0`；未显式设置 API `max_tokens`；关闭 thinking；无 seed | 同样为 90 秒客户端超时、1 次 SDK 重试；4 个 worker；子进程超时 600 秒 | -| Jev 排名 | 每个证据块一个二元概率问题，每批 10 块 | 适配器超时 45 秒，`max_retries=2`，重试间隔 0.5 秒 | -| 回答 | `temperature=0`、`max_tokens=2400`、`response_format={"type":"json_object"}`；关闭 thinking；无 seed | 客户端超时 90 秒，`max_retries=1`；4 种条件并发，每种依次回答 2 次 | - -直接 Qwen 请求通过 `reasoning_effort="none"` 和 `extra_body={"enable_thinking":false}` 关闭 thinking。 -采集阶段的实际网络请求在顶层携带 `enable_thinking=false`。H 的 API 输出限制与摘要正文上限不同： -是否接受摘要由本地正文 token 检查决定。实际运行的请求参数以保存的请求正文为准。 - -采集仅开放白名单内的 `read_file`，来源工作区只读；关闭自动压缩与 micro compaction，跳过记忆文件、 -上下文文件和后台 review。每个 worker 使用独立 `HERMES_HOME`、工作区和会话数据库。 -实现见 [`hermes_worker.py`](./hermes_worker.py)。 - -## 数据、配对与预算 - -- **14 个任务模板**：1 条复用的合成支付轨迹，以及在上游任务只读改写上实际运行 Hermes 的 13 条轨迹。 - 长来源采用确定性的原文开头／匹配行视图，不人为增加噪声。 -- 四个主任务为 `payment`、`checkout_504`、`server_error`、`build_token`；另有十个短任务／仅依赖问题的对照。 - 分组在采集前按来源长度固定，不依据结果选择。 -- 每个任务独立请求 3 轮摘要／排名，每种条件回答 2 次：**14 × 3 × 13 = 546 个上下文，1092 次回答**。 - 重复回答不等于独立任务样本。 -- S、H 在各自 Top 0～5 中逐字相同，两种机制使用完全一致的 Jev 分数与来源块。 - 排名依据原任务，不以摘要为条件。严格按分数降序选择，相同分数按块 ID 排序，再恢复来源顺序。 - 不设阈值，不跳过块，也不截断块；实际 N 为 `min(N, available units)`。 -- 摘要正文预算为 `max(64, min(256, int(0.2 * evidence_tokens)))`。 - S 最多进行 3 次长度检查尝试，字数目标为 `max(12, int(allowance * 0.4 * 0.8**attempt))`。 -- H 最多进行 3 次长度检查尝试。请求预算为 `max(12, int(allowance * 0.8**attempt))`， - 附加字数上限为 `max(18, int(requested_budget * 0.35))`。 - 保留原生 Historical Task Snapshot／无用户消息标记和 Critical Context，允许省略其他章节。 -- 回答前根据受保护提示、最大的五个完整证据块、摘要预算及 96 tokens 预留／封装空间计算完整输入上限。 - 若序列化后的 S 需要更多空间，则先提高上限。它不是统一的 1000／2000-token 上限，也不是等长比较。 - 计算见 [`suite_run.py`](./suite_run.py)。 -- 保留原始 system／任务，排除采集结束时的最终报告和内部元数据。评测问题与标准答案不会进入采集、 - 摘要或决策请求。精确的后续问题构造见 [`suite.py`](./suite.py)。 - -以下主任务预算取自格式说明修正后的**最终评分记录**： - -| 任务 | 证据块数 | 摘要正文上限 | 完整输入上限 | -| --- | ---: | ---: | ---: | -| payment | 10 | 256 | 4,496 | -| checkout_504 | 5 | 256 | 5,246 | -| server_error | 41 | 256 | 3,185 | -| build_token | 8 | 256 | 3,859 | - -保留的支付任务早期 `budget.json` 记录为 4459；最终问题需要额外 37 tokens，最终 `results.json` 中的记录为 4496， -评分对照以此为准。冻结产物保留原貌，不回写修改。所有条件均满足其上限；纯 H 的 42 个上下文都短于 Full。 -Top 3～5 每种机制的每组只有 9/12 个主任务上下文缩短。全部样例保留评分，平均节省不代表每个样例都节省。 - -## 评分 - -每次回答返回 `{"facts": {...}}`。字段按递归的精确类型和值规则检查，只有支付任务的 HTTP 状态码顺序可以忽略。 -缺失、格式错误或计划中的失败条件记零分。先平均回答内字段，再平均同一轮次回答、同一任务轮次， -最后对任务等权平均。主任务和对照任务分别报告。 - -回答输入采用回答 API 实际返回的 `prompt_tokens`，包含 system／任务、压缩历史和后续评测问题。 -摘要生成和 Jev 排名属于独立请求，不加入该指标。 - -## 复现层次与产物可用性 - -1. **冻结结果离线复核**：核验精确文件哈希、模型请求／响应和全部 1092 次评分，不调用任何模型 API。 -2. **固定输入扩展／重跑**：`suite_fusion` 复用轨迹、摘要和排名，重新回答 H + Top 1～5，并保留其他八组。 - 新分数可能变化；该命令不会重新回答全部十三组。 -3. **重新采集并运行**：使用固定 Hermes 素材及 [README 命令](./README_ZH.md)。新的 agent 调用可能形成不同轨迹， - 因而复现的是协议，不保证结果逐字一致。 - -公开 Git 包含实验代码、任务定义、规则与整理后的结果，**不提交**原始支付轨迹、冻结语料及实际 API 记录。 -精确复核需要本地产物包，目前不属于公开的一条命令即可复现的实验。不能替换为另一条支付轨迹，预期 SHA-256 为 -`0f37e6510ffafd920d088452d8719026b00fd4e74f87f7dd1a42a85c8287e6f1`。 - -产物包位于 Git 仓库外： -`/Users/yuli/workspace/trajectory-runs/compaction-dual-summary-reproduction-20261016.tar.gz`。 -其 SHA-256 为 `40ee93c9117dfe59cf0b4dc8887ba4b2c62fa16f3bc97f2dac6d536343f942a3`。 -其中含 3187 个文件、相对路径 SHA-256 清单、环境元数据和 `verify.py`。 -安装 ReMe 开发环境后，在 ReMe 仓库中运行： - -```bash -# 在仓库外解压，替换为实际存放归档文件的目录。 -export REPRO_DIR=/path/outside/repository -shasum -a 256 "$REPRO_DIR/compaction-dual-summary-reproduction-20261016.tar.gz" -tar -xzf "$REPRO_DIR/compaction-dual-summary-reproduction-20261016.tar.gz" -C "$REPRO_DIR" -PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}" python \ - "$REPRO_DIR/compaction-dual-summary-reproduction-20261016/verify.py" -``` - -预期结果：**核验 3187 个产物哈希、546 个上下文和 1092 次评分；零 API 调用**。 -产物包内的 `corpus/`、`evaluation/`、`native/`、`native-source/`、`combined/`、`final/` 对应 -[README 中的运行阶段](./README_ZH.md)。复用其中 H 摘要时传入 `--native-evaluation /native-source`。 - -新运行需要在 `.env` 中配置 `LLM_API_KEY`、`LLM_BASE_URL`、`DECISION_MODEL_API_KEY`、可选的 -`DECISION_MODEL_BASE_URL`，并在服务可用时设置 `DECISION_MODEL_NAME=jev-1.13.0` 以请求记录中的 Jev 版本。 -使用独立 Python 环境、上表依赖版本、固定 Hermes checkout 与 tokenizer，以及新的输出目录。 -不要提交 `.env`、原始记录或产物包。冻结结果离线复核不需要新的模型推理。 diff --git a/benchmark/compaction/STUDY.md b/benchmark/compaction/STUDY.md deleted file mode 100644 index 415f51f1..00000000 --- a/benchmark/compaction/STUDY.md +++ /dev/null @@ -1,114 +0,0 @@ -# Fourteen-task study: frozen queries, answers and rules - -**English** | [简体中文](./STUDY_ZH.md) - -Model/framework versions, exact parameters and artifact requirements are in [experiment settings](./SETTINGS.md). - -The initial task panel was frozen before collection. Explicit format corrections and native additions are recorded below. -It evaluates **14 distinct task/answer templates**: -one existing payment trajectory and thirteen newly collected read-only adaptations of Hermes task fixtures. -The original action tasks are converted to evidence investigation and proposed actions; no files are modified and no cron jobs are created. -Hermes performs real read_file calls over immutable source snapshots. Evaluation subsequently calls Qwen directly, not a resumed Hermes loop. - -## Queries and expected answers - -Each initial query below supplies the collection goal. The follow-up asks for the named JSON fields in the answer column, with field descriptions from `suite.py`. -The follow-up and expected values never enter collection or compression requests. This is one task with a follow-up, not two independent samples. - -| ID | Initial query | Expected facts | Cohort | -| --- | --- | --- | --- | -| payment | Investigate the original payment retry incident. | `{"incident_id": "PAY-4827", "invoice_id": "INV-731", "successful_charge_id": "CH-991", "amount_minor": 12900, "attempt_keys": ["pay-a", "pay-b"], "first_attempt_committed": true, "max_attempts": 3, "no_retry_http_statuses": [402, 422], "key_generation_line": 5, "regression_attempt_counts": {"timeout_then_success": 2, "validation_422": 1, "three_timeouts": 3}}` | Primary candidate | -| backup | A while back I told you one of my repos needed nightly backups — find which repo that was in our past conversations (do not guess). Then: write backup.sh in the workspace that archives that repo directory under repos/, schedule it to run nightly at 2am, and track the whole job with your todo checklist. Report back which repo it was and what you set up. | `{"repo": "sparks-data", "ignored_repo": "toybox", "data_text": "important"}` | Control: short_fixture | -| refactor | In src/, every requests call uses timeout=30. Bump them all to timeout=45, add a CHANGELOG.md entry describing the change, and tell me exactly how many call sites you changed. | `{"old_timeout": 30, "call_sites": 3, "functions": ["fetch_a", "fetch_b", "fetch_c"], "methods": ["get", "post", "head"], "old_version": "1.0"}` | Control: short_fixture | -| three_files | This is a multi-step job — track it with your todo checklist tool and keep it updated as you go, marking each step completed when verified. Steps: (1) create greet.py containing a greet(name) function that returns `'hello '`; (2) create data/rows.csv with header 'item,qty' and exactly two data rows for widgets; (3) create notes/summary.md stating how many files you created. Finish only when all three are done and checked off. | `{"file_count": 3, "header": "item,qty", "data_rows": 2}` | Control: query_only | -| active_config | Search this workspace for wherever the retry_limit setting is configured and tell me its current value. | `{"retry_limit": 7, "backoff": 2, "active_file": "conf/app.ini", "old_file": "conf/old.ini"}` | Control: short_fixture | -| checkout_504 | Investigate the checkout 504 incident: distinguish the failed rollback, root cause and final database fix. | `{"setting": "statement_timeout", "value_seconds": 45, "index_mode": "CONCURRENTLY", "root_cause": "CREATE INDEX without CONCURRENTLY", "rolled_back_version": "v2.30.2", "rollback_fixed": false, "deployed_version": "v2.31.0", "initial_error_percent": 8}` | Primary candidate | -| dependency_version | What exact left-pad version is pinned, and which file declares the test script? Record the package scripts for a handoff. | `{"version": "1.3.0", "declaration": "package.json", "test": "vitest run", "build": "tsc -p .", "package_version": "4.2.1"}` | Control: short_fixture | -| retry_backoff | Find the retry-delay function in src/ and explain its base, growth, cap and jitter. | `{"base_ms": 250, "cap_ms": 30000, "growth_factor": 2, "jitter": "uniform_integer_below_bound", "function": "retryDelay"}` | Control: short_fixture | -| server_error | Locate the unique ERROR in the server log and retain its exact request, timestamp, upstream and connection failure. | `{"request_id": "req-7f3d9", "timestamp": "2026-08-08T23:41:17Z", "status": 502, "upstream": "payments", "error": "ECONNREFUSED", "address": "10.0.4.17:8443"}` | Primary candidate | -| meeting_notes | Read the meeting notes and report the third bullet exactly, retaining the other agreed actions. | `{"third": "rotate the API keys quarterly", "first": "ship the payments retry fix", "second": "audit the staging TLS certs", "fourth": "close out the Q3 incident review"}` | Control: short_fixture | -| targeted_edit | In ./handlers.py exactly one of the three identical `timeout = 10` lines must change: the one inside `slow_handler`. Change it to `timeout = 60` using the patch tool. Then run `python3 check_handlers.py` from . and reply with its output. | `{"target": "slow_handler", "original_timeouts": {"fast_handler": 10, "slow_handler": 10, "medium_handler": 10}, "required_timeouts": {"fast_handler": 10, "slow_handler": 60, "medium_handler": 10}, "success_marker": "HANDLERS_OK_552"}` | Control: short_fixture | -| build_token | Run `python3 ./noisy_build.py` (it prints a lot). Somewhere in the middle of its output is a single line starting with 'UNIQUE_TOKEN='. Reply with the full token value. | `{"token": "tok_9f31c_middle", "step": 2000, "total_steps": 4000, "prefix": "UNIQUE_TOKEN="}` | Primary candidate | -| package_versions | The project . has three package dirs: pkg_a, pkg_b, pkg_c, each containing version.txt. Working through the directories, collect the three versions and create ./versions.txt containing them comma-separated in order (a,b,c). Reply DONE plus the joined string. | `{"versions": {"pkg_a": "1.4.2", "pkg_b": "0.9.7", "pkg_c": "3.2.1"}, "joined": "1.4.2,0.9.7,3.2.1"}` | Control: short_fixture | -| checksum_anomaly | The file ./records.log contains exactly one line starting with 'ANOMALY:'. Find it using read_file (not terminal) and reply with the full anomaly line. | `{"line": "ANOMALY: checksum drift detected in shard 7 (code X99Q)", "code": "X99Q", "shard": 7, "line_number": 4201}` | Control: short_fixture | - -## Scoring - -**No LLM judge.** The model must return `{"facts": {...}}`. Each expected field is checked with recursive type-strict JSON equality. -Integers differ from booleans and floats. Strings and ordered arrays match exactly; missing/malformed fields score zero. -Only payment `no_retry_http_statuses` ignores list order. No answer aliases, rewrites or retries are added after scoring. -Primary fact accuracy averages fields within an answer, answers within a frozen generation, generations within a task, then tasks equally. -Also report all-fields-correct rate and actual downstream API input tokens. Failed planned conditions remain zero in the denominator. -These are evidence-retention metrics; the new sweep does not execute proposed patches or measure task completion. - -## Cohorts fixed before collection - -Primary candidates require at least 2,000 source-snapshot characters and evidence-dependent facts. The original payment source is pinned separately. -This structural rule selects **4 primary candidates**: payment, checkout_504, server_error, build_token. - -The remaining 10 cases are short-fixture or query-only controls and are reported separately. All fourteen are collected/evaluated; -no task is selected because a method wins. Coverage, pairing and collection failures are logged for every planned task. - -## Paired experiment - -- Model: `qwen3.8-max`; same actual system prompt, task and follow-up within each case. Final collection reports are excluded. -- Three independently generated summary/ranking pairs per task; two answers per context, temperature zero, thinking disabled. -- Methods: Full context, S, S + Jev Top 1, 2, 3, 4, 5. Total planned: **294 contexts and 588 answers**. -- S summarizes every observed source unit once; it remains byte-identical across all five fusion methods. -- Jev sees only the initial task and source evidence. Scores are shared across N, with no probability threshold. -- Select the strict score prefix, then render in source order. If fewer than N units exist, retain all of them and report the actual count. -- Summary allowance: 64–256 local Qwen tokens, equal to 20% of rendered evidence tokens clamped to those limits. The allowance is frozen from source length. -- Before answering, calculate a per-task complete-input ceiling that fits the protected prompts, the frozen summary allowance, and the five largest whole units, plus a 64-token reserve and 32 extra framing tokens; the ceiling is enlarged before answers if needed to fit serialized S. - This size-only ceiling permits the entire N sweep without changing S or truncating evidence; actual lengths are reported rather than forced equal. -- Report Full-to-fusion input reduction. A short control may use more input tokens with fusion; it is not presented as a successful compression. -- Save protocol, source hashes, code snapshots, requests/responses, nested selections and per-field grades outside the repository. No archive recovery. -- The original payment-only table remains a development result; this new task panel is not an independent held-out confirmation because the payment task was already used. - -Fixture source: local Hermes commit `30463589eb54fa0f48724594cc0e98e6b995729b`, MIT licensed. Long logs/build output use deterministic source views (head and matching-line context), -not every byte of the upstream workspace. No artificial noise or identifier replacements are added. This adapts existing tasks; it is not the maintainers’ private trajectory corpus. - -## Format validation before the final comparison - -The first answer pass exposed two under-specified follow-up formats: payment test counts were returned as arrays, -and meeting-note bullets retained their list markers. Expected answers and the strict grader remain unchanged. -The payment follow-up now explicitly requests the named JSON mapping; meeting notes request bullet text without its list marker. -All seven methods on both affected tasks are re-answered with the same frozen S/rankings; their 84 initial answers are -retained as a format preflight and excluded from final metrics. The other twelve tasks retain all 504 original answers, -with identical questions. This correction changes no cohort, evidence, answer values, ranking or summary. - -## Hermes compaction summary: small-budget setting - -H is **Hermes compaction summary under a small-budget setting**. It calls the real -`agent.context_compressor.ContextCompressor._generate_summary` with native serialization, checkpoint template, -redaction and provenance validation. Use the built-in `tail_mode=legacy` to omit lean's additional 4,000-token session -log and deterministic appendices. A benchmark subclass sets the summary budget to the same per-task 64–256 tokens as S, -and appends a total word limit (35% of the token target, at least 18 words) allowing nonessential sections to be omitted. -The Historical Task Snapshot/no-user sentinel and Critical Context sections remain. Native validation is retained. - -Strip the native control prefix with Hermes' own helper before using the same downstream `Summary:` carrier as S. -This compares generated summary bodies without duplicating handoff framing. No model response is edited -for factual content. The native call sees only the pre-final evidence history and the original task as its focus; -no exam or gold. Allow at most three length-check attempts, reducing the requested target by 0.8, without answer feedback. - -Three summaries × two answers × fourteen tasks add **42 contexts and 84 planned answers**: **336 contexts and 672 planned -answers** across eight methods. Keep the same original system/task and exact scoring follow-up as other methods. -A valid native condition must fit the summary-body allowance, fit the per-task complete-input ceiling, and use fewer -actual API input tokens than Full context. Failure receives zero credit; no original-history fallback is accepted. - -Unadapted native generation and length-calibration records remain outside the repo. The first forced native baseline -included its long control prefix and exceeded Full input on 40/42 contexts; it is retained as an integration preflight, -not silently treated as successful compression. The explicit body carrier and word-budget adapter above are fixed -before their new scored answers. No expected answer, cohort, Jev ranking or existing S is changed. - - -## Current comparison: two frozen summary mechanisms - -The current experiment is one 13-arm panel: Full; S + Jev Top 0–5; H + Jev Top 0–5. Top 0 means summary only. -H is the Hermes compaction summary under the small-budget setting defined above. The extension reuses every original S/H summary, ranking, -selected unit and scored answer; no new summarization, Jev request or trajectory collection occurs. It adds 210 contexts -and 420 answers, for 546 contexts and 1,092 answers overall. All new inputs were frozen before answering. -S/H use identical selected units per task/generation/N, restored to source order. No answer-based reranking is allowed. - -The extension keeps the existing per-task ceilings. Strict shrinkage versus Full is reported separately, without -filtering or removing conditions: Top 3–5 each shorten 9/12 primary contexts in both mechanisms. The H fusion arms -shorten 172/210 contexts (51/60 primary, 121/150 controls). Summary-only H still shortens all 42/42. diff --git a/benchmark/compaction/STUDY_ZH.md b/benchmark/compaction/STUDY_ZH.md deleted file mode 100644 index e89f7c0e..00000000 --- a/benchmark/compaction/STUDY_ZH.md +++ /dev/null @@ -1,110 +0,0 @@ -# 十四任务实验：固定问题、标准答案与评分规则 - -[English](./STUDY.md) | **简体中文** - -模型／框架版本、精确参数及产物要求见[实验设置](./SETTINGS_ZH.md)。 - -初始任务集合在采集前已固定；后续明确修正的格式说明与 Hermes 对照扩展在下文记录。 -实验评估 **14 个不同的任务／答案模板**：1 条已有支付轨迹，以及 13 条新采集、基于 Hermes 任务素材只读改写的轨迹。 -原本要求执行操作的任务被改为调查证据和提出行动建议，不修改文件，也不创建定时任务。 -Hermes 对不可变来源快照实际调用 `read_file`；评分随后直接调用 Qwen，不恢复 Hermes 的工具循环。 - -## 问题与标准答案 - -下表给出初始 query 的**中文释义**；英文原文保留在[英文版](./STUDY.md#queries-and-expected-answers)， -实际发送的 query、只读采集改写和评测问题以 `suite.py` 及冻结请求为准，中文释义不参与模型调用。 -初始 query 提供采集目标；后续评测要求返回答案列中的 JSON 字段，字段说明来自 `suite.py`。 -评测问题和标准答案不进入采集或压缩请求。每项是一个任务加一个后续问题，不是两个独立样本。 -标准答案的字段名、类型和英文字符串保持原样，以便精确评分。 - -| ID | 初始 query 的中文释义 | 标准事实 | 分组 | -| --- | --- | --- | --- | -| payment | 调查原始支付重试事件。 | `{"incident_id": "PAY-4827", "invoice_id": "INV-731", "successful_charge_id": "CH-991", "amount_minor": 12900, "attempt_keys": ["pay-a", "pay-b"], "first_attempt_committed": true, "max_attempts": 3, "no_retry_http_statuses": [402, 422], "key_generation_line": 5, "regression_attempt_counts": {"timeout_then_success": 2, "validation_422": 1, "three_timeouts": 3}}` | 主任务候选 | -| backup | 从历史对话确认哪个仓库需要每晚备份，不要猜测；在工作区编写 backup.sh，将 repos/ 下该仓库归档，安排每天凌晨 2 点运行，用待办清单跟踪，并报告仓库与设置。 | `{"repo": "sparks-data", "ignored_repo": "toybox", "data_text": "important"}` | 对照：短来源 | -| refactor | 把 src/ 中所有 requests 调用的 timeout=30 改为 timeout=45，在 CHANGELOG.md 记录变更，并报告实际修改的调用点数量。 | `{"old_timeout": 30, "call_sites": 3, "functions": ["fetch_a", "fetch_b", "fetch_c"], "methods": ["get", "post", "head"], "old_version": "1.0"}` | 对照：短来源 | -| three_files | 用待办清单跟踪并核验：创建含 greet(name)、返回 `'hello '` 的 greet.py；创建表头为 item,qty、含两行 widgets 数据的 data/rows.csv；创建说明新建文件数量的 notes/summary.md。完成并检查三项后再结束。 | `{"file_count": 3, "header": "item,qty", "data_rows": 2}` | 对照：仅依赖问题 | -| active_config | 在工作区查找 retry_limit 配置，并报告当前值。 | `{"retry_limit": 7, "backoff": 2, "active_file": "conf/app.ini", "old_file": "conf/old.ini"}` | 对照：短来源 | -| checkout_504 | 调查 checkout 504 事件，区分未奏效的回滚、根因及最终数据库修复。 | `{"setting": "statement_timeout", "value_seconds": 45, "index_mode": "CONCURRENTLY", "root_cause": "CREATE INDEX without CONCURRENTLY", "rolled_back_version": "v2.30.2", "rollback_fixed": false, "deployed_version": "v2.31.0", "initial_error_percent": 8}` | 主任务候选 | -| dependency_version | 确认固定的 left-pad 精确版本，以及哪个文件声明测试脚本；为任务交接记录 package 脚本。 | `{"version": "1.3.0", "declaration": "package.json", "test": "vitest run", "build": "tsc -p .", "package_version": "4.2.1"}` | 对照：短来源 | -| retry_backoff | 查找 src/ 中的重试延迟函数，解释基数、增长方式、上限与随机抖动。 | `{"base_ms": 250, "cap_ms": 30000, "growth_factor": 2, "jitter": "uniform_integer_below_bound", "function": "retryDelay"}` | 对照：短来源 | -| server_error | 定位服务器日志中唯一的 ERROR，保留精确请求标识、时间戳、上游和连接失败信息。 | `{"request_id": "req-7f3d9", "timestamp": "2026-08-08T23:41:17Z", "status": 502, "upstream": "payments", "error": "ECONNREFUSED", "address": "10.0.4.17:8443"}` | 主任务候选 | -| meeting_notes | 读取会议记录，准确报告第三条要点，并保留其他已商定行动。 | `{"third": "rotate the API keys quarterly", "first": "ship the payments retry fix", "second": "audit the staging TLS certs", "fourth": "close out the Q3 incident review"}` | 对照：短来源 | -| targeted_edit | 在 ./handlers.py 的三处 timeout = 10 中，仅将 slow_handler 内的一处用 patch 工具改为 timeout = 60；随后在当前目录运行 python3 check_handlers.py 并回复输出。 | `{"target": "slow_handler", "original_timeouts": {"fast_handler": 10, "slow_handler": 10, "medium_handler": 10}, "required_timeouts": {"fast_handler": 10, "slow_handler": 60, "medium_handler": 10}, "success_marker": "HANDLERS_OK_552"}` | 对照：短来源 | -| build_token | 运行输出很长的 python3 ./noisy_build.py，找出中间唯一以 UNIQUE_TOKEN= 开头的行，回复完整 token 值。 | `{"token": "tok_9f31c_middle", "step": 2000, "total_steps": 4000, "prefix": "UNIQUE_TOKEN="}` | 主任务候选 | -| package_versions | 依次读取 pkg_a、pkg_b、pkg_c 中的 version.txt，将 a,b,c 顺序的版本以逗号连接写入 ./versions.txt，回复 DONE 和连接后的字符串。 | `{"versions": {"pkg_a": "1.4.2", "pkg_b": "0.9.7", "pkg_c": "3.2.1"}, "joined": "1.4.2,0.9.7,3.2.1"}` | 对照：短来源 | -| checksum_anomaly | 用 read_file 而非 terminal 查找 ./records.log 中唯一以 ANOMALY: 开头的行，并回复该行全文。 | `{"line": "ANOMALY: checksum drift detected in shard 7 (code X99Q)", "code": "X99Q", "shard": 7, "line_number": 4201}` | 对照：短来源 | - -## 评分 - -**不使用 LLM judge。** 模型必须返回 `{"facts": {...}}`，每个预期字段按递归、类型严格的 JSON 等值规则检查。 -整数不同于布尔值与浮点数；字符串和有序数组必须精确一致。缺失或格式错误的字段记零分。 -只有支付任务的 `no_retry_http_statuses` 忽略数组顺序。评分后不添加答案别名、改写或重试。 -主指标先平均同一回答中的字段，再平均同一固定轮次的回答、同一任务的轮次，最后对任务等权平均。 -另报告全部字段均正确的比例和下游 API 实际输入 tokens。计划中的失败条件仍留在分母中，计零分。 -这些是证据保留指标，本轮对照不执行建议补丁，也不衡量任务完成率。 - -## 采集前固定分组 - -主任务候选要求来源快照至少 2000 个字符，且事实依赖证据。原始支付来源单独固定。 -该结构性规则确定 **4 个主任务候选**：payment、checkout_504、server_error、build_token。 - -其余 10 项为短来源或仅依赖问题的对照，单独报告。全部 14 项均采集／评估，不因某种方法表现好而挑选任务。 -每个计划任务都记录来源覆盖、工具调用配对与采集失败情况。 - -## 配对实验 - -- 模型为 `qwen3.8-max`；同一任务各方法保留相同的实际 system 提示、原任务和评测问题，排除采集结束时的最终报告。 -- 每个任务独立请求 3 轮摘要／排名，每个上下文回答 2 次，温度为 0，关闭 thinking。 -- 初始方法为 Full context、S、S + Jev Top 1、2、3、4、5，计划 **294 个上下文、588 次回答**。 -- S 对全部观察到的来源块进行摘要，在五种融合方法中逐字不变。 -- Jev 只接收初始任务和来源证据。不同 N 共用分数，不设置概率阈值。 -- 严格选择分数排序的前 N 块，再按来源顺序排列。可用块不足 N 时保留全部，并报告实际数量。 -- 摘要正文上限为本地 Qwen tokenizer 估算的 64～256 tokens，即渲染后证据 tokens 的 20%，再限制到该范围；根据来源长度预先固定。 -- 回答前按任务计算完整输入上限，以容纳受保护提示、固定摘要预算、最大的五个完整块、64 tokens 预留及额外 32 tokens 封装空间。 - 若序列化后的 S 需要更多空间，则在回答前提高上限。该纯长度规则允许完整 N 对照，不修改 S、不截断证据；报告实际长度，不强制等长。 -- 报告融合相对 Full 的输入减少比例。短任务融合后可能使用更多输入 tokens，不能据此认定成功压缩。 -- 在仓库外保存实验协议、来源哈希、代码快照、请求／响应、嵌套选择和逐字段评分；不从归档恢复原文。 -- 早期仅支付任务的表属于开发结果；支付任务已经使用过，因此这个新任务集合不是独立留出验证。 - -任务素材来自 Hermes commit `30463589eb54fa0f48724594cc0e98e6b995729b`，采用 MIT 许可。 -长日志／构建输出使用确定性的原始来源视图（开头和匹配行附近），不包含上游工作区每一个字节。 -没有人为添加噪声或替换标识符。这是已有任务的改写，不是维护者私有的生产轨迹语料。 - -## 最终对照前的格式校验 - -第一次回答暴露了两处问题格式不够明确：支付测试次数被返回为数组，会议要点保留了列表标记。 -标准答案和严格评分器保持不变。支付问题现已明确要求指定名称的 JSON 映射；会议记录要求只返回要点文字、不含列表标记。 -这两个任务的全部七种方法使用同样的固定 S／排名重新回答；初始 84 次回答作为格式预检保留，不计入最终指标。 -其他 12 个任务保留原有 504 次回答，问题保持一致。修正没有改变分组、证据、答案值、排名或摘要。 - -## Hermes 上下文压缩摘要：小预算设置 - -H 是**小预算设置下的 Hermes 上下文压缩摘要**，实际调用 -`agent.context_compressor.ContextCompressor._generate_summary`，保留原生序列化、检查点模板、敏感信息遮蔽和来源校验。 -采用内置 `tail_mode=legacy`，不使用 lean 模式额外的 4000-token 会话日志及确定性附录。 -实验子类把摘要预算设为与 S 相同的每任务 64～256 tokens，并附加总字数上限（目标 token 数的 35%，至少 18 个单词）， -允许省略非必要章节。保留 Historical Task Snapshot／无用户消息标记和 Critical Context 章节，以及原生校验。 - -用 Hermes 自己的辅助函数去掉控制前缀，再与 S 使用相同的下游 `Summary:` 封装，避免重复加入交接提示。 -不改写模型输出中的事实。原生生成器仅接收最终报告前的证据历史，以原任务作为关注主题，不接收评测问题或标准答案。 -最多进行 3 次长度检查尝试，每次把请求目标乘以 0.8，不使用回答反馈。 - -14 个任务各 3 份摘要、每份 2 次回答，新增 **42 个上下文、84 次计划回答**，累计八种方法 -**336 个上下文、672 次计划回答**。与其他方法保持相同的原始 system／任务和精确评测问题。 -有效 H 条件必须满足摘要正文预算、每任务完整输入上限，且实际 API 输入少于 Full。 -失败计零分，不接受原始历史回退。 - -未适配的原生生成与长度校准记录仍在仓库外。第一次强制原生对照包含较长控制前缀，40/42 个上下文输入超过 Full； -该结果作为集成预检保留，没有被当作成功压缩。上述正文封装和字数预算设置均在新的评分回答前固定， -没有改变标准答案、分组、Jev 排名或已有 S。 - -## 当前对照：两种固定摘要机制 - -当前统一为 13 组：Full、S + Jev Top 0～5、H + Jev Top 0～5。Top 0 表示纯摘要，H 使用上述小预算设置。 -扩展复用已有全部 S／H、排名、入选块和评分回答，不重新生成摘要、不调用 Jev、不采集轨迹。 -新增 210 个上下文、420 次回答，总计 546 个上下文、1092 次回答；全部新增输入在回答前固定。 -同一任务／轮次／N 的 S、H 使用相同原文块，并按来源顺序排列，不允许根据回答结果重新排名。 - -扩展沿用已有每任务上限，单独报告是否严格短于 Full，不筛掉或删除条件。 -Top 3～5 每种机制的每组均有 9/12 个主任务上下文缩短。H 融合组共有 172/210 个上下文缩短， -其中主任务 51/60、对照 121/150；纯 H 仍为 42/42。 diff --git a/benchmark/compaction/suite_collect.py b/benchmark/compaction/suite_collect.py index 2fdd0f7e..c899ac2c 100644 --- a/benchmark/compaction/suite_collect.py +++ b/benchmark/compaction/suite_collect.py @@ -22,7 +22,7 @@ async def run(args): manifest = freeze(args.hermes.resolve(), output) shutil.copy2(args.payment_trajectory, output / "payment/trajectory.full.json") write(output / "payment/collection.json", {"completed": True, "reused": True, "trajectory_sha256": SOURCE_SHA256}) - for name in ("suite.py", "suite_collect.py", "hermes_worker.py", "STUDY.md"): + for name in ("suite.py", "suite_collect.py", "hermes_worker.py", "README.md"): shutil.copy2(Path(__file__).parent / name, output / name) semaphore = asyncio.Semaphore(args.concurrency) statuses = [] diff --git a/benchmark/compaction/suite_fusion.py b/benchmark/compaction/suite_fusion.py index 68f222b7..a9738211 100644 --- a/benchmark/compaction/suite_fusion.py +++ b/benchmark/compaction/suite_fusion.py @@ -96,7 +96,7 @@ async def run(args): if not values.get("LLM_API_KEY") or not values.get("LLM_BASE_URL"): raise ValueError("Missing Qwen credentials") output.mkdir(parents=True) - for name in ("suite_fusion.py", "suite_run.py", "STUDY.md"): + for name in ("suite_fusion.py", "suite_run.py", "README.md"): shutil.copy2(Path(__file__).parent / name, output / name) write( output / "protocol.json", diff --git a/benchmark/compaction/suite_native.py b/benchmark/compaction/suite_native.py index b57febb9..6ac93930 100644 --- a/benchmark/compaction/suite_native.py +++ b/benchmark/compaction/suite_native.py @@ -49,7 +49,7 @@ async def prepare_native(args, cases, repo): if native == repo or repo in native.parents: raise ValueError("Native artifacts must stay outside repository") native.mkdir(parents=True) - for name in ("hermes_summary.py", "STUDY.md"): + for name in ("hermes_summary.py", "README.md"): shutil.copy2(Path(__file__).parent / name, native / name) limit = asyncio.Semaphore(4) @@ -111,7 +111,7 @@ async def run(args): await prepare_native(args, cases, repo) rows = deepcopy(original) output.mkdir(parents=True) - for name in ("suite_native.py", "hermes_summary.py", "suite.py", "STUDY.md"): + for name in ("suite_native.py", "hermes_summary.py", "suite.py", "README.md"): shutil.copy2(Path(__file__).parent / name, output / name) write( output / "protocol.json", diff --git a/benchmark/compaction/suite_run.py b/benchmark/compaction/suite_run.py index 386339e3..f5365f6f 100644 --- a/benchmark/compaction/suite_run.py +++ b/benchmark/compaction/suite_run.py @@ -152,7 +152,7 @@ async def run(args): raise ValueError("Missing " + key) output.mkdir(parents=True) write(output / "corpus.json", manifest) - for name in ("suite.py", "suite_run.py", "run.py", "cases.py", "STUDY.md"): + for name in ("suite.py", "suite_run.py", "run.py", "cases.py", "README.md"): shutil.copy2(Path(__file__).parent / name, output / name) write( output / "protocol.json",