diff --git a/benchmark/compaction/README.md b/benchmark/compaction/README.md index 2bfe33a9..e1b19f3d 100644 --- a/benchmark/compaction/README.md +++ b/benchmark/compaction/README.md @@ -83,3 +83,16 @@ repair test, not a full agent task benchmark. Each context is answered twice; ea Reports include evidence and actual input tokens, factual accuracy, repair pass rates, compaction latency, usage, full requests/responses and source offsets. Compression savings alone do not establish lower total cost or better quality. The experiment may fail to show a Jev advantage; report that outcome without changing the frozen budgets. + + +To render the English comparison chart from the saved 2026-10-15 results, install optional `matplotlib` in your +plotting environment and run from the repository root (no provider calls): + +```bash +python -m benchmark.compaction.plot_results \ + --results /path/outside/repository/span-experiment/results.json \ + --output docs/figure/decision-compaction-benchmark.svg +``` + +The chart shows the controlled long-log variant at the 1000-token evidence allowance, including all four arms. +Axes start at zero; timings that were not measured are labeled explicitly. All plotted values come from `results.json`. diff --git a/benchmark/compaction/RESULTS.md b/benchmark/compaction/RESULTS.md index 56935b0e..3a124af9 100644 --- a/benchmark/compaction/RESULTS.md +++ b/benchmark/compaction/RESULTS.md @@ -86,6 +86,8 @@ | 长日志变体 | 首尾截取 | 1000 | 996 | 3593 | 60% | 100% | — | | 长日志变体 | 普通摘要 | 1000 | 877 | 3474 | 90% | 100% | 16.49 | +![Long-log compaction: input tokens, fact accuracy and compaction time](../../docs/figure/decision-compaction-benchmark.svg) + 长日志 1000 token 预算下，Jev 总输入从 8617 降至 3557，减少 **58.7%**；普通摘要降至 3474，减少 **59.7%**。 原始样本的同档预算下，Jev 保留全部事实，但长日志中的两档预算均落后于摘要。 Jev 本轮压缩耗时约 0.8 秒，摘要为 9.5—17.1 秒；首尾截取的耗时未单独测量。 diff --git a/benchmark/compaction/plot_results.py b/benchmark/compaction/plot_results.py new file mode 100644 index 00000000..2f1cc7a4 --- /dev/null +++ b/benchmark/compaction/plot_results.py @@ -0,0 +1,144 @@ +"""Render the frozen long-log comparison as a documentation SVG using optional Matplotlib.""" + +import argparse +import json +from pathlib import Path + + +def plot(results, output): + """Show all four arms at the 1000-token evidence allowance without truncating axes.""" + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + from matplotlib.ticker import PercentFormatter + + rows = { + row["arm"]: row + for row in results + if row["scenario"] == "long_log_control" and row.get("valid") and row["budget"] in (None, 1000) + } + arms = ["full", "jev_spans", "head_tail", "llm_summary"] + labels = ["Full evidence", "Jev spans", "Head/tail", "LLM summary"] + colors = ["#94a3b8", "#2563eb", "#e6a23c", "#0d9488"] + plt.rcParams.update( + { + "font.family": "DejaVu Sans", + "font.size": 11, + "svg.fonttype": "none", + "svg.hashsalt": "reme-compaction-benchmark", + }, + ) + figure, axes = plt.subplots(1, 3, figsize=(15, 7)) + figure.patch.set_facecolor("#ffffff") + figure.subplots_adjust(left=0.09, right=0.97, top=0.70, bottom=0.30, wspace=0.63) + figure.text(0.04, 0.92, "Context compaction: long-log comparison", size=23, weight="bold", color="#0f172a") + figure.text( + 0.04, + 0.86, + "Same task. Same protected context. 1,000-token evidence allowance.", + size=12, + color="#475569", + ) + configurations = [ + ("Total input tokens", "Lower is better", [rows[a]["prompt_tokens"] for a in arms], 10500), + ("Fact accuracy", "Higher is better", [100 * rows[a]["fact_accuracy"] for a in arms], 119), + ( + "Compaction time", + "Lower is better", + [ + None, + rows["jev_spans"]["compression"]["seconds"], + None, + rows["llm_summary"]["compression"]["seconds"], + ], + 20, + ), + ] + for index, (axis, (title, direction, values, ceiling)) in enumerate(zip(axes, configurations)): + axis.set_title(title + "\n" + direction, loc="left", fontsize=12, fontweight="bold", pad=20, color="#0f172a") + axis.set_xlim(0, ceiling) + axis.set_ylim(3.6, -0.65) + axis.set_yticks(range(4), labels, fontsize=10) + axis.tick_params(axis="both", length=0, colors="#475569", pad=8) + axis.set_axisbelow(True) + axis.grid(axis="x", color="#e2e8f0", linewidth=0.8) + for spine in axis.spines.values(): + spine.set_visible(False) + for y, value in enumerate(values): + if value is None: + axis.text(0, y, "Not measured", va="center", fontsize=10, color="#94a3b8") + continue + axis.barh(y, value, height=0.48, color=colors[y], zorder=3) + formatted = f"{value:,.0f}" if index == 0 else f"{value:.0f}%" if index == 1 else f"{value:.2f} s" + axis.text(value + ceiling * 0.025, y, formatted, va="center", fontsize=10, color="#0f172a") + if index == 0: + axis.set_xticks([0, 3000, 6000, 9000], ["0", "3k", "6k", "9k"]) + elif index == 1: + axis.set_xticks([0, 50, 100]) + axis.xaxis.set_major_formatter(PercentFormatter()) + else: + axis.set_xticks([0, 5, 10, 15], ["0 s", "5 s", "10 s", "15 s"]) + reduction = 100 * (1 - rows["jev_spans"]["prompt_tokens"] / rows["full"]["prompt_tokens"]) + figure.text( + 0.04, + 0.23, + f"Jev: {reduction:.1f}% fewer input tokens than full evidence.", + size=13, + weight="bold", + color="#2563eb", + ) + figure.text( + 0.04, + 0.18, + "Jev is faster in this run; the summary retains more facts (90% vs. 85%).", + size=12, + color="#334155", + ) + figure.text( + 0.04, + 0.11, + "One synthetic payment task with added routine logs; two answers per context. " + "All arms pass all six repair tests.", + size=10, + color="#64748b", + ) + figure.text( + 0.04, + 0.07, + "Budget: local Qwen tokenizer proxy. Input: API usage. " + "Latency: one local compaction measurement. No archive retrieval.", + size=10, + color="#64748b", + ) + figure.text( + 0.04, + 0.03, + "Run: 2026-10-15 | Qwen: qwen3.8-max | Jev: jev-1.13.0 | " + "Protocol and results: benchmark/compaction/RESULTS.md", + size=9, + color="#64748b", + ) + output.parent.mkdir(parents=True, exist_ok=True) + figure.savefig( + output, + format=output.suffix.lstrip("."), + metadata={"Date": "2026-10-15", "Title": "Long-log compaction comparison"}, + ) + if output.suffix == ".svg": + lines = output.read_text(encoding="utf-8").splitlines() + output.write_text("\n".join(line.rstrip() for line in lines) + "\n", encoding="utf-8") + plt.close(figure) + + +def main(): + """Read saved experiment results; plotting never calls model providers.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--results", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + plot(json.loads(args.results.read_text(encoding="utf-8")), args.output) + + +if __name__ == "__main__": + main() diff --git a/docs/en/context_compaction.md b/docs/en/context_compaction.md index 35661086..daf04be8 100644 --- a/docs/en/context_compaction.md +++ b/docs/en/context_compaction.md @@ -131,6 +131,8 @@ versus 90% on the long-log variant. Jev reduced the long-log request from 8617 t about 0.8 seconds of compaction versus 16.5 seconds for summarization. Every arm passed all six function-level repair tests, so those tests did not distinguish repair quality. +![Long-log compaction: input tokens, fact accuracy and compaction time](../figure/decision-compaction-benchmark.svg) + This run demonstrates input reduction and lower observed compaction latency, without establishing consistent quality improvements over summarization. It uses one task and its synthetic variant, with two answers per context. Budgets use a local tokenizer proxy, input usage comes from the API, and total billing cost and archive recovery were not tested. diff --git a/docs/figure/decision-compaction-benchmark.svg b/docs/figure/decision-compaction-benchmark.svg new file mode 100644 index 00000000..6c6988e1 --- /dev/null +++ b/docs/figure/decision-compaction-benchmark.svg @@ -0,0 +1,429 @@ + + + + Long-log compaction comparison + + + + + Long-log compaction comparison + 2026-10-15 + image/svg+xml + + + Matplotlib v3.11.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + 3k + + + + + + + + + 6k + + + + + + + + + 9k + + + + + + + + Full evidence + + + + + + Jev spans + + + + + + Head/tail + + + + + + LLM summary + + + + + + + + 8,617 + + + + + + 3,557 + + + + + + 3,593 + + + + + + 3,474 + + + Total input tokens + Lower is better + + + + + + + + + + + + + + 0% + + + + + + + + + 50% + + + + + + + + + 100% + + + + + + + + Full evidence + + + + + + Jev spans + + + + + + Head/tail + + + + + + LLM summary + + + + + + + + 100% + + + + + + 85% + + + + + + 60% + + + + + + 90% + + + Fact accuracy + Higher is better + + + + + + + + + + + + + + 0 s + + + + + + + + + 5 s + + + + + + + + + 10 s + + + + + + + + + 15 s + + + + + + + + Full evidence + + + + + + Jev spans + + + + + + Head/tail + + + + + + LLM summary + + + + + Not measured + + + + + + 0.79 s + + + Not measured + + + + + + 16.49 s + + + Compaction time + Lower is better + + + + Context compaction: long-log comparison + + + Same task. Same protected context. 1,000-token evidence allowance. + + + Jev: 58.7% fewer input tokens than full evidence. + + + Jev is faster in this run; the summary retains more facts (90% vs. 85%). + + + One synthetic payment task with added routine logs; two answers per context. All arms pass all six repair tests. + + + Budget: local Qwen tokenizer proxy. Input: API usage. Latency: one local compaction measurement. No archive retrieval. + + + Run: 2026-10-15 | Qwen: qwen3.8-max | Jev: jev-1.13.0 | Protocol and results: benchmark/compaction/RESULTS.md + + + + + + + + + + + + + + diff --git a/docs/zh/context_compaction.md b/docs/zh/context_compaction.md index 0fed73e6..a1406b4c 100644 --- a/docs/zh/context_compaction.md +++ b/docs/zh/context_compaction.md @@ -120,6 +120,8 @@ Jev 分块筛选、首尾截取和普通 Qwen 摘要。证据预算为 600 / 100 长日志 Jev 总输入从 8617 降至 3557 token，减少 58.7%；压缩耗时约 0.8 秒，摘要约 16.5 秒。 所有组的候选修复代码均通过六项行为测试，未区分出修复效果差异。 +![Long-log compaction: input tokens, fact accuracy and compaction time](../figure/decision-compaction-benchmark.svg) + 这项实验显示了输入压缩和较短的压缩耗时，尚未证明稳定优于普通摘要。它只有一个任务及其合成变体， 每个上下文回答两次；预算使用本地 tokenizer 代理，总输入使用 API usage，未计算总费用或测试原文恢复。 具体评分方法、完整结果、遗漏分析与复现命令均见 Benchmark 文档。