mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-10-23 05:35:30 +00:00
docs: visualize compaction benchmark results with English charts
This commit is contained in:
jinli.yl
parent
49f76bfcc4
commit
cf55876810
6 changed files with 592 additions and 0 deletions
|
|
@ -83,3 +83,16 @@ repair test, not a full agent task benchmark. Each context is answered twice; ea
|
|||
Reports include evidence and actual input tokens, factual accuracy, repair pass rates, compaction latency, usage,
|
||||
full requests/responses and source offsets. Compression savings alone do not establish lower total cost or better
|
||||
quality. The experiment may fail to show a Jev advantage; report that outcome without changing the frozen budgets.
|
||||
|
||||
|
||||
To render the English comparison chart from the saved 2026-10-15 results, install optional `matplotlib` in your
|
||||
plotting environment and run from the repository root (no provider calls):
|
||||
|
||||
```bash
|
||||
python -m benchmark.compaction.plot_results \
|
||||
--results /path/outside/repository/span-experiment/results.json \
|
||||
--output docs/figure/decision-compaction-benchmark.svg
|
||||
```
|
||||
|
||||
The chart shows the controlled long-log variant at the 1000-token evidence allowance, including all four arms.
|
||||
Axes start at zero; timings that were not measured are labeled explicitly. All plotted values come from `results.json`.
|
||||
|
|
|
|||
|
|
@ -86,6 +86,8 @@
|
|||
| 长日志变体 | 首尾截取 | 1000 | 996 | 3593 | 60% | 100% | — |
|
||||
| 长日志变体 | 普通摘要 | 1000 | 877 | 3474 | 90% | 100% | 16.49 |
|
||||
|
||||
|
||||
|
||||
长日志 1000 token 预算下，Jev 总输入从 8617 降至 3557，减少 **58.7%**；普通摘要降至 3474，减少 **59.7%**。
|
||||
原始样本的同档预算下，Jev 保留全部事实，但长日志中的两档预算均落后于摘要。
|
||||
Jev 本轮压缩耗时约 0.8 秒，摘要为 9.5—17.1 秒；首尾截取的耗时未单独测量。
|
||||
|
|
|
|||
144
benchmark/compaction/plot_results.py
Normal file
144
benchmark/compaction/plot_results.py
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
"""Render the frozen long-log comparison as a documentation SVG using optional Matplotlib."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def plot(results, output):
|
||||
"""Show all four arms at the 1000-token evidence allowance without truncating axes."""
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.ticker import PercentFormatter
|
||||
|
||||
rows = {
|
||||
row["arm"]: row
|
||||
for row in results
|
||||
if row["scenario"] == "long_log_control" and row.get("valid") and row["budget"] in (None, 1000)
|
||||
}
|
||||
arms = ["full", "jev_spans", "head_tail", "llm_summary"]
|
||||
labels = ["Full evidence", "Jev spans", "Head/tail", "LLM summary"]
|
||||
colors = ["#94a3b8", "#2563eb", "#e6a23c", "#0d9488"]
|
||||
plt.rcParams.update(
|
||||
{
|
||||
"font.family": "DejaVu Sans",
|
||||
"font.size": 11,
|
||||
"svg.fonttype": "none",
|
||||
"svg.hashsalt": "reme-compaction-benchmark",
|
||||
},
|
||||
)
|
||||
figure, axes = plt.subplots(1, 3, figsize=(15, 7))
|
||||
figure.patch.set_facecolor("#ffffff")
|
||||
figure.subplots_adjust(left=0.09, right=0.97, top=0.70, bottom=0.30, wspace=0.63)
|
||||
figure.text(0.04, 0.92, "Context compaction: long-log comparison", size=23, weight="bold", color="#0f172a")
|
||||
figure.text(
|
||||
0.04,
|
||||
0.86,
|
||||
"Same task. Same protected context. 1,000-token evidence allowance.",
|
||||
size=12,
|
||||
color="#475569",
|
||||
)
|
||||
configurations = [
|
||||
("Total input tokens", "Lower is better", [rows[a]["prompt_tokens"] for a in arms], 10500),
|
||||
("Fact accuracy", "Higher is better", [100 * rows[a]["fact_accuracy"] for a in arms], 119),
|
||||
(
|
||||
"Compaction time",
|
||||
"Lower is better",
|
||||
[
|
||||
None,
|
||||
rows["jev_spans"]["compression"]["seconds"],
|
||||
None,
|
||||
rows["llm_summary"]["compression"]["seconds"],
|
||||
],
|
||||
20,
|
||||
),
|
||||
]
|
||||
for index, (axis, (title, direction, values, ceiling)) in enumerate(zip(axes, configurations)):
|
||||
axis.set_title(title + "\n" + direction, loc="left", fontsize=12, fontweight="bold", pad=20, color="#0f172a")
|
||||
axis.set_xlim(0, ceiling)
|
||||
axis.set_ylim(3.6, -0.65)
|
||||
axis.set_yticks(range(4), labels, fontsize=10)
|
||||
axis.tick_params(axis="both", length=0, colors="#475569", pad=8)
|
||||
axis.set_axisbelow(True)
|
||||
axis.grid(axis="x", color="#e2e8f0", linewidth=0.8)
|
||||
for spine in axis.spines.values():
|
||||
spine.set_visible(False)
|
||||
for y, value in enumerate(values):
|
||||
if value is None:
|
||||
axis.text(0, y, "Not measured", va="center", fontsize=10, color="#94a3b8")
|
||||
continue
|
||||
axis.barh(y, value, height=0.48, color=colors[y], zorder=3)
|
||||
formatted = f"{value:,.0f}" if index == 0 else f"{value:.0f}%" if index == 1 else f"{value:.2f} s"
|
||||
axis.text(value + ceiling * 0.025, y, formatted, va="center", fontsize=10, color="#0f172a")
|
||||
if index == 0:
|
||||
axis.set_xticks([0, 3000, 6000, 9000], ["0", "3k", "6k", "9k"])
|
||||
elif index == 1:
|
||||
axis.set_xticks([0, 50, 100])
|
||||
axis.xaxis.set_major_formatter(PercentFormatter())
|
||||
else:
|
||||
axis.set_xticks([0, 5, 10, 15], ["0 s", "5 s", "10 s", "15 s"])
|
||||
reduction = 100 * (1 - rows["jev_spans"]["prompt_tokens"] / rows["full"]["prompt_tokens"])
|
||||
figure.text(
|
||||
0.04,
|
||||
0.23,
|
||||
f"Jev: {reduction:.1f}% fewer input tokens than full evidence.",
|
||||
size=13,
|
||||
weight="bold",
|
||||
color="#2563eb",
|
||||
)
|
||||
figure.text(
|
||||
0.04,
|
||||
0.18,
|
||||
"Jev is faster in this run; the summary retains more facts (90% vs. 85%).",
|
||||
size=12,
|
||||
color="#334155",
|
||||
)
|
||||
figure.text(
|
||||
0.04,
|
||||
0.11,
|
||||
"One synthetic payment task with added routine logs; two answers per context. "
|
||||
"All arms pass all six repair tests.",
|
||||
size=10,
|
||||
color="#64748b",
|
||||
)
|
||||
figure.text(
|
||||
0.04,
|
||||
0.07,
|
||||
"Budget: local Qwen tokenizer proxy. Input: API usage. "
|
||||
"Latency: one local compaction measurement. No archive retrieval.",
|
||||
size=10,
|
||||
color="#64748b",
|
||||
)
|
||||
figure.text(
|
||||
0.04,
|
||||
0.03,
|
||||
"Run: 2026-10-15 | Qwen: qwen3.8-max | Jev: jev-1.13.0 | "
|
||||
"Protocol and results: benchmark/compaction/RESULTS.md",
|
||||
size=9,
|
||||
color="#64748b",
|
||||
)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
figure.savefig(
|
||||
output,
|
||||
format=output.suffix.lstrip("."),
|
||||
metadata={"Date": "2026-10-15", "Title": "Long-log compaction comparison"},
|
||||
)
|
||||
if output.suffix == ".svg":
|
||||
lines = output.read_text(encoding="utf-8").splitlines()
|
||||
output.write_text("\n".join(line.rstrip() for line in lines) + "\n", encoding="utf-8")
|
||||
plt.close(figure)
|
||||
|
||||
|
||||
def main():
|
||||
"""Read saved experiment results; plotting never calls model providers."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--results", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
plot(json.loads(args.results.read_text(encoding="utf-8")), args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -131,6 +131,8 @@ versus 90% on the long-log variant. Jev reduced the long-log request from 8617 t
|
|||
about 0.8 seconds of compaction versus 16.5 seconds for summarization. Every arm passed all six function-level repair
|
||||
tests, so those tests did not distinguish repair quality.
|
||||
|
||||
|
||||
|
||||
This run demonstrates input reduction and lower observed compaction latency, without establishing consistent quality
|
||||
improvements over summarization. It uses one task and its synthetic variant, with two answers per context. Budgets use
|
||||
a local tokenizer proxy, input usage comes from the API, and total billing cost and archive recovery were not tested.
|
||||
|
|
|
|||
429
docs/figure/decision-compaction-benchmark.svg
Normal file
429
docs/figure/decision-compaction-benchmark.svg
Normal file
|
|
@ -0,0 +1,429 @@
|
|||
<?xml version="1.0" encoding="utf-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="1080pt" height="504pt" viewBox="0 0 1080 504" xmlns="http://www.w3.org/2000/svg" version="1.1">
|
||||
<title>Long-log compaction comparison</title>
|
||||
<metadata>
|
||||
<rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
|
||||
<cc:Work>
|
||||
<dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
|
||||
<dc:title>Long-log compaction comparison</dc:title>
|
||||
<dc:date>2026-10-15</dc:date>
|
||||
<dc:format>image/svg+xml</dc:format>
|
||||
<dc:creator>
|
||||
<cc:Agent>
|
||||
<dc:title>Matplotlib v3.11.2, https://matplotlib.org/</dc:title>
|
||||
</cc:Agent>
|
||||
</dc:creator>
|
||||
</cc:Work>
|
||||
</rdf:RDF>
|
||||
</metadata>
|
||||
<defs>
|
||||
<style type="text/css">*{stroke-linejoin: round; stroke-linecap: butt}</style>
|
||||
</defs>
|
||||
<g id="figure_1">
|
||||
<g id="patch_1">
|
||||
<path d="M 0 504
|
||||
L 1080 504
|
||||
L 1080 0
|
||||
L 0 0
|
||||
z
|
||||
" style="fill: #ffffff"/>
|
||||
</g>
|
||||
<g id="axes_1">
|
||||
<g id="patch_2">
|
||||
<path d="M 97.2 352.8
|
||||
L 320.298592 352.8
|
||||
L 320.298592 151.2
|
||||
L 97.2 151.2
|
||||
z
|
||||
" style="fill: #ffffff"/>
|
||||
</g>
|
||||
<g id="matplotlib.axis_1">
|
||||
<g id="xtick_1">
|
||||
<g id="line2d_1">
|
||||
<path d="M 97.2 352.8
|
||||
L 97.2 151.2
|
||||
" clip-path="url(#p7da9dc29e0)" style="fill: none; stroke: #e2e8f0; stroke-width: 0.8; stroke-linecap: square"/>
|
||||
</g>
|
||||
<g id="line2d_2"/>
|
||||
<g id="text_1">
|
||||
<text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #475569" x="97.2" y="369.157422" transform="rotate(-0 97.2 369.157422)">0</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="xtick_2">
|
||||
<g id="line2d_3">
|
||||
<path d="M 160.942455 352.8
|
||||
L 160.942455 151.2
|
||||
" clip-path="url(#p7da9dc29e0)" style="fill: none; stroke: #e2e8f0; stroke-width: 0.8; stroke-linecap: square"/>
|
||||
</g>
|
||||
<g id="line2d_4"/>
|
||||
<g id="text_2">
|
||||
<text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #475569" x="160.942455" y="369.158281" transform="rotate(-0 160.942455 369.158281)">3k</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="xtick_3">
|
||||
<g id="line2d_5">
|
||||
<path d="M 224.684909 352.8
|
||||
L 224.684909 151.2
|
||||
" clip-path="url(#p7da9dc29e0)" style="fill: none; stroke: #e2e8f0; stroke-width: 0.8; stroke-linecap: square"/>
|
||||
</g>
|
||||
<g id="line2d_6"/>
|
||||
<g id="text_3">
|
||||
<text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #475569" x="224.684909" y="369.158281" transform="rotate(-0 224.684909 369.158281)">6k</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="xtick_4">
|
||||
<g id="line2d_7">
|
||||
<path d="M 288.427364 352.8
|
||||
L 288.427364 151.2
|
||||
" clip-path="url(#p7da9dc29e0)" style="fill: none; stroke: #e2e8f0; stroke-width: 0.8; stroke-linecap: square"/>
|
||||
</g>
|
||||
<g id="line2d_8"/>
|
||||
<g id="text_4">
|
||||
<text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #475569" x="288.427364" y="369.158281" transform="rotate(-0 288.427364 369.158281)">9k</text>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
<g id="matplotlib.axis_2">
|
||||
<g id="ytick_1">
|
||||
<g id="line2d_9"/>
|
||||
<g id="text_5">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #475569" x="89.2" y="185.83216" transform="rotate(-0 89.2 185.83216)">Full evidence</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="ytick_2">
|
||||
<g id="line2d_10"/>
|
||||
<g id="text_6">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #475569" x="89.2" y="233.267063" transform="rotate(-0 89.2 233.267063)">Jev spans</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="ytick_3">
|
||||
<g id="line2d_11"/>
|
||||
<g id="text_7">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #475569" x="89.2" y="280.702748" transform="rotate(-0 89.2 280.702748)">Head/tail</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="ytick_4">
|
||||
<g id="line2d_12"/>
|
||||
<g id="text_8">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #475569" x="89.2" y="328.137652" transform="rotate(-0 89.2 328.137652)">LLM summary</text>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
<g id="patch_3">
|
||||
<path d="M 97.2 170.648471
|
||||
L 280.289577 170.648471
|
||||
L 280.289577 193.417412
|
||||
L 97.2 193.417412
|
||||
z
|
||||
" clip-path="url(#p7da9dc29e0)" style="fill: #94a3b8"/>
|
||||
</g>
|
||||
<g id="text_9">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0f172a" x="285.867042" y="184.630597" transform="rotate(-0 285.867042 184.630597)">8,617</text>
|
||||
</g>
|
||||
<g id="patch_4">
|
||||
<path d="M 97.2 218.083765
|
||||
L 172.777304 218.083765
|
||||
L 172.777304 240.852706
|
||||
L 97.2 240.852706
|
||||
z
|
||||
" clip-path="url(#p7da9dc29e0)" style="fill: #2563eb"/>
|
||||
</g>
|
||||
<g id="text_10">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0f172a" x="178.354769" y="232.065892" transform="rotate(-0 178.354769 232.065892)">3,557</text>
|
||||
</g>
|
||||
<g id="patch_5">
|
||||
<path d="M 97.2 265.519059
|
||||
L 173.542213 265.519059
|
||||
L 173.542213 288.288
|
||||
L 97.2 288.288
|
||||
z
|
||||
" clip-path="url(#p7da9dc29e0)" style="fill: #e6a23c"/>
|
||||
</g>
|
||||
<g id="text_11">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0f172a" x="179.119678" y="279.501186" transform="rotate(-0 179.119678 279.501186)">3,593</text>
|
||||
</g>
|
||||
<g id="patch_6">
|
||||
<path d="M 97.2 312.954353
|
||||
L 171.013763 312.954353
|
||||
L 171.013763 335.723294
|
||||
L 97.2 335.723294
|
||||
z
|
||||
" clip-path="url(#p7da9dc29e0)" style="fill: #0d9488"/>
|
||||
</g>
|
||||
<g id="text_12">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0f172a" x="176.591227" y="326.93648" transform="rotate(-0 176.591227 326.93648)">3,474</text>
|
||||
</g>
|
||||
<g id="text_13">
|
||||
<text style="font-weight: 700; font-size: 12px; font-family: 'DejaVu Sans'; fill: #0f172a" transform="translate(97.2 116.796719)">Total input tokens</text>
|
||||
<text style="font-weight: 700; font-size: 12px; font-family: 'DejaVu Sans'; fill: #0f172a" transform="translate(97.2 131.2)">Lower is better</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="axes_2">
|
||||
<g id="patch_7">
|
||||
<path d="M 460.850704 352.8
|
||||
L 683.949296 352.8
|
||||
L 683.949296 151.2
|
||||
L 460.850704 151.2
|
||||
z
|
||||
" style="fill: #ffffff"/>
|
||||
</g>
|
||||
<g id="matplotlib.axis_3">
|
||||
<g id="xtick_5">
|
||||
<g id="line2d_13">
|
||||
<path d="M 460.850704 352.8
|
||||
L 460.850704 151.2
|
||||
" clip-path="url(#pd44790abba)" style="fill: none; stroke: #e2e8f0; stroke-width: 0.8; stroke-linecap: square"/>
|
||||
</g>
|
||||
<g id="line2d_14"/>
|
||||
<g id="text_14">
|
||||
<text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #475569" x="460.850704" y="369.157422" transform="rotate(-0 460.850704 369.157422)">0%</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="xtick_6">
|
||||
<g id="line2d_15">
|
||||
<path d="M 554.589608 352.8
|
||||
L 554.589608 151.2
|
||||
" clip-path="url(#pd44790abba)" style="fill: none; stroke: #e2e8f0; stroke-width: 0.8; stroke-linecap: square"/>
|
||||
</g>
|
||||
<g id="line2d_16"/>
|
||||
<g id="text_15">
|
||||
<text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #475569" x="554.589608" y="369.157422" transform="rotate(-0 554.589608 369.157422)">50%</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="xtick_7">
|
||||
<g id="line2d_17">
|
||||
<path d="M 648.328512 352.8
|
||||
L 648.328512 151.2
|
||||
" clip-path="url(#pd44790abba)" style="fill: none; stroke: #e2e8f0; stroke-width: 0.8; stroke-linecap: square"/>
|
||||
</g>
|
||||
<g id="line2d_18"/>
|
||||
<g id="text_16">
|
||||
<text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #475569" x="648.328512" y="369.157422" transform="rotate(-0 648.328512 369.157422)">100%</text>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
<g id="matplotlib.axis_4">
|
||||
<g id="ytick_5">
|
||||
<g id="line2d_19"/>
|
||||
<g id="text_17">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #475569" x="452.850704" y="185.83216" transform="rotate(-0 452.850704 185.83216)">Full evidence</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="ytick_6">
|
||||
<g id="line2d_20"/>
|
||||
<g id="text_18">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #475569" x="452.850704" y="233.267063" transform="rotate(-0 452.850704 233.267063)">Jev spans</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="ytick_7">
|
||||
<g id="line2d_21"/>
|
||||
<g id="text_19">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #475569" x="452.850704" y="280.702748" transform="rotate(-0 452.850704 280.702748)">Head/tail</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="ytick_8">
|
||||
<g id="line2d_22"/>
|
||||
<g id="text_20">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #475569" x="452.850704" y="328.137652" transform="rotate(-0 452.850704 328.137652)">LLM summary</text>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
<g id="patch_8">
|
||||
<path d="M 460.850704 170.648471
|
||||
L 648.328512 170.648471
|
||||
L 648.328512 193.417412
|
||||
L 460.850704 193.417412
|
||||
z
|
||||
" clip-path="url(#pd44790abba)" style="fill: #94a3b8"/>
|
||||
</g>
|
||||
<g id="text_21">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0f172a" x="653.905977" y="184.630597" transform="rotate(-0 653.905977 184.630597)">100%</text>
|
||||
</g>
|
||||
<g id="patch_9">
|
||||
<path d="M 460.850704 218.083765
|
||||
L 620.206841 218.083765
|
||||
L 620.206841 240.852706
|
||||
L 460.850704 240.852706
|
||||
z
|
||||
" clip-path="url(#pd44790abba)" style="fill: #2563eb"/>
|
||||
</g>
|
||||
<g id="text_22">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0f172a" x="625.784306" y="232.065892" transform="rotate(-0 625.784306 232.065892)">85%</text>
|
||||
</g>
|
||||
<g id="patch_10">
|
||||
<path d="M 460.850704 265.519059
|
||||
L 573.337389 265.519059
|
||||
L 573.337389 288.288
|
||||
L 460.850704 288.288
|
||||
z
|
||||
" clip-path="url(#pd44790abba)" style="fill: #e6a23c"/>
|
||||
</g>
|
||||
<g id="text_23">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0f172a" x="578.914854" y="279.501186" transform="rotate(-0 578.914854 279.501186)">60%</text>
|
||||
</g>
|
||||
<g id="patch_11">
|
||||
<path d="M 460.850704 312.954353
|
||||
L 629.580731 312.954353
|
||||
L 629.580731 335.723294
|
||||
L 460.850704 335.723294
|
||||
z
|
||||
" clip-path="url(#pd44790abba)" style="fill: #0d9488"/>
|
||||
</g>
|
||||
<g id="text_24">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0f172a" x="635.158196" y="326.93648" transform="rotate(-0 635.158196 326.93648)">90%</text>
|
||||
</g>
|
||||
<g id="text_25">
|
||||
<text style="font-weight: 700; font-size: 12px; font-family: 'DejaVu Sans'; fill: #0f172a" transform="translate(460.850704 116.796719)">Fact accuracy</text>
|
||||
<text style="font-weight: 700; font-size: 12px; font-family: 'DejaVu Sans'; fill: #0f172a" transform="translate(460.850704 131.2)">Higher is better</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="axes_3">
|
||||
<g id="patch_12">
|
||||
<path d="M 824.501408 352.8
|
||||
L 1047.6 352.8
|
||||
L 1047.6 151.2
|
||||
L 824.501408 151.2
|
||||
z
|
||||
" style="fill: #ffffff"/>
|
||||
</g>
|
||||
<g id="matplotlib.axis_5">
|
||||
<g id="xtick_8">
|
||||
<g id="line2d_23">
|
||||
<path d="M 824.501408 352.8
|
||||
L 824.501408 151.2
|
||||
" clip-path="url(#p1ac8612505)" style="fill: none; stroke: #e2e8f0; stroke-width: 0.8; stroke-linecap: square"/>
|
||||
</g>
|
||||
<g id="line2d_24"/>
|
||||
<g id="text_26">
|
||||
<text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #475569" x="824.501408" y="369.157422" transform="rotate(-0 824.501408 369.157422)">0 s</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="xtick_9">
|
||||
<g id="line2d_25">
|
||||
<path d="M 880.276056 352.8
|
||||
L 880.276056 151.2
|
||||
" clip-path="url(#p1ac8612505)" style="fill: none; stroke: #e2e8f0; stroke-width: 0.8; stroke-linecap: square"/>
|
||||
</g>
|
||||
<g id="line2d_26"/>
|
||||
<g id="text_27">
|
||||
<text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #475569" x="880.276056" y="369.157422" transform="rotate(-0 880.276056 369.157422)">5 s</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="xtick_10">
|
||||
<g id="line2d_27">
|
||||
<path d="M 936.050704 352.8
|
||||
L 936.050704 151.2
|
||||
" clip-path="url(#p1ac8612505)" style="fill: none; stroke: #e2e8f0; stroke-width: 0.8; stroke-linecap: square"/>
|
||||
</g>
|
||||
<g id="line2d_28"/>
|
||||
<g id="text_28">
|
||||
<text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #475569" x="936.050704" y="369.157422" transform="rotate(-0 936.050704 369.157422)">10 s</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="xtick_11">
|
||||
<g id="line2d_29">
|
||||
<path d="M 991.825352 352.8
|
||||
L 991.825352 151.2
|
||||
" clip-path="url(#p1ac8612505)" style="fill: none; stroke: #e2e8f0; stroke-width: 0.8; stroke-linecap: square"/>
|
||||
</g>
|
||||
<g id="line2d_30"/>
|
||||
<g id="text_29">
|
||||
<text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #475569" x="991.825352" y="369.157422" transform="rotate(-0 991.825352 369.157422)">15 s</text>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
<g id="matplotlib.axis_6">
|
||||
<g id="ytick_9">
|
||||
<g id="line2d_31"/>
|
||||
<g id="text_30">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #475569" x="816.501408" y="185.83216" transform="rotate(-0 816.501408 185.83216)">Full evidence</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="ytick_10">
|
||||
<g id="line2d_32"/>
|
||||
<g id="text_31">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #475569" x="816.501408" y="233.267063" transform="rotate(-0 816.501408 233.267063)">Jev spans</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="ytick_11">
|
||||
<g id="line2d_33"/>
|
||||
<g id="text_32">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #475569" x="816.501408" y="280.702748" transform="rotate(-0 816.501408 280.702748)">Head/tail</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="ytick_12">
|
||||
<g id="line2d_34"/>
|
||||
<g id="text_33">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #475569" x="816.501408" y="328.137652" transform="rotate(-0 816.501408 328.137652)">LLM summary</text>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
<g id="text_34">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #94a3b8" x="824.501408" y="184.630988" transform="rotate(-0 824.501408 184.630988)">Not measured</text>
|
||||
</g>
|
||||
<g id="patch_13">
|
||||
<path d="M 824.501408 218.083765
|
||||
L 833.343425 218.083765
|
||||
L 833.343425 240.852706
|
||||
L 824.501408 240.852706
|
||||
z
|
||||
" clip-path="url(#p1ac8612505)" style="fill: #2563eb"/>
|
||||
</g>
|
||||
<g id="text_35">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0f172a" x="838.92089" y="232.065892" transform="rotate(-0 838.92089 232.065892)">0.79 s</text>
|
||||
</g>
|
||||
<g id="text_36">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #94a3b8" x="824.501408" y="279.501576" transform="rotate(-0 824.501408 279.501576)">Not measured</text>
|
||||
</g>
|
||||
<g id="patch_14">
|
||||
<path d="M 824.501408 312.954353
|
||||
L 1008.458266 312.954353
|
||||
L 1008.458266 335.723294
|
||||
L 824.501408 335.723294
|
||||
z
|
||||
" clip-path="url(#p1ac8612505)" style="fill: #0d9488"/>
|
||||
</g>
|
||||
<g id="text_37">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0f172a" x="1014.035731" y="326.93648" transform="rotate(-0 1014.035731 326.93648)">16.49 s</text>
|
||||
</g>
|
||||
<g id="text_38">
|
||||
<text style="font-weight: 700; font-size: 12px; font-family: 'DejaVu Sans'; fill: #0f172a" transform="translate(824.501408 116.796719)">Compaction time</text>
|
||||
<text style="font-weight: 700; font-size: 12px; font-family: 'DejaVu Sans'; fill: #0f172a" transform="translate(824.501408 131.2)">Lower is better</text>
|
||||
</g>
|
||||
</g>
|
||||
<g id="text_39">
|
||||
<text style="font-weight: 700; font-size: 23px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0f172a" x="43.2" y="40.32" transform="rotate(-0 43.2 40.32)">Context compaction: long-log comparison</text>
|
||||
</g>
|
||||
<g id="text_40">
|
||||
<text style="font-size: 12px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #475569" x="43.2" y="70.56" transform="rotate(-0 43.2 70.56)">Same task. Same protected context. 1,000-token evidence allowance.</text>
|
||||
</g>
|
||||
<g id="text_41">
|
||||
<text style="font-weight: 700; font-size: 13px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #2563eb" x="43.2" y="388.08" transform="rotate(-0 43.2 388.08)">Jev: 58.7% fewer input tokens than full evidence.</text>
|
||||
</g>
|
||||
<g id="text_42">
|
||||
<text style="font-size: 12px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #334155" x="43.2" y="413.28" transform="rotate(-0 43.2 413.28)">Jev is faster in this run; the summary retains more facts (90% vs. 85%).</text>
|
||||
</g>
|
||||
<g id="text_43">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #64748b" x="43.2" y="448.56" transform="rotate(-0 43.2 448.56)">One synthetic payment task with added routine logs; two answers per context. All arms pass all six repair tests.</text>
|
||||
</g>
|
||||
<g id="text_44">
|
||||
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #64748b" x="43.2" y="468.72" transform="rotate(-0 43.2 468.72)">Budget: local Qwen tokenizer proxy. Input: API usage. Latency: one local compaction measurement. No archive retrieval.</text>
|
||||
</g>
|
||||
<g id="text_45">
|
||||
<text style="font-size: 9px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #64748b" x="43.2" y="488.88" transform="rotate(-0 43.2 488.88)">Run: 2026-10-15 | Qwen: qwen3.8-max | Jev: jev-1.13.0 | Protocol and results: benchmark/compaction/RESULTS.md</text>
|
||||
</g>
|
||||
</g>
|
||||
<defs>
|
||||
<clipPath id="p7da9dc29e0">
|
||||
<rect x="97.2" y="151.2" width="223.098592" height="201.6"/>
|
||||
</clipPath>
|
||||
<clipPath id="pd44790abba">
|
||||
<rect x="460.850704" y="151.2" width="223.098592" height="201.6"/>
|
||||
</clipPath>
|
||||
<clipPath id="p1ac8612505">
|
||||
<rect x="824.501408" y="151.2" width="223.098592" height="201.6"/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 18 KiB |
|
|
@ -120,6 +120,8 @@ Jev 分块筛选、首尾截取和普通 Qwen 摘要。证据预算为 600 / 100
|
|||
长日志 Jev 总输入从 8617 降至 3557 token，减少 58.7%；压缩耗时约 0.8 秒，摘要约 16.5 秒。
|
||||
所有组的候选修复代码均通过六项行为测试，未区分出修复效果差异。
|
||||
|
||||
|
||||
|
||||
这项实验显示了输入压缩和较短的压缩耗时，尚未证明稳定优于普通摘要。它只有一个任务及其合成变体，
|
||||
每个上下文回答两次；预算使用本地 tokenizer 代理，总输入使用 API usage，未计算总费用或测试原文恢复。
|
||||
具体评分方法、完整结果、遗漏分析与复现命令均见 Benchmark 文档。
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue