diff --git a/.github/workflows/_build-python-packages.yml b/.github/workflows/_build-python-packages.yml index b4bfcf85..ed95d679 100644 --- a/.github/workflows/_build-python-packages.yml +++ b/.github/workflows/_build-python-packages.yml @@ -64,6 +64,7 @@ jobs: python -m zipfile -l "${REME_WHEEL}" | (! grep 'reme_studio/') python -m venv "${RUNNER_TEMP}/reme-package-smoke" "${RUNNER_TEMP}/reme-package-smoke/bin/python" -m pip install "${REME_WHEEL}[as]" + "${RUNNER_TEMP}/reme-package-smoke/bin/python" -m pip check cd "${RUNNER_TEMP}" "${RUNNER_TEMP}/reme-package-smoke/bin/python" -c "import reme" @@ -93,16 +94,6 @@ jobs: assert runtime.config["jobs"]["auto_bash"]["steps"] == [{"backend": "auto_bash_step"}] PY - - name: Reject incompatible published ReMe for Home - run: | - HOME_WHEEL="$(pwd)/$(ls dist/home/reme_home-[0-9]*.whl)" - if python -m pip install --dry-run --ignore-installed "${HOME_WHEEL}" "reme-ai[as]==0.4.2.0" > "${RUNNER_TEMP}/home-incompatible.log" 2>&1; then - echo 'Home must not install with the published ReMe 0.4.2.0 wheel.' - exit 1 - fi - cat "${RUNNER_TEMP}/home-incompatible.log" - grep -q 'ResolutionImpossible' "${RUNNER_TEMP}/home-incompatible.log" - - name: Verify released core dependencies if: inputs.upload_artifacts run: | diff --git a/.gitignore b/.gitignore index cd78a0c2..2e1599f9 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,9 @@ tmp*/ temp*/ .trash/ +# Local tool indexes (rebuildable, machine-local) +.codegraph/ + # ReMe runtime data .reme/ reme_workspace/ @@ -59,6 +62,9 @@ docs/_build/ site/ evaluation/ +# Home's evaluation package is source code, not benchmark output. +!plugins/home/src/reme_home/evaluation/ +!plugins/home/src/reme_home/evaluation/*.py # The pi-Bench suite ships its own trace-history render config, which must # stay in git even though it lives under an evaluation/ directory. !benchmark/pibench/config/bench/evaluation/ diff --git a/benchmark/beam/README.md b/benchmark/beam/README.md index e8425956..8dd438aa 100644 --- a/benchmark/beam/README.md +++ b/benchmark/beam/README.md @@ -142,3 +142,101 @@ Memory Construction average token consumption (default agent, full build over 35 | Agent | input tok/case | output tok/case | total tok/case | |---|---|---|---| | default | 31,943,817 | 1,417,061 | 33,360,878 | + + +## Prepare a Home experiment + +`run_home.py` prepares all BEAM **100K** cases, the workspace, and committed candidate +code. It exits after preparation; it does not evaluate cases or start optimization. +Run from the repository root after obtaining the dataset described above: + +```bash +reme plugins install plugins/home +python benchmark/beam/run_home.py +# Optional separate workspace (a successful preparation cannot be repeated): +python benchmark/beam/run_home.py --workspace-dir /path/to/new-workspace +``` + +[config_home.yaml](config_home.yaml) controls the preparation. Its paths are +repository-relative regardless of the current working directory; `--config` +accepts an alternative runner YAML file. The default workspace is +`benchmark/beam/workspaces/home-100K`. The evaluated runtime loads `beam`; +the host application loads `home`. `source_ref: HEAD` exports committed source +only, excluding uncommitted modifications. Preparation failure rolls back its +outputs; after success, another call fails instead of resetting the experiment. + +The top-level `sampling_policy` configures Home evaluation defaults: `max_cases`, +`max_queries` (null means unlimited), `temperature`, `ewma_alpha` and +`uniform_exploration`. The runner validates these and passes them to the host +`home.evaluation` Component. It still only prepares; it does not run evaluations. +Each evaluation controls `sampling` and `record_history` separately. + +The top-level `validation_cycle_time_budget_minutes` configures the host optimizer's +advisory time between completed formal evaluations. It defaults to 150 minutes; +set a positive number to change it or null to disable timed reminders. + +The top-level `max_evaluations` sets the optimizer's formal-evaluation budget +(positive integer, default 10). A slot is consumed at first sandbox startup, +including later failures or cancellations; initial baseline and debug are excluded. + +The top-level `memory_limit_mib` sets the host agent sandbox memory limit for +optimization, diagnosis, case studies and evaluation reviews. Set `memory_limit_mib: 0` +in the runner YAML to disable the Linux cgroup limit. Null or omission keeps platform +defaults (4096 MiB on Linux); macOS does not enforce memory limits. + +After preparation, run local one-shot optimization with the same configuration: + +```bash +python benchmark/beam/run_home.py --optimize +# Use the same --config and --workspace-dir overrides as preparation, if supplied. +python benchmark/beam/run_home.py --optimize --instruction "Improve retrieval quality" +``` + +This starts a fresh application, applies both budgets and the sampling policy, +runs `home_optimize`, and closes the application. It does not repeat preparation. +Python callers can use `await optimize(config_path, workspace_dir=...)` from the +same module. The default `run()` and CLI behavior remain preparation only. + +Python callers can use the same lifecycle: + +```python +from benchmark.beam.run_home import run + +result = await run(workspace_dir="/path/to/new-workspace") +``` + + +## Evaluate a candidate on an independent test set + +The default evaluates every 100K case, matching the previous standalone evaluation. This is an independent +workspace, not a disjoint held-out split. Select another `variant`, or replace `all_cases: true` with +`case_ids: [...]`, in [config_home_eval.yaml](config_home_eval.yaml) for a different test set. + +The entry point imports a full candidate SHA or unambiguous hexadecimal prefix from an existing Home +optimization workspace and copies its prepared fixed runtime. It does not require the original ReMe source +repository or change the optimizer's checkout. Paths in the configuration are repository-relative. + +Prepare first, then use the service from two terminals (replace `CANDIDATE_SHA`): + +```bash +python benchmark/beam/run_home_eval.py --commit CANDIDATE_SHA --prepare-only +# Terminal 1: +reme start workspace_dir=benchmark/beam/workspaces/home-test-100K 'plugins=["home","beam-judge"]' +# Terminal 2: +reme home_evaluate workspace_dir=benchmark/beam/workspaces/home-test-100K record_history=false +``` + +### Local one-shot test evaluation + +```bash +python benchmark/beam/run_home_eval.py --commit CANDIDATE_SHA +# Optional overrides: --config /path/to/test.yaml --source-workspace /path/to/search --test-workspace /path/to/test +``` + +Python callers can use `await benchmark.beam.run_home_eval.run(commit="CANDIDATE_SHA")`. +The runner rebuilds construction and scores every selected query with sampling and history recording disabled. +It uses the existing Docker evaluation runtime and host judge configuration; model credentials are required +for evaluation. Results stay in the test workspace's `home/evaluations/`, with original candidate provenance +in each manifest's `test_evaluation`. The exported snapshot has its own local `evaluation_commit_sha`. +Rerunning the same request reopens unchanged prepared inputs and creates another evaluation result. Use a +new test workspace for another candidate or selection. Source and test workspaces must not overlap. diff --git a/benchmark/beam/README_ZH.md b/benchmark/beam/README_ZH.md index 781d30ba..f78d33a1 100644 --- a/benchmark/beam/README_ZH.md +++ b/benchmark/beam/README_ZH.md @@ -130,3 +130,84 @@ Memory Construction 平均 token 消耗（default agent，35 cases 全量构建 | Agent | input tok/case | output tok/case | total tok/case | |---|---|---|---| | default | 31,943,817 | 1,417,061 | 33,360,878 | + + +## 准备 Home 实验 + +安装 Home 插件后，在仓库根目录执行： + +```bash +reme plugins install plugins/home +python benchmark/beam/run_home.py +``` + +该入口只创建工作区、准备数据和代码，不运行评测或优化。 +参数见 [config_home.yaml](config_home.yaml)，路径相对仓库根目录； +可通过 `--config` 更换配置，通过 `--workspace-dir` 指定新工作区。 +默认工作区为 `benchmark/beam/workspaces/home-100K`。 +代码从 `source_ref: HEAD` 对应的 Git 提交导出，不包含未提交修改。 +准备失败会回滚；成功后禁止再次准备同一工作区。 +Python 中也可调用 `await benchmark.beam.run_home.run(workspace_dir=...)`。 + +顶层 `sampling_policy` 配置采样默认值：`max_cases`、`max_queries`（null 表示不限）、 +`temperature`、`ewma_alpha` 和 `uniform_exploration`。入口校验后将其传给宿主 +`home.evaluation` Component，仍只准备、不执行评测；每次评测分别控制 `sampling` +和 `record_history`。 + +顶层 `validation_cycle_time_budget_minutes` 配置宿主 optimizer 两次已完成正式评测之间的 +建议用时，默认 150 分钟；可改为其他正数，或设为 null 关闭时间提醒。 + +顶层 `max_evaluations` 配置 optimizer 的正式评测次数预算，必须为正整数，默认 10。 +首次成功启动 sandbox 时扣一次，之后失败或取消也不退还；初始 baseline 和 debug 不计入。 + +顶层 `memory_limit_mib` 配置优化、诊断、case study 和评估审查的宿主 agent 沙箱内存限制。 +在 runner YAML 中设置 `memory_limit_mib: 0` 可关闭 Linux cgroup 内存限制； +省略或设为 null 时保留平台默认值（Linux 为 4096 MiB），macOS 不执行内存限制。 + +准备完成后，用同一份配置进行本地一次性优化： + +```bash +python benchmark/beam/run_home.py --optimize +``` + +若准备时指定了 `--config` 或 `--workspace-dir`，优化时传入相同参数。 +可用 `--instruction "优化检索质量"` 指定目标。该入口将配置中的两个预算和采样策略 +应用到实际 `home_optimize` 调用，不重复准备；Python 可调用同模块的 `await optimize(...)`。 +不加 `--optimize` 时仍只准备。 + +默认选择 BEAM **100K 全部 case**。 + + +## 独立测试集评测 + +默认在独立工作区评测 **100K 全部 case**，与旧独立评测入口一致，不代表与训练集互斥的留出集。 +可在 [config_home_eval.yaml](config_home_eval.yaml) 更换 `variant`，或把 `all_cases: true` +替换为 `case_ids: [...]`，指定测试数据。 + +入口从已有 Home 优化工作区导出指定候选提交，接受完整 SHA 或无歧义的十六进制短 SHA， +并复制该实验已准备的固定 runtime。无需原始 ReMe 源码仓库，也不会切换或修改优化工作区。 +配置内路径均相对仓库根目录。 + +先准备测试工作区，再分别在两个终端启动服务和调用评测（替换 `CANDIDATE_SHA`）： + +```bash +python benchmark/beam/run_home_eval.py --commit CANDIDATE_SHA --prepare-only +# 终端 1： +reme start workspace_dir=benchmark/beam/workspaces/home-test-100K 'plugins=["home","beam-judge"]' +# 终端 2： +reme home_evaluate workspace_dir=benchmark/beam/workspaces/home-test-100K record_history=false +``` + +### 本地一次性测试评测 + +```bash +python benchmark/beam/run_home_eval.py --commit CANDIDATE_SHA +# 可选：--config /path/to/test.yaml --source-workspace /path/to/search --test-workspace /path/to/test +``` + +Python 可调用 `await benchmark.beam.run_home_eval.run(commit="CANDIDATE_SHA")`。 +入口完整重建记忆并对所选 case 的全部 query 评分，关闭采样与优化历史记录，使用现有 Docker +评测环境和宿主 judge 配置；实际评测需要模型凭证。结果保存在测试工作区的 `home/evaluations/`， +manifest 的 `test_evaluation` 记录来源候选，导出后的本地提交另记为 `evaluation_commit_sha`。 +相同请求可复用未改动的准备结果并新增一次评测；更换候选或数据选择时需使用新测试工作区。 +测试工作区与来源工作区必须互不包含。 diff --git a/benchmark/beam/config_home.yaml b/benchmark/beam/config_home.yaml new file mode 100644 index 00000000..10163110 --- /dev/null +++ b/benchmark/beam/config_home.yaml @@ -0,0 +1,31 @@ +# Preparation by default; --optimize applies these settings to the prepared workspace. +# Paths are relative to the ReMe repository root, not the cwd. +workspace_dir: benchmark/beam/workspaces/home-100K +# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug. +max_evaluations: 10 +# Host agent sandbox memory limit in MiB; null keeps platform defaults, 0 disables the Linux cgroup limit. +memory_limit_mib: 0 +# Target minutes between completed formal optimizer evaluations; null disables timed reminders. +validation_cycle_time_budget_minutes: 150 +# Policy defaults for Home evaluation; each call still controls sampling=true/false. +sampling_policy: + max_cases: 5 + max_queries: null + temperature: 1.0 + ewma_alpha: 0.3 + uniform_exploration: 0.2 +preparation: + benchmark: + name: beam + source: benchmark/beam/dataset/BEAM + variant: 100K + all_cases: true + runtime: + source_repo: . + source_ref: HEAD # Committed source only; resolved and saved as a full SHA. + config: benchmark.yaml + plugins: [beam] + debug: + enabled: true + sessions_per_case: 3 + messages_per_session: 6 diff --git a/benchmark/beam/config_home_eval.yaml b/benchmark/beam/config_home_eval.yaml new file mode 100644 index 00000000..5a43a8bf --- /dev/null +++ b/benchmark/beam/config_home_eval.yaml @@ -0,0 +1,13 @@ +# Independent full-dataset evaluation; paths are relative to the ReMe repository. +source_workspace: benchmark/beam/workspaces/home-100K +test_workspace: benchmark/beam/workspaces/home-test-100K +code_commit: "" # Required: set a candidate SHA/prefix, or pass --commit. +benchmark: + name: beam + source: benchmark/beam/dataset/BEAM + variant: 100K + all_cases: true # Full 100K evaluation; not a disjoint held-out split. +evaluation: + concurrency: 4 + timeout_seconds: 7200 + record_history: false diff --git a/benchmark/beam/run_home.py b/benchmark/beam/run_home.py new file mode 100644 index 00000000..c219f591 --- /dev/null +++ b/benchmark/beam/run_home.py @@ -0,0 +1,173 @@ +"""Prepare the BEAM Home workspace, or optimize it with --optimize after preparation.""" + +import argparse +import asyncio +import json +import sys +from pathlib import Path + +from pydantic import BaseModel, ConfigDict, Field +import yaml + +# Support direct execution from any working directory as well as module imports. +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +# pylint: disable=wrong-import-position +from reme_home.preparation.spec import PreparationSpec # noqa: E402 +from reme_home.judge import JUDGE_PLUGINS # noqa: E402 +from reme_home.sampling import SamplingPolicy # noqa: E402 + +from reme.application import Application # noqa: E402 +from reme.utils.env_utils import load_env # noqa: E402 + +# pylint: enable=wrong-import-position + +DEFAULT_CONFIG = Path(__file__).with_name("config_home.yaml") + + +class RunHomeConfig(BaseModel): + """Runner settings kept separate from Home's public preparation contract.""" + + model_config = ConfigDict(extra="forbid", strict=True) + workspace_dir: str = Field(min_length=1) + preparation: dict + sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy) + validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False) + max_evaluations: int = Field(default=10, ge=1) + memory_limit_mib: int | None = Field(default=None, ge=0) + + +def project_path(value: str, project_root: Path) -> Path: + """Use repository-relative paths consistently regardless of the caller's cwd.""" + if value.startswith("~"): + raise ValueError("Use absolute or project-relative paths, not home-relative paths") + return (project_root / value).resolve() + + +def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT): + """Resolve source paths and validate explicit Home preparation parameters.""" + raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) + config = RunHomeConfig.model_validate(raw) + config.preparation.setdefault("candidate", {"paths": ["plugins/beam/src"]}) + spec = PreparationSpec.model_validate(config.preparation) + spec.benchmark.source = str(project_path(spec.benchmark.source, project_root)) + if spec.benchmark.evidence_source is not None: + spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root)) + spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root)) + workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root) + return workspace, spec.model_dump(mode="json"), config + + +def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT): + """Return resolved preparation inputs, preserving the preparation-only caller contract.""" + workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root) + return workspace, spec + + +async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, instruction=None): + """Apply the same runner configuration to each preparation or optimization lifecycle.""" + workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir) + load_env(PROJECT_ROOT / ".env", override=False) + memory_options = {} + jobs = {} + if config.memory_limit_mib is not None: + memory_options = { + "candidate_gate_memory_limit_mib": config.memory_limit_mib, + "construction_equivalence_memory_limit_mib": config.memory_limit_mib, + } + for name, backend, wrapper in ( + ("home_optimize", "home_optimize_step", "home_optimizer"), + ("home_diagnose", "home_diagnose_step", "home_diagnostic"), + ("home_case_study", "home_case_study_step", "home_case_study"), + ): + jobs[name] = { + "steps": [ + { + "backend": backend, + "agent_wrapper": wrapper, + "memory_limit_mib": config.memory_limit_mib, + **({"concurrency": 4, "output_token_limit": 2000} if name == "home_case_study" else {}), + }, + ], + } + app = Application( + workspace_dir=str(workspace), + plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]], + components={ + "home.evaluation": { + "default": {"sampling_policy": config.sampling_policy.model_dump(mode="json"), **memory_options}, + }, + "agent_wrapper": { + "home_optimizer": { + "validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes, + "max_evaluations": config.max_evaluations, + }, + }, + }, + jobs=jobs, + service={"backend": "cli"}, + enable_logo=False, + log_to_file=False, + ) + try: + await app.start() + if optimize_workspace: + arguments = {} if instruction is None else {"instruction": instruction} + response = await app.run_job("home_optimize", **arguments) + else: + response = await app.run_job("home_prepare", spec=spec) + if not response.success: + raise RuntimeError(response.answer) + return {"workspace_dir": str(workspace), **response.answer} + finally: + await app.close() + + +async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict: + """Prepare once without running evaluations or optimization.""" + return await _run(config_path, workspace_dir=workspace_dir) + + +async def optimize(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir=None, instruction=None) -> dict: + """Optimize an already prepared workspace using this config's actual model budgets.""" + return await _run(config_path, workspace_dir=workspace_dir, optimize_workspace=True, instruction=instruction) + + +async def run(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir: str | None = None) -> dict: + """Prepare through the same Home Job as the CLI, then close the application.""" + return await prepare(config_path, workspace_dir=workspace_dir) + + +def main(default_config: Path = DEFAULT_CONFIG) -> None: + """Run preparation or explicit local one-shot optimization with the benchmark config.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=default_config, help="Home runner YAML configuration") + parser.add_argument("--workspace-dir", help="Override workspace directory (absolute or project-relative)") + parser.add_argument("--optimize", action="store_true", help="Optimize an already prepared workspace") + parser.add_argument("--instruction", help="Optimization focus; requires --optimize") + args = parser.parse_args() + if args.instruction is not None and not args.optimize: + parser.error("--instruction requires --optimize") + if args.optimize: + result = asyncio.run(optimize(args.config, workspace_dir=args.workspace_dir, instruction=args.instruction)) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return + result = asyncio.run(prepare(args.config, workspace_dir=args.workspace_dir)) + print( + json.dumps( + { + "workspace_dir": result["workspace_dir"], + "case_count": result["dataset"]["case_count"], + "query_count": result["dataset"]["query_count"], + "source_commit": result["code"]["source_commit"], + "initial_commit": result["code"]["initial_commit"], + }, + indent=2, + ), + ) + + +if __name__ == "__main__": + main() diff --git a/benchmark/beam/run_home_eval.py b/benchmark/beam/run_home_eval.py new file mode 100644 index 00000000..13513d37 --- /dev/null +++ b/benchmark/beam/run_home_eval.py @@ -0,0 +1,226 @@ +"""Evaluate a BEAM Home candidate in an independent test workspace.""" + +import argparse +import asyncio +import json +import sys +from pathlib import Path + +from pydantic import Field, model_validator +import yaml + +# Support direct execution from any working directory as well as module imports. +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +# pylint: disable=wrong-import-position +from reme_home.code.repository import resolve_revision, validate_repository # noqa: E402 +from reme_home.evaluation.artifacts import contained, read_json # noqa: E402 +from reme_home.evaluation.component import blocking # noqa: E402 +from reme_home.evaluation.inputs import EvaluationSpec, load_inputs # noqa: E402 +from reme_home.judge import JUDGE_PLUGINS # noqa: E402 +from reme_home.preparation.code import tree_fingerprint # noqa: E402 +from reme_home.preparation.data.models import fingerprint # noqa: E402 +from reme_home.preparation.evaluation_source import EvaluationSource # noqa: E402 +from reme_home.preparation.spec import BenchmarkSpec, DebugSpec, SpecModel # noqa: E402 + +from reme.application import Application # noqa: E402 +from reme.utils.env_utils import load_env # noqa: E402 + +# pylint: enable=wrong-import-position + +DEFAULT_CONFIG = Path(__file__).with_name("config_home_eval.yaml") + + +class TestRunConfig(SpecModel): + """Explicit dataset selection and execution options for one pinned candidate.""" + + source_workspace: str = Field(min_length=1) + test_workspace: str = Field(min_length=1) + code_commit: str = Field(min_length=4) + benchmark: BenchmarkSpec + exclude_training_cases: bool = False + evaluation: EvaluationSpec = Field(default_factory=lambda: EvaluationSpec(record_history=False)) + + @model_validator(mode="after") + def validate_test_mode(self): + """Test runs score every prepared query and never consume optimizer budgets.""" + options = self.evaluation + if any( + ( + options.debug, + options.sampling, + options.record_history, + options.max_evaluations is not None, + options.sampling_policy is not None, + options.case_ids is not None, + options.query_ids is not None, + options.reuse_construction_from is not None, + ), + ): + raise ValueError( + "Test evaluation requires full scoring with record_history=false; select cases in benchmark", + ) + if self.exclude_training_cases and (self.benchmark.name != "longmemeval" or not self.benchmark.all_cases): + raise ValueError("exclude_training_cases requires LongMemEval with all_cases=true") + return self + + +def project_path(value: str, project_root: Path) -> Path: + """Resolve runner paths independently of the launching working directory.""" + if value.startswith("~"): + raise ValueError("Use absolute or project-relative paths, not home-relative paths") + return (project_root / value).resolve() + + +def load_request(config_path, *, commit=None, source_workspace=None, test_workspace=None, project_root=PROJECT_ROOT): + """Validate all source inputs before starting an application or writing test data.""" + raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) + for key, value in ( + ("code_commit", commit), + ("source_workspace", source_workspace), + ("test_workspace", test_workspace), + ): + if value is not None: + raw[key] = value + # Execution settings can be partial while retaining the test-only history default. + raw["evaluation"] = {"record_history": False, **raw.get("evaluation", {})} + config = TestRunConfig.model_validate(raw) + source_path = project_path(config.source_workspace, project_root) + target = project_path(config.test_workspace, project_root) + if target.is_relative_to(source_path) or source_path.is_relative_to(target): + raise ValueError("Test and source workspaces must be separate, non-nested directories") + if target.exists() and ( + not target.is_dir() or (any(target.iterdir()) and not (target / "home/metadata/workspace.json").is_file()) + ): + raise ValueError("Use an empty test workspace or a compatible prepared Home test workspace") + source = EvaluationSource.load(source_path, config.code_commit) + benchmark = config.benchmark.model_copy(deep=True) + if benchmark.name != source.spec.benchmark.name: + raise ValueError("Test benchmark must match the source experiment") + benchmark.source = str(project_path(benchmark.source, project_root).resolve(strict=True)) + if benchmark.evidence_source is not None: + benchmark.evidence_source = str(project_path(benchmark.evidence_source, project_root).resolve(strict=True)) + if config.exclude_training_cases: + items = json.loads(Path(benchmark.source).read_text(encoding="utf-8")) + ids = [str(item["question_id"]) for item in items] + if len(ids) != len(set(ids)): + raise ValueError("Duplicate LongMemEval question IDs") + training = set(source.record["dataset"]["case_ids"]) + if training - set(ids): + raise ValueError("Test dataset is missing training IDs; provide the original full dataset") + benchmark.case_ids = [case_id for case_id in ids if case_id not in training] + benchmark.all_cases = False + benchmark = BenchmarkSpec.model_validate(benchmark.model_dump()) + spec = source.spec.model_copy(deep=True) + spec.benchmark = benchmark + spec.debug = DebugSpec(enabled=False) + return config, target, source, spec + + +def existing_preparation(home, source, spec): + """Reopen only the same request with unchanged code, runtime and normalized data.""" + marker = contained(home, "metadata/preparation.json") + if not marker.exists(): + return None + record = read_json(marker) + expected = {**source.provenance, "request_fingerprint": fingerprint(spec.model_dump(mode="json"))} + if record.get("test_evaluation") != expected: + raise ValueError("Test workspace belongs to a different candidate or dataset; use a new workspace") + domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8")) + if fingerprint(domain) != record["config_fingerprint"]: + raise ValueError("Test preparation configuration has changed") + repository, _ = validate_repository(home, record) + if ( + resolve_revision(repository, "HEAD") != record["code"]["initial_commit"] + or tree_fingerprint(repository) != record["code"]["candidate_fingerprint"] + or tree_fingerprint(contained(home, "code/runtime", directory=True)) != source.provenance["runtime_fingerprint"] + ): + raise ValueError("Prepared test code or runtime has changed") + load_inputs(home, EvaluationSpec(record_history=False)) + return record + + +async def run_evaluation( + config_path, + *, + commit=None, + source_workspace=None, + test_workspace=None, + prepare_only=False, + expected_benchmark=None, +): + """Prepare transactionally, then score through the normal application lifecycle.""" + config, target, source, spec = await blocking( + load_request, + config_path, + commit=commit, + source_workspace=source_workspace, + test_workspace=test_workspace, + ) + if expected_benchmark is not None and config.benchmark.name != expected_benchmark: + raise ValueError(f"This entry point requires benchmark.name={expected_benchmark}") + load_env(PROJECT_ROOT / ".env", override=False) + app = Application( + workspace_dir=str(target), + plugins=["home", JUDGE_PLUGINS[config.benchmark.name]], + service={"backend": "cli"}, + enable_logo=False, + log_to_file=False, + ) + try: + await app.start() + home = target / "home" + prepared = await blocking(existing_preparation, home, source, spec) + if prepared is None: + preparation = app.context.components["home.preparation"]["default"] + prepared = await preparation.prepare(spec=spec.model_dump(mode="json"), code_source=source) + result = { + "test_workspace": str(target), + **source.provenance, + "evaluation_commit_sha": prepared["code"]["initial_commit"], + "case_count": prepared["dataset"]["case_count"], + "query_count": prepared["dataset"]["query_count"], + } + if not prepare_only: + response = await app.run_job( + "home_evaluate", + **config.evaluation.model_dump(mode="json", exclude_unset=True), + ) + if not response.success: + raise RuntimeError(response.answer) + result["evaluation"] = response.answer + return result + finally: + await app.close() + + +async def run(config_path=DEFAULT_CONFIG, **kwargs): + """Score the selected commit in its own Home workspace.""" + return await run_evaluation(config_path, expected_benchmark="beam", **kwargs) + + +def main(): + """Explicit local one-shot entry point, with preparation-only service setup.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + parser.add_argument("--commit", help="Candidate commit SHA or unambiguous hexadecimal prefix") + parser.add_argument("--source-workspace", help="Prepared optimization workspace") + parser.add_argument("--test-workspace", help="Independent test output workspace") + parser.add_argument("--prepare-only", action="store_true", help="Prepare for a later home_evaluate service call") + args = parser.parse_args() + result = asyncio.run( + run( + args.config, + commit=args.commit, + source_workspace=args.source_workspace, + test_workspace=args.test_workspace, + prepare_only=args.prepare_only, + ), + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/benchmark/longmemeval/README.md b/benchmark/longmemeval/README.md index 7ddda3cb..58b4ff88 100644 --- a/benchmark/longmemeval/README.md +++ b/benchmark/longmemeval/README.md @@ -112,3 +112,111 @@ agentscope==2.0.4.post1, conda reme env, 32 workers, eval-only (reusing prebuilt | single-session-user | 0.986 | 27,433 | 359 | 27,792 | 2.60 | | temporal-reasoning | 0.902 | 62,674 | 985 | 63,659 | 4.97 | | **OVERALL** | **0.894** | **43,448** | **876** | **44,324** | **3.69** | + + +## Prepare a Home experiment + +`run_home.py` prepares the fixed **group_1–group_5** training split (250 cases), the workspace, and committed candidate +code. It exits after preparation; it does not evaluate cases or start optimization. +Run from the repository root after obtaining the dataset described above: + +```bash +reme plugins install plugins/home +python benchmark/longmemeval/run_home.py +# Optional separate workspace (a successful preparation cannot be repeated): +python benchmark/longmemeval/run_home.py --workspace-dir /path/to/new-workspace +``` + +[config_home.yaml](config_home.yaml) controls the preparation. Its paths are +repository-relative regardless of the current working directory; `--config` +accepts an alternative runner YAML file. The default workspace is +`benchmark/longmemeval/workspaces/home-groups-1-5`. The evaluated runtime loads `lme`; +the host application loads `home`. `source_ref: HEAD` exports committed source +only, excluding uncommitted modifications. Preparation failure rolls back its +outputs; after success, another call fails instead of resetting the experiment. + +The top-level `sampling_policy` configures Home evaluation defaults: `max_cases`, +`max_queries` (null means unlimited), `temperature`, `ewma_alpha` and +`uniform_exploration`. The runner validates these and passes them to the host +`home.evaluation` Component. It still only prepares; it does not run evaluations. +Each evaluation controls `sampling` and `record_history` separately. + +The top-level `validation_cycle_time_budget_minutes` configures the host optimizer's +advisory time between completed formal evaluations. It defaults to 150 minutes; +set a positive number to change it or null to disable timed reminders. + +The top-level `max_evaluations` sets the optimizer's formal-evaluation budget +(positive integer, default 10). A slot is consumed at first sandbox startup, +including later failures or cancellations; initial baseline and debug are excluded. + +The top-level `memory_limit_mib` sets the host agent sandbox memory limit for +optimization, diagnosis, case studies and evaluation reviews. Set `memory_limit_mib: 0` +in the runner YAML to disable the Linux cgroup limit. Null or omission keeps platform +defaults (4096 MiB on Linux); macOS does not enforce memory limits. + +After preparation, run local one-shot optimization with the same configuration: + +```bash +python benchmark/longmemeval/run_home.py --optimize +# Use the same --config and --workspace-dir overrides as preparation, if supplied. +python benchmark/longmemeval/run_home.py --optimize --instruction "Improve retrieval quality" +``` + +This starts a fresh application, applies both budgets and the sampling policy, +runs `home_optimize`, and closes the application. It does not repeat preparation. +Python callers can use `await optimize(config_path, workspace_dir=...)` from the +same module. The default `run()` and CLI behavior remain preparation only. + +Python callers can use the same lifecycle: + +```python +from benchmark.longmemeval.run_home import run + +result = await run(workspace_dir="/path/to/new-workspace") +``` + +The 250 `benchmark.case_ids` in [config_home.yaml](config_home.yaml) are copied +in order from groups 1–5 of `dataset/groups.json` at `dev_meta` commit +`b53048e44833937df80a52fccceea78d054790e8`. The original splitter used seed 42 +and type-stratified allocation into ten 50-case groups; groups 6–10 remain held +out. The runner reads the IDs directly from YAML without loading group files. +[home_evidence_sessions.jsonl](home_evidence_sessions.jsonl) is a byte-for-byte +copy of that revision's `dataset/longmemeval_evidence_sessions.jsonl`; +reviewed evidence-session annotations are preserved during normalization. + + +## Evaluate a candidate on an independent test set + +The default excludes the source experiment's actual training IDs from the full dataset. With the default +250-case training workspace this selects held-out groups 6–10. For an explicit selection, set +`exclude_training_cases: false` and replace `all_cases: true` with `case_ids: [...]` in +[config_home_eval.yaml](config_home_eval.yaml). An empty held-out selection fails before preparation. + +The entry point imports a full candidate SHA or unambiguous hexadecimal prefix from an existing Home +optimization workspace and copies its prepared fixed runtime. It does not require the original ReMe source +repository or change the optimizer's checkout. Paths in the configuration are repository-relative. + +Prepare first, then use the service from two terminals (replace `CANDIDATE_SHA`): + +```bash +python benchmark/longmemeval/run_home_eval.py --commit CANDIDATE_SHA --prepare-only +# Terminal 1: +reme start workspace_dir=benchmark/longmemeval/workspaces/home-test-held-out 'plugins=["home","lme-judge"]' +# Terminal 2: +reme home_evaluate workspace_dir=benchmark/longmemeval/workspaces/home-test-held-out record_history=false +``` + +### Local one-shot test evaluation + +```bash +python benchmark/longmemeval/run_home_eval.py --commit CANDIDATE_SHA +# Optional overrides: --config /path/to/test.yaml --source-workspace /path/to/search --test-workspace /path/to/test +``` + +Python callers can use `await benchmark.longmemeval.run_home_eval.run(commit="CANDIDATE_SHA")`. +The runner rebuilds construction and scores every selected query with sampling and history recording disabled. +It uses the existing Docker evaluation runtime and host judge configuration; model credentials are required +for evaluation. Results stay in the test workspace's `home/evaluations/`, with original candidate provenance +in each manifest's `test_evaluation`. The exported snapshot has its own local `evaluation_commit_sha`. +Rerunning the same request reopens unchanged prepared inputs and creates another evaluation result. Use a +new test workspace for another candidate or selection. Source and test workspaces must not overlap. diff --git a/benchmark/longmemeval/README_ZH.md b/benchmark/longmemeval/README_ZH.md index cd17ee95..eca0693e 100644 --- a/benchmark/longmemeval/README_ZH.md +++ b/benchmark/longmemeval/README_ZH.md @@ -101,3 +101,89 @@ agentscope==2.0.4.post1, conda reme env, 32 workers, eval-only（复用预构建 | single-session-user | 0.986 | 27,433 | 359 | 27,792 | 2.60 | | temporal-reasoning | 0.902 | 62,674 | 985 | 63,659 | 4.97 | | **OVERALL** | **0.894** | **43,448** | **876** | **44,324** | **3.69** | + + +## 准备 Home 实验 + +安装 Home 插件后，在仓库根目录执行： + +```bash +reme plugins install plugins/home +python benchmark/longmemeval/run_home.py +``` + +该入口只创建工作区、准备数据和代码，不运行评测或优化。 +参数见 [config_home.yaml](config_home.yaml)，路径相对仓库根目录； +可通过 `--config` 更换配置，通过 `--workspace-dir` 指定新工作区。 +默认工作区为 `benchmark/longmemeval/workspaces/home-groups-1-5`。 +代码从 `source_ref: HEAD` 对应的 Git 提交导出，不包含未提交修改。 +准备失败会回滚；成功后禁止再次准备同一工作区。 +Python 中也可调用 `await benchmark.longmemeval.run_home.run(workspace_dir=...)`。 + +顶层 `sampling_policy` 配置采样默认值：`max_cases`、`max_queries`（null 表示不限）、 +`temperature`、`ewma_alpha` 和 `uniform_exploration`。入口校验后将其传给宿主 +`home.evaluation` Component，仍只准备、不执行评测；每次评测分别控制 `sampling` +和 `record_history`。 + +顶层 `validation_cycle_time_budget_minutes` 配置宿主 optimizer 两次已完成正式评测之间的 +建议用时，默认 150 分钟；可改为其他正数，或设为 null 关闭时间提醒。 + +顶层 `max_evaluations` 配置 optimizer 的正式评测次数预算，必须为正整数，默认 10。 +首次成功启动 sandbox 时扣一次，之后失败或取消也不退还；初始 baseline 和 debug 不计入。 + +顶层 `memory_limit_mib` 配置优化、诊断、case study 和评估审查的宿主 agent 沙箱内存限制。 +在 runner YAML 中设置 `memory_limit_mib: 0` 可关闭 Linux cgroup 内存限制； +省略或设为 null 时保留平台默认值（Linux 为 4096 MiB），macOS 不执行内存限制。 + +准备完成后，用同一份配置进行本地一次性优化： + +```bash +python benchmark/longmemeval/run_home.py --optimize +``` + +若准备时指定了 `--config` 或 `--workspace-dir`，优化时传入相同参数。 +可用 `--instruction "优化检索质量"` 指定目标。该入口将配置中的两个预算和采样策略 +应用到实际 `home_optimize` 调用，不重复准备；Python 可调用同模块的 `await optimize(...)`。 +不加 `--optimize` 时仍只准备。 + +默认使用 `dev_meta` 已保存的 **group_1–group_5，共 250 个 case**， +ID 已按原顺序直接写入 [config_home.yaml](config_home.yaml) 的 `benchmark.case_ids`， +运行时无需读取分组文件，group_6–group_10 保持为留出集。 +独立审阅的证据标注保存在 +[home_evidence_sessions.jsonl](home_evidence_sessions.jsonl)，来源见英文说明。 + + +## 独立测试集评测 + +默认从完整数据中排除来源工作区实际使用的训练 ID；使用默认 250 个训练 case 时，测试集为 +**group_6–group_10，共 250 个 case**。若要显式选取测试 case，可在 +[config_home_eval.yaml](config_home_eval.yaml) 设置 `exclude_training_cases: false`， +并把 `all_cases: true` 替换为 `case_ids: [...]`。留出集为空时会直接报错。 + +入口从已有 Home 优化工作区导出指定候选提交，接受完整 SHA 或无歧义的十六进制短 SHA， +并复制该实验已准备的固定 runtime。无需原始 ReMe 源码仓库，也不会切换或修改优化工作区。 +配置内路径均相对仓库根目录。 + +先准备测试工作区，再分别在两个终端启动服务和调用评测（替换 `CANDIDATE_SHA`）： + +```bash +python benchmark/longmemeval/run_home_eval.py --commit CANDIDATE_SHA --prepare-only +# 终端 1： +reme start workspace_dir=benchmark/longmemeval/workspaces/home-test-held-out 'plugins=["home","lme-judge"]' +# 终端 2： +reme home_evaluate workspace_dir=benchmark/longmemeval/workspaces/home-test-held-out record_history=false +``` + +### 本地一次性测试评测 + +```bash +python benchmark/longmemeval/run_home_eval.py --commit CANDIDATE_SHA +# 可选：--config /path/to/test.yaml --source-workspace /path/to/search --test-workspace /path/to/test +``` + +Python 可调用 `await benchmark.longmemeval.run_home_eval.run(commit="CANDIDATE_SHA")`。 +入口完整重建记忆并对所选 case 的全部 query 评分，关闭采样与优化历史记录，使用现有 Docker +评测环境和宿主 judge 配置；实际评测需要模型凭证。结果保存在测试工作区的 `home/evaluations/`， +manifest 的 `test_evaluation` 记录来源候选，导出后的本地提交另记为 `evaluation_commit_sha`。 +相同请求可复用未改动的准备结果并新增一次评测；更换候选或数据选择时需使用新测试工作区。 +测试工作区与来源工作区必须互不包含。 diff --git a/benchmark/longmemeval/config_home.yaml b/benchmark/longmemeval/config_home.yaml new file mode 100644 index 00000000..2ebbf1c9 --- /dev/null +++ b/benchmark/longmemeval/config_home.yaml @@ -0,0 +1,288 @@ +# Preparation by default; --optimize applies these settings to the prepared workspace. +# Paths are relative to the ReMe repository root, not the cwd. +# Preserve dev_meta's fixed training split; groups 6-10 remain held out. +workspace_dir: benchmark/longmemeval/workspaces/home-groups-1-5 +# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug. +max_evaluations: 10 +# Host agent sandbox memory limit in MiB; null keeps platform defaults, 0 disables the Linux cgroup limit. +memory_limit_mib: 0 +# Policy defaults for Home evaluation; each call still controls sampling=true/false. +sampling_policy: + max_cases: 5 + max_queries: null + temperature: 1.0 + ewma_alpha: 0.3 + uniform_exploration: 0.2 +preparation: + benchmark: + name: longmemeval + source: benchmark/longmemeval/dataset/longmemeval_s_reme_cleaned.json + evidence_source: benchmark/longmemeval/home_evidence_sessions.jsonl + # dev_meta b53048e44833937df80a52fccceea78d054790e8: dataset/groups.json, groups 1-5. + case_ids: + # group_1 + - "gpt4_1e4a8aec" + - "e3038f8c" + - "gpt4_70e84552_abs" + - "edced276_abs" + - "c6853660" + - "gpt4_372c3eed" + - "gpt4_ec93e27f" + - "c8c3f81d" + - "dccbc061" + - "a06e4cfe" + - "a82c026e" + - "fca762bc" + - "603deb26" + - "bb7c3b45" + - "4baee567" + - "af082822" + - "1a1907b4" + - "gpt4_468eb064" + - "ef9cf60a" + - "gpt4_a2d1d1f6" + - "8fb83627" + - "1c0ddc50" + - "2ebe6c92" + - "e25c3b8d" + - "6aeb4375" + - "bbf86515" + - "8e91e7d9" + - "f0e564bc" + - "eaca4986" + - "gpt4_4fc4f797" + - "gpt4_5501fe77" + - "7a8d0b71" + - "982b5123" + - "f4f1d8a4_abs" + - "e831120c" + - "0977f2af" + - "51c32626" + - "gpt4_ab202e7f" + - "eace081b" + - "d01c6aa8" + - "8aef76bc" + - "d682f1a2" + - "gpt4_468eb063" + - "c5e8278d" + - "ccb36322" + - "affe2881" + - "75f70248" + - "6e984302" + - "ed4ddc30" + - "a11281a2" + # group_2 + - "cc6d1ec1" + - "6613b389" + - "c4f10528" + - "1faac195" + - "3d86fd0a" + - "9d25d4e0" + - "b759caee" + - "gpt4_b4a80587" + - "b3c15d39" + - "370a8ff4" + - "bf659f65" + - "5a7937c8" + - "57f827a0" + - "545bd2b5" + - "a96c20ee" + - "caf03d32" + - "dad224aa" + - "50635ada" + - "21436231" + - "031748ae" + - "41275add" + - "51a45a95" + - "gpt4_4929293a" + - "3e321797" + - "0e5e2d1a" + - "b01defab" + - "gpt4_4ef30696" + - "91b15a6e" + - "d6233ab6" + - "c18a7dc8" + - "f0853d11" + - "031748ae_abs" + - "60d45044" + - "bcbe585f" + - "41698283" + - "0ddfec37" + - "46a3abf7" + - "dd2973ad" + - "129d1232" + - "60159905" + - "2b8f3739" + - "c8090214_abs" + - "8464fc84" + - "gpt4_93f6379c" + - "gpt4_2c50253f" + - "gpt4_18c2b244" + - "bc8a6e93_abs" + - "09ba9854" + - "2133c1b5_abs" + - "gpt4_4edbafa2" + # group_3 + - "7e00a6cb" + - "0e4e4c46" + - "gpt4_0b2f1d21" + - "55241a1f" + - "77eafa52" + - "dc439ea3" + - "86f00804" + - "830ce83f" + - "cc5ded98" + - "2311e44b_abs" + - "gpt4_d12ceb0e" + - "71a3fd6b" + - "07741c45" + - "e6041065" + - "gpt4_68e94288" + - "32260d93" + - "db467c8c" + - "4adc0475" + - "6cb6f249" + - "c9f37c46" + - "4388e9dd" + - "9ea5eabc" + - "gpt4_fe651585_abs" + - "9a707b81" + - "e9327a54" + - "eac54adc" + - "gpt4_7fce9456" + - "f9e8c073" + - "ba358f49_abs" + - "561fabcd" + - "42ec0761" + - "gpt4_4929293b" + - "c19f7a0b" + - "0bc8ad93" + - "e47becba" + - "15745da0" + - "2e6d26dc" + - "efc3f7c2" + - "37f165cf" + - "gpt4_f420262d" + - "66f24dbb" + - "gpt4_194be4b3" + - "gpt4_c27434e8_abs" + - "f4f1d8a4" + - "a89d7624" + - "19b5f2b3" + - "gpt4_8279ba02" + - "gpt4_74aed68e" + - "1da05512" + - "gpt4_f420262c" + # group_4 + - "gpt4_fa19884d" + - "6f9b354f" + - "a3045048" + - "81507db6" + - "61f8c8f8" + - "a346bb18" + - "87f22b4a" + - "gpt4_2ba83207" + - "01493427" + - "08e075c7" + - "09d032c9" + - "6a27ffc2" + - "edced276" + - "8550ddae" + - "e5ba910e_abs" + - "3f1e9474" + - "8a2466db" + - "gpt4_2f8be40d" + - "2698e78f" + - "gpt4_1a1dc16d" + - "6b168ec8" + - "d905b33f" + - "60472f9c" + - "6e984301" + - "gpt4_1e4a8aeb" + - "gpt4_98f46fc6" + - "e66b632c" + - "gpt4_1d4ab0c9" + - "gpt4_0a05b494" + - "72e3ee87" + - "16c90bf4" + - "5809eb10" + - "e8a79c70" + - "gpt4_213fd887" + - "0ddfec37_abs" + - "gpt4_cd90e484" + - "06f04340" + - "eeda8a6d" + - "9a707b82" + - "29f2956b" + - "09ba9854_abs" + - "gpt4_5dcc0aab" + - "gpt4_7bc6cf22" + - "5d3d2817" + - "4c36ccef" + - "e3fc4d6e" + - "aae3761f" + - "d596882b" + - "ce6d2d27" + - "ad7109d1" + # group_5 + - "76d63226" + - "18dcd5a5" + - "e48988bc" + - "099778bb" + - "2c63a862" + - "gpt4_7abb270c" + - "f523d9fe" + - "06878be2" + - "6456829e_abs" + - "faba32e5" + - "gpt4_15e38248" + - "gpt4_8e165409" + - "35a27287" + - "gpt4_59c863d7" + - "gpt4_59149c77" + - "8ebdbe50" + - "f685340e" + - "21d02d0d" + - "4fd1909e" + - "3249768e" + - "gpt4_f49edff3" + - "1d4e3b97" + - "1192316e" + - "e61a7584" + - "0100672e" + - "8cf51dda" + - "95bcc1c8" + - "71017277" + - "a3838d2b" + - "c4ea545c" + - "2318644b" + - "18bc8abd" + - "gpt4_61e13b3c" + - "gpt4_e05b82a6" + - "b46e15ed" + - "3a704032" + - "eac54add" + - "9ee3ecd6" + - "80ec1f4f" + - "gpt4_1916e0ea" + - "b6019101" + - "45dc21b6" + - "gpt4_c27434e8" + - "bc149d6b" + - "6222b6eb" + - "06db6396" + - "29f2956b_abs" + - "gpt4_5438fa52" + - "cf22b7bf" + - "853b0a1d" + runtime: + source_repo: . + source_ref: HEAD # Committed source only; resolved and saved as a full SHA. + config: benchmark.yaml + plugins: [lme] + candidate: + paths: [plugins/lme/src] + debug: + enabled: true + sessions_per_case: 3 + messages_per_session: 7 diff --git a/benchmark/longmemeval/config_home_eval.yaml b/benchmark/longmemeval/config_home_eval.yaml new file mode 100644 index 00000000..66cf7598 --- /dev/null +++ b/benchmark/longmemeval/config_home_eval.yaml @@ -0,0 +1,14 @@ +# Evaluate the complement of the source experiment's training IDs in the full dataset. +# With the default training workspace this selects held-out groups 6-10 (250 cases). +source_workspace: benchmark/longmemeval/workspaces/home-groups-1-5 +test_workspace: benchmark/longmemeval/workspaces/home-test-held-out +code_commit: "" # Required: set a candidate SHA/prefix, or pass --commit. +exclude_training_cases: true +benchmark: + name: longmemeval + source: benchmark/longmemeval/dataset/longmemeval_s_reme_cleaned.json + all_cases: true +evaluation: + concurrency: 4 + timeout_seconds: 7200 + record_history: false diff --git a/benchmark/longmemeval/home_evidence_sessions.jsonl b/benchmark/longmemeval/home_evidence_sessions.jsonl new file mode 100644 index 00000000..5af9c7fa --- /dev/null +++ b/benchmark/longmemeval/home_evidence_sessions.jsonl @@ -0,0 +1,500 @@ +{"case_id":"e47becba","evidence_sessions":["answer_280352e9"],"is_certain":true} +{"case_id":"118b2229","evidence_sessions":["answer_40a90d51"],"is_certain":true} +{"case_id":"51a45a95","evidence_sessions":["answer_d61669c7"],"is_certain":true} +{"case_id":"58bf7951","evidence_sessions":["answer_355c48bb"],"is_certain":true} +{"case_id":"1e043500","evidence_sessions":["answer_3e012175"],"is_certain":true} +{"case_id":"c5e8278d","evidence_sessions":["answer_f6168136"],"is_certain":true} +{"case_id":"6ade9755","evidence_sessions":["answer_9398da02"],"is_certain":true} +{"case_id":"6f9b354f","evidence_sessions":["answer_feb5200f"],"is_certain":true} +{"case_id":"58ef2f1c","evidence_sessions":["answer_59547700"],"is_certain":true} +{"case_id":"f8c5f88b","evidence_sessions":["answer_c3567066"],"is_certain":true} +{"case_id":"5d3d2817","evidence_sessions":["answer_235eb6fb"],"is_certain":true} +{"case_id":"7527f7e2","evidence_sessions":["answer_7cb94507"],"is_certain":true} +{"case_id":"c960da58","evidence_sessions":["answer_e05e4612"],"is_certain":true} +{"case_id":"3b6f954b","evidence_sessions":["answer_94030872"],"is_certain":true} +{"case_id":"726462e0","evidence_sessions":["answer_f38f679b"],"is_certain":true} +{"case_id":"94f70d80","evidence_sessions":["answer_c63c0458"],"is_certain":true} +{"case_id":"66f24dbb","evidence_sessions":["answer_fea2e4d3"],"is_certain":true} +{"case_id":"ad7109d1","evidence_sessions":["answer_679840f8"],"is_certain":true} +{"case_id":"af8d2e46","evidence_sessions":["answer_82a04f59"],"is_certain":true} +{"case_id":"dccbc061","evidence_sessions":["answer_8f276838"],"is_certain":true} +{"case_id":"c8c3f81d","evidence_sessions":["answer_761acef8"],"is_certain":true} +{"case_id":"8ebdbe50","evidence_sessions":["answer_8ad8a34f"],"is_certain":true} +{"case_id":"6b168ec8","evidence_sessions":["answer_e623ae87"],"is_certain":true} +{"case_id":"75499fd8","evidence_sessions":["answer_723bf11f"],"is_certain":true} +{"case_id":"21436231","evidence_sessions":["answer_1e6d4567"],"is_certain":true} +{"case_id":"95bcc1c8","evidence_sessions":["answer_cb742a61"],"is_certain":true} +{"case_id":"0862e8bf","evidence_sessions":["answer_c6fd8ebd"],"is_certain":true} +{"case_id":"853b0a1d","evidence_sessions":["answer_69811d4a"],"is_certain":true} +{"case_id":"a06e4cfe","evidence_sessions":["answer_6fe9fb49"],"is_certain":true} +{"case_id":"37d43f65","evidence_sessions":["answer_55161935"],"is_certain":true} +{"case_id":"b86304ba","evidence_sessions":["answer_645b0329"],"is_certain":true} +{"case_id":"d52b4f67","evidence_sessions":["answer_0df6aa4b"],"is_certain":true} +{"case_id":"25e5aa4f","evidence_sessions":["answer_986de8c3"],"is_certain":true} +{"case_id":"caf9ead2","evidence_sessions":["answer_0714183a"],"is_certain":true} +{"case_id":"8550ddae","evidence_sessions":["answer_c8354ae9"],"is_certain":true} +{"case_id":"60d45044","evidence_sessions":["answer_9cddca88"],"is_certain":true} +{"case_id":"3f1e9474","evidence_sessions":["answer_57fc1954"],"is_certain":true} +{"case_id":"86b68151","evidence_sessions":["answer_dc11c1eb"],"is_certain":true} +{"case_id":"577d4d32","evidence_sessions":["answer_0dd4d99a"],"is_certain":true} +{"case_id":"ec81a493","evidence_sessions":["answer_ed1982fc"],"is_certain":true} +{"case_id":"15745da0","evidence_sessions":["answer_586de428"],"is_certain":true} +{"case_id":"e01b8e2f","evidence_sessions":["answer_5ca6cd28"],"is_certain":true} +{"case_id":"bc8a6e93","evidence_sessions":["answer_e6143162"],"is_certain":true} +{"case_id":"ccb36322","evidence_sessions":["answer_f1fbb330"],"is_certain":true} +{"case_id":"001be529","evidence_sessions":["answer_530960c1"],"is_certain":true} +{"case_id":"b320f3f8","evidence_sessions":["answer_5cc9b056"],"is_certain":true} +{"case_id":"19b5f2b3","evidence_sessions":["answer_5ff494b9"],"is_certain":true} +{"case_id":"4fd1909e","evidence_sessions":["answer_2952aee4"],"is_certain":true} +{"case_id":"545bd2b5","evidence_sessions":["answer_47ffab4c"],"is_certain":true} +{"case_id":"8a137a7f","evidence_sessions":["answer_15d63a22"],"is_certain":true} +{"case_id":"76d63226","evidence_sessions":["answer_bbdc7b0a"],"is_certain":true} +{"case_id":"86f00804","evidence_sessions":["answer_96b8c9ee"],"is_certain":true} +{"case_id":"8e9d538c","evidence_sessions":["answer_7bdcbd23"],"is_certain":true} +{"case_id":"311778f1","evidence_sessions":["answer_e40b054e"],"is_certain":true} +{"case_id":"c19f7a0b","evidence_sessions":["answer_f442ccbe"],"is_certain":true} +{"case_id":"4100d0a0","evidence_sessions":["answer_83c13ff9"],"is_certain":true} +{"case_id":"29f2956b","evidence_sessions":["answer_7cc5362f"],"is_certain":true} +{"case_id":"1faac195","evidence_sessions":["answer_d01949bf"],"is_certain":true} +{"case_id":"faba32e5","evidence_sessions":["answer_39b12014"],"is_certain":true} +{"case_id":"f4f1d8a4","evidence_sessions":["answer_f5b33470"],"is_certain":true} +{"case_id":"c14c00dd","evidence_sessions":["answer_304511ce"],"is_certain":true} +{"case_id":"36580ce8","evidence_sessions":["answer_93e1bd22"],"is_certain":true} +{"case_id":"3d86fd0a","evidence_sessions":["answer_19c24c11"],"is_certain":true} +{"case_id":"a82c026e","evidence_sessions":["answer_787e6a6d"],"is_certain":true} +{"case_id":"0862e8bf_abs","evidence_sessions":["answer_c6fd8ebd_abs"],"is_certain":true} +{"case_id":"15745da0_abs","evidence_sessions":["answer_586de428_abs"],"is_certain":true} +{"case_id":"bc8a6e93_abs","evidence_sessions":["answer_e6143162_abs"],"is_certain":true} +{"case_id":"19b5f2b3_abs","evidence_sessions":["answer_5ff494b9_abs"],"is_certain":true} +{"case_id":"29f2956b_abs","evidence_sessions":["answer_7cc5362f_abs"],"is_certain":true} +{"case_id":"f4f1d8a4_abs","evidence_sessions":["answer_f5b33470_abs"],"is_certain":true} +{"case_id":"0a995998","evidence_sessions":["answer_afa9873b_2","answer_afa9873b_3"],"is_certain":true} +{"case_id":"6d550036","evidence_sessions":["answer_ec904b3c_1","answer_ec904b3c_4","answer_ec904b3c_3","answer_ec904b3c_2"],"is_certain":false} +{"case_id":"gpt4_59c863d7","evidence_sessions":["answer_593bdffd_4","answer_593bdffd_1","answer_593bdffd_3","answer_593bdffd_2"],"is_certain":true} +{"case_id":"b5ef892d","evidence_sessions":["answer_a8b4290f_3","answer_a8b4290f_1","answer_a8b4290f_2"],"is_certain":true} +{"case_id":"e831120c","evidence_sessions":["answer_86c505e7_1","answer_86c505e7_2"],"is_certain":true} +{"case_id":"3a704032","evidence_sessions":["answer_c2204106_2","answer_c2204106_3","answer_c2204106_1"],"is_certain":false} +{"case_id":"gpt4_d84a3211","evidence_sessions":["answer_2880eb6c_2","answer_2880eb6c_1"],"is_certain":true} +{"case_id":"aae3761f","evidence_sessions":["answer_526354c8_1","answer_526354c8_3","answer_526354c8_2"],"is_certain":true} +{"case_id":"gpt4_f2262a51","evidence_sessions":["answer_55a6940c_3","answer_55a6940c_1","answer_55a6940c_2"],"is_certain":true} +{"case_id":"dd2973ad","evidence_sessions":["answer_f9de4602_2","answer_f9de4602_1"],"is_certain":true} +{"case_id":"c4a1ceb8","evidence_sessions":["answer_56d02cab_3","answer_56d02cab_4","answer_56d02cab_2","answer_56d02cab_1"],"is_certain":false} +{"case_id":"gpt4_a56e767c","evidence_sessions":["answer_cf9e3940_2","answer_cf9e3940_1","answer_cf9e3940_3"],"is_certain":true} +{"case_id":"6cb6f249","evidence_sessions":["answer_a4204937_2","answer_a4204937_1"],"is_certain":true} +{"case_id":"46a3abf7","evidence_sessions":["answer_c65042d7_1","answer_c65042d7_2","answer_c65042d7_3"],"is_certain":true} +{"case_id":"36b9f61e","evidence_sessions":["answer_ef74281f_2","answer_ef74281f_1","answer_ef74281f_3"],"is_certain":true} +{"case_id":"28dc39ac","evidence_sessions":["answer_8d015d9d_3","answer_8d015d9d_5","answer_8d015d9d_2","answer_8d015d9d_4","answer_8d015d9d_1"],"is_certain":true} +{"case_id":"gpt4_2f8be40d","evidence_sessions":["answer_e7b0637e_2","answer_e7b0637e_1","answer_e7b0637e_3"],"is_certain":true} +{"case_id":"2e6d26dc","evidence_sessions":["answer_fa526fc0_4","answer_fa526fc0_3","answer_fa526fc0_1","answer_fa526fc0_2"],"is_certain":true} +{"case_id":"gpt4_15e38248","evidence_sessions":["answer_8858d9dc_3","answer_8858d9dc_1","answer_8858d9dc_4","answer_8858d9dc_2"],"is_certain":true} +{"case_id":"88432d0a","evidence_sessions":["answer_733e443a_3","answer_733e443a_4","answer_733e443a_2","answer_733e443a_1"],"is_certain":false} +{"case_id":"80ec1f4f","evidence_sessions":["answer_990c8992_2","answer_990c8992_1","answer_990c8992_3"],"is_certain":true} +{"case_id":"d23cf73b","evidence_sessions":["answer_5a0d28f8_4","answer_5a0d28f8_2","answer_5a0d28f8_3","answer_5a0d28f8_1"],"is_certain":true} +{"case_id":"gpt4_7fce9456","evidence_sessions":["answer_a679a86a_5","answer_a679a86a_4","answer_a679a86a_2","answer_a679a86a_3","answer_a679a86a_1"],"is_certain":true} +{"case_id":"d682f1a2","evidence_sessions":["answer_c008e5df_1","answer_c008e5df_2","answer_c008e5df_3"],"is_certain":true} +{"case_id":"7024f17c","evidence_sessions":["answer_a21f3697_1","answer_a21f3697_2"],"is_certain":true} +{"case_id":"gpt4_5501fe77","evidence_sessions":["answer_203bf3fa_1","answer_203bf3fa_3","answer_203bf3fa_2"],"is_certain":false} +{"case_id":"gpt4_2ba83207","evidence_sessions":["answer_6a3b5c13_3","answer_6a3b5c13_1","answer_6a3b5c13_2","answer_6a3b5c13_4"],"is_certain":true} +{"case_id":"2318644b","evidence_sessions":["answer_eaa8e3ef_1","answer_eaa8e3ef_2"],"is_certain":true} +{"case_id":"2ce6a0f2","evidence_sessions":["answer_901a6763_2","answer_901a6763_4","answer_901a6763_1","answer_901a6763_3"],"is_certain":true} +{"case_id":"gpt4_d12ceb0e","evidence_sessions":["answer_2504635e_3","answer_2504635e_2","answer_2504635e_1"],"is_certain":true} +{"case_id":"00ca467f","evidence_sessions":["answer_39900a0a_3","answer_39900a0a_2","answer_39900a0a_1"],"is_certain":true} +{"case_id":"b3c15d39","evidence_sessions":["answer_05d808e6_1","answer_05d808e6_2"],"is_certain":true} +{"case_id":"gpt4_31ff4165","evidence_sessions":["answer_02b63d04_1","answer_02b63d04_5","answer_02b63d04_2","answer_02b63d04_3","answer_02b63d04_4"],"is_certain":true} +{"case_id":"eeda8a6d","evidence_sessions":["answer_3e5fea0e_1","answer_3e5fea0e_2"],"is_certain":true} +{"case_id":"2788b940","evidence_sessions":["answer_8f6b938d_1","answer_8f6b938d_3","answer_8f6b938d_2"],"is_certain":true} +{"case_id":"60bf93ed","evidence_sessions":["answer_e0956e0a_1","answer_e0956e0a_2"],"is_certain":true} +{"case_id":"9d25d4e0","evidence_sessions":["answer_fcff2dc4_2","answer_fcff2dc4_1","answer_fcff2dc4_3"],"is_certain":true} +{"case_id":"129d1232","evidence_sessions":["answer_1de862d6_1","answer_1de862d6_3","answer_1de862d6_2"],"is_certain":true} +{"case_id":"60472f9c","evidence_sessions":["answer_e7fe8c8b_1","answer_e7fe8c8b_2","answer_e7fe8c8b_3"],"is_certain":true} +{"case_id":"gpt4_194be4b3","evidence_sessions":["answer_3826dc55_1","answer_3826dc55_3","answer_3826dc55_5","answer_3826dc55_2","answer_3826dc55_4"],"is_certain":true} +{"case_id":"a9f6b44c","evidence_sessions":["answer_cc021f81_2","answer_cc021f81_3","answer_cc021f81_1"],"is_certain":true} +{"case_id":"d851d5ba","evidence_sessions":["answer_5cdf9bd2_2","answer_5cdf9bd2_1","answer_5cdf9bd2_3","answer_5cdf9bd2_4"],"is_certain":false} +{"case_id":"5a7937c8","evidence_sessions":["answer_4cef8a3c_3","answer_4cef8a3c_1","answer_4cef8a3c_2"],"is_certain":true} +{"case_id":"gpt4_ab202e7f","evidence_sessions":["answer_728deb4d_5","answer_728deb4d_2","answer_728deb4d_3","answer_728deb4d_1","answer_728deb4d_4"],"is_certain":true} +{"case_id":"gpt4_e05b82a6","evidence_sessions":["answer_6350aa4f_1","answer_6350aa4f_2","answer_6350aa4f_3","answer_6350aa4f_4"],"is_certain":true} +{"case_id":"gpt4_731e37d7","evidence_sessions":["answer_826d51da_3","answer_826d51da_4","answer_826d51da_2","answer_826d51da_1"],"is_certain":true} +{"case_id":"edced276","evidence_sessions":["answer_60e8941a_2","answer_60e8941a_1"],"is_certain":true} +{"case_id":"10d9b85a","evidence_sessions":["answer_e0585cb5_2","answer_e0585cb5_1"],"is_certain":true} +{"case_id":"e3038f8c","evidence_sessions":["answer_b6018747_2","answer_b6018747_4","answer_b6018747_1","answer_b6018747_3"],"is_certain":true} +{"case_id":"2b8f3739","evidence_sessions":["answer_23759615_2","answer_23759615_3","answer_23759615_1"],"is_certain":true} +{"case_id":"1a8a66a6","evidence_sessions":["answer_2bd23659_3","answer_2bd23659_2"],"is_certain":true} +{"case_id":"c2ac3c61","evidence_sessions":["answer_923c0221_1","answer_923c0221_2"],"is_certain":true} +{"case_id":"bf659f65","evidence_sessions":["answer_7726e7e9_1","answer_7726e7e9_3","answer_7726e7e9_2"],"is_certain":false} +{"case_id":"gpt4_372c3eed","evidence_sessions":["answer_35c5419d_3","answer_35c5419d_2","answer_35c5419d_1"],"is_certain":false} +{"case_id":"gpt4_2f91af09","evidence_sessions":["answer_669318cf_2","answer_669318cf_1","answer_669318cf_3"],"is_certain":true} +{"case_id":"81507db6","evidence_sessions":["answer_da3c1266_4","answer_da3c1266_1","answer_da3c1266_5","answer_da3c1266_2"],"is_certain":true} +{"case_id":"88432d0a_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"80ec1f4f_abs","evidence_sessions":["answer_990c8992_abs_2","answer_990c8992_abs_3","answer_990c8992_abs_1"],"is_certain":true} +{"case_id":"eeda8a6d_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"60bf93ed_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"edced276_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_372c3eed_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"8a2466db","evidence_sessions":["answer_edb03329"],"is_certain":true} +{"case_id":"06878be2","evidence_sessions":["answer_555dfb94"],"is_certain":true} +{"case_id":"75832dbd","evidence_sessions":["answer_d87a6ef8"],"is_certain":true} +{"case_id":"0edc2aef","evidence_sessions":["answer_d586e9cd"],"is_certain":true} +{"case_id":"35a27287","evidence_sessions":["answer_9b182436"],"is_certain":true} +{"case_id":"32260d93","evidence_sessions":["answer_0250ae1c"],"is_certain":true} +{"case_id":"195a1a1b","evidence_sessions":["answer_6dc4305e"],"is_certain":true} +{"case_id":"afdc33df","evidence_sessions":["answer_8549e5e0"],"is_certain":true} +{"case_id":"caf03d32","evidence_sessions":["answer_2fc6aabb"],"is_certain":true} +{"case_id":"54026fce","evidence_sessions":["answer_f7b22c66"],"is_certain":true} +{"case_id":"06f04340","evidence_sessions":["answer_92d5f7cd"],"is_certain":true} +{"case_id":"6b7dfb22","evidence_sessions":["answer_f6502d0f"],"is_certain":true} +{"case_id":"1a1907b4","evidence_sessions":["answer_719502eb"],"is_certain":true} +{"case_id":"09d032c9","evidence_sessions":["answer_b10dce5e"],"is_certain":true} +{"case_id":"38146c39","evidence_sessions":["answer_772472c8"],"is_certain":true} +{"case_id":"d24813b1","evidence_sessions":["answer_7c0ade93"],"is_certain":true} +{"case_id":"57f827a0","evidence_sessions":["answer_1bde8d3b"],"is_certain":true} +{"case_id":"95228167","evidence_sessions":["answer_5e613445"],"is_certain":true} +{"case_id":"505af2f5","evidence_sessions":["answer_f3164f2c"],"is_certain":true} +{"case_id":"75f70248","evidence_sessions":["answer_8ee04a2e"],"is_certain":true} +{"case_id":"d6233ab6","evidence_sessions":["answer_b0fac439"],"is_certain":true} +{"case_id":"1da05512","evidence_sessions":["answer_4d3be2ab"],"is_certain":true} +{"case_id":"fca70973","evidence_sessions":["answer_a1e169b1"],"is_certain":true} +{"case_id":"b6025781","evidence_sessions":["answer_8414cc57"],"is_certain":true} +{"case_id":"a89d7624","evidence_sessions":["answer_8f15ac24"],"is_certain":true} +{"case_id":"b0479f84","evidence_sessions":["answer_30f63ddb"],"is_certain":true} +{"case_id":"1d4e3b97","evidence_sessions":["answer_e6b6353d"],"is_certain":true} +{"case_id":"07b6f563","evidence_sessions":["answer_d03098f9"],"is_certain":true} +{"case_id":"1c0ddc50","evidence_sessions":["answer_8da8c7e0"],"is_certain":true} +{"case_id":"0a34ad58","evidence_sessions":["answer_cebb7159"],"is_certain":true} +{"case_id":"d3ab962e","evidence_sessions":["answer_5237bb0b_2","answer_5237bb0b_1"],"is_certain":true} +{"case_id":"2311e44b","evidence_sessions":["answer_bf633415_2","answer_bf633415_1"],"is_certain":true} +{"case_id":"cc06de0d","evidence_sessions":["answer_4fb01417_2","answer_4fb01417_1"],"is_certain":true} +{"case_id":"a11281a2","evidence_sessions":["answer_c69ee1f9_2","answer_c69ee1f9_1"],"is_certain":true} +{"case_id":"4f54b7c9","evidence_sessions":["answer_50940cb7_2","answer_50940cb7_1"],"is_certain":true} +{"case_id":"85fa3a3f","evidence_sessions":["answer_dcb18b9b_2","answer_dcb18b9b_1"],"is_certain":true} +{"case_id":"9aaed6a3","evidence_sessions":[],"is_certain":true} +{"case_id":"1f2b8d4f","evidence_sessions":["answer_d8454588_1","answer_d8454588_2"],"is_certain":true} +{"case_id":"e6041065","evidence_sessions":["answer_4eb6d671_2","answer_4eb6d671_1"],"is_certain":true} +{"case_id":"51c32626","evidence_sessions":["answer_58820c75_1","answer_58820c75_2"],"is_certain":false} +{"case_id":"d905b33f","evidence_sessions":["answer_85a77c48_2","answer_85a77c48_1"],"is_certain":true} +{"case_id":"7405e8b1","evidence_sessions":["answer_80323f3f_1","answer_80323f3f_2"],"is_certain":true} +{"case_id":"f35224e0","evidence_sessions":["answer_e9bb9500_2","answer_e9bb9500_1"],"is_certain":true} +{"case_id":"6456829e","evidence_sessions":["answer_743f03a1_2","answer_743f03a1_1"],"is_certain":true} +{"case_id":"a4996e51","evidence_sessions":["answer_feb5f98a_1","answer_feb5f98a_2"],"is_certain":true} +{"case_id":"3c1045c8","evidence_sessions":["answer_c8cc60d6_2","answer_c8cc60d6_1"],"is_certain":true} +{"case_id":"60036106","evidence_sessions":["answer_e552e1f9_1","answer_e552e1f9_2"],"is_certain":true} +{"case_id":"681a1674","evidence_sessions":["answer_3be95d43_1","answer_3be95d43_2"],"is_certain":true} +{"case_id":"e25c3b8d","evidence_sessions":["answer_6702277b_1","answer_6702277b_2"],"is_certain":true} +{"case_id":"4adc0475","evidence_sessions":["answer_6efce493_1","answer_6efce493_2"],"is_certain":true} +{"case_id":"4bc144e2","evidence_sessions":["answer_9ef115d4_1","answer_9ef115d4_2"],"is_certain":true} +{"case_id":"ef66a6e5","evidence_sessions":["answer_f7fd1029_2","answer_f7fd1029_1"],"is_certain":true} +{"case_id":"5025383b","evidence_sessions":["answer_688f9a3f_1","answer_688f9a3f_2"],"is_certain":true} +{"case_id":"a1cc6108","evidence_sessions":["answer_17dc2f5b_2","answer_17dc2f5b_1"],"is_certain":true} +{"case_id":"9ee3ecd6","evidence_sessions":["answer_66c23110_1"],"is_certain":true} +{"case_id":"3fdac837","evidence_sessions":["answer_419d21d5_2","answer_419d21d5_1"],"is_certain":true} +{"case_id":"91b15a6e","evidence_sessions":["answer_5404a208_1","answer_5404a208_2"],"is_certain":true} +{"case_id":"27016adc","evidence_sessions":["answer_a37bdf22_2","answer_a37bdf22_1"],"is_certain":true} +{"case_id":"720133ac","evidence_sessions":["answer_c9dfeaea_1","answer_c9dfeaea_2"],"is_certain":true} +{"case_id":"77eafa52","evidence_sessions":["answer_33c251f0_2","answer_33c251f0_1"],"is_certain":true} +{"case_id":"8979f9ec","evidence_sessions":["answer_35e36341_1","answer_35e36341_2"],"is_certain":true} +{"case_id":"0100672e","evidence_sessions":["answer_35c9798c_2","answer_35c9798c_1"],"is_certain":true} +{"case_id":"a96c20ee","evidence_sessions":["answer_ef84b994_1","answer_ef84b994_2"],"is_certain":true} +{"case_id":"92a0aa75","evidence_sessions":["answer_6cb8f792_1","answer_6cb8f792_2"],"is_certain":true} +{"case_id":"3fe836c9","evidence_sessions":["answer_1bb63ea5_2","answer_1bb63ea5_1"],"is_certain":true} +{"case_id":"1c549ce4","evidence_sessions":["answer_4cb841a8_1","answer_4cb841a8_2"],"is_certain":true} +{"case_id":"6c49646a","evidence_sessions":["answer_cc1ced21_2","answer_cc1ced21_1"],"is_certain":true} +{"case_id":"1192316e","evidence_sessions":["answer_e184e4c3_2","answer_e184e4c3_1"],"is_certain":true} +{"case_id":"0ea62687","evidence_sessions":["answer_dc5e537d_1","answer_dc5e537d_2"],"is_certain":true} +{"case_id":"67e0d0f2","evidence_sessions":["answer_3a5010af_2","answer_3a5010af_1"],"is_certain":true} +{"case_id":"bb7c3b45","evidence_sessions":["answer_de64539a_1","answer_de64539a_2"],"is_certain":true} +{"case_id":"ba358f49","evidence_sessions":["answer_cbd08e3c_1","answer_cbd08e3c_2"],"is_certain":true} +{"case_id":"61f8c8f8","evidence_sessions":["answer_872e8da2_2","answer_872e8da2_1"],"is_certain":true} +{"case_id":"60159905","evidence_sessions":["answer_75eca223_2","answer_75eca223_1"],"is_certain":true} +{"case_id":"ef9cf60a","evidence_sessions":["answer_87e3a1cb_1","answer_87e3a1cb_2"],"is_certain":true} +{"case_id":"73d42213","evidence_sessions":["answer_1881e7db_1","answer_1881e7db_2"],"is_certain":true} +{"case_id":"bc149d6b","evidence_sessions":["answer_92147866_1","answer_92147866_2"],"is_certain":true} +{"case_id":"099778bb","evidence_sessions":["answer_80d6d664_2","answer_80d6d664_1"],"is_certain":true} +{"case_id":"09ba9854","evidence_sessions":["answer_96c743d0_2","answer_96c743d0_1"],"is_certain":true} +{"case_id":"d6062bb9","evidence_sessions":["answer_23f3a657_2","answer_23f3a657_1"],"is_certain":true} +{"case_id":"157a136e","evidence_sessions":["answer_8de18468_2","answer_8de18468_1"],"is_certain":true} +{"case_id":"c18a7dc8","evidence_sessions":["answer_2e2085fa_2","answer_2e2085fa_1"],"is_certain":true} +{"case_id":"a3332713","evidence_sessions":["answer_16ece55f_2","answer_16ece55f_1"],"is_certain":true} +{"case_id":"55241a1f","evidence_sessions":["answer_fa08bf49_1","answer_fa08bf49_2"],"is_certain":true} +{"case_id":"a08a253f","evidence_sessions":["answer_47152166_2","answer_47152166_1"],"is_certain":true} +{"case_id":"f0e564bc","evidence_sessions":["answer_cfcf5340_1","answer_cfcf5340_2"],"is_certain":true} +{"case_id":"078150f1","evidence_sessions":["answer_254d8b09_1","answer_254d8b09_2"],"is_certain":true} +{"case_id":"8cf4d046","evidence_sessions":["answer_e2278b24_1","answer_e2278b24_2"],"is_certain":true} +{"case_id":"a346bb18","evidence_sessions":["answer_4934b2d7_2","answer_4934b2d7_1"],"is_certain":true} +{"case_id":"37f165cf","evidence_sessions":["answer_6b9b2b1e_2","answer_6b9b2b1e_1"],"is_certain":false} +{"case_id":"8e91e7d9","evidence_sessions":["answer_477ae455_1","answer_477ae455_2"],"is_certain":true} +{"case_id":"87f22b4a","evidence_sessions":["answer_f56e6152_2","answer_f56e6152_1"],"is_certain":true} +{"case_id":"e56a43b9","evidence_sessions":["answer_430d0a87_1","answer_430d0a87_2"],"is_certain":true} +{"case_id":"efc3f7c2","evidence_sessions":["answer_fc81d1a2_1","answer_fc81d1a2_2"],"is_certain":true} +{"case_id":"21d02d0d","evidence_sessions":["answer_2c637141_2","answer_2c637141_1"],"is_certain":true} +{"case_id":"2311e44b_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"6456829e_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"e5ba910e_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"a96c20ee_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"ba358f49_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"09ba9854_abs","evidence_sessions":["answer_96c743d0_abs_1","answer_96c743d0_abs_2"],"is_certain":true} +{"case_id":"gpt4_59149c77","evidence_sessions":["answer_d00ba6d0_1","answer_d00ba6d0_2"],"is_certain":true} +{"case_id":"gpt4_f49edff3","evidence_sessions":["answer_3e9fce53_1","answer_3e9fce53_2","answer_3e9fce53_3"],"is_certain":true} +{"case_id":"71017276","evidence_sessions":["answer_0b4a8adc_1"],"is_certain":true} +{"case_id":"b46e15ed","evidence_sessions":["answer_4bfcc250_4","answer_4bfcc250_3","answer_4bfcc250_2","answer_4bfcc250_1"],"is_certain":true} +{"case_id":"gpt4_fa19884c","evidence_sessions":["answer_ff201786_1","answer_ff201786_2"],"is_certain":true} +{"case_id":"0bc8ad92","evidence_sessions":["answer_f4ea84fb_3","answer_f4ea84fb_2","answer_f4ea84fb_1"],"is_certain":true} +{"case_id":"af082822","evidence_sessions":["answer_b51b6115_1"],"is_certain":true} +{"case_id":"gpt4_4929293a","evidence_sessions":["answer_add9b012_2","answer_add9b012_1"],"is_certain":true} +{"case_id":"gpt4_b5700ca9","evidence_sessions":["answer_a17423e7_1"],"is_certain":true} +{"case_id":"9a707b81","evidence_sessions":["answer_dba89487_2","answer_dba89487_1"],"is_certain":false} +{"case_id":"gpt4_1d4ab0c9","evidence_sessions":["answer_febde667_1","answer_febde667_2"],"is_certain":true} +{"case_id":"gpt4_e072b769","evidence_sessions":["answer_c19bd2bf_1"],"is_certain":true} +{"case_id":"0db4c65d","evidence_sessions":["answer_b9e32ff8_1","answer_b9e32ff8_2"],"is_certain":true} +{"case_id":"gpt4_1d80365e","evidence_sessions":["answer_661b711f_1","answer_661b711f_2"],"is_certain":true} +{"case_id":"gpt4_7f6b06db","evidence_sessions":["answer_5d8c99d3_1","answer_5d8c99d3_2","answer_5d8c99d3_3"],"is_certain":true} +{"case_id":"gpt4_6dc9b45b","evidence_sessions":["answer_c4df007f_1"],"is_certain":true} +{"case_id":"gpt4_8279ba02","evidence_sessions":["answer_56521e65_1"],"is_certain":true} +{"case_id":"gpt4_18c2b244","evidence_sessions":["answer_c862f65a_2","answer_c862f65a_3","answer_c862f65a_1"],"is_certain":true} +{"case_id":"gpt4_a1b77f9c","evidence_sessions":["answer_e9ad5914_1","answer_e9ad5914_2","answer_e9ad5914_3","answer_e9ad5914_4","answer_e9ad5914_5","answer_e9ad5914_6"],"is_certain":false} +{"case_id":"gpt4_1916e0ea","evidence_sessions":["answer_447052a5_2","answer_447052a5_1"],"is_certain":true} +{"case_id":"gpt4_7a0daae1","evidence_sessions":["answer_4d5490f1_1","answer_4d5490f1_2"],"is_certain":true} +{"case_id":"gpt4_468eb063","evidence_sessions":["answer_9b09d95b_1"],"is_certain":true} +{"case_id":"gpt4_7abb270c","evidence_sessions":["answer_7093d898_1","answer_7093d898_2","answer_7093d898_3","answer_7093d898_4","answer_7093d898_5","answer_7093d898_6"],"is_certain":false} +{"case_id":"gpt4_1e4a8aeb","evidence_sessions":["answer_16bd5ea5_1","answer_16bd5ea5_2"],"is_certain":true} +{"case_id":"gpt4_4fc4f797","evidence_sessions":["answer_be07688f_1","answer_be07688f_2"],"is_certain":true} +{"case_id":"4dfccbf7","evidence_sessions":["answer_4bebc782_1","answer_4bebc782_2"],"is_certain":true} +{"case_id":"gpt4_61e13b3c","evidence_sessions":["answer_e831a29f_1","answer_e831a29f_2"],"is_certain":true} +{"case_id":"gpt4_45189cb4","evidence_sessions":["answer_e6c20e52_3","answer_e6c20e52_2","answer_e6c20e52_1"],"is_certain":true} +{"case_id":"2ebe6c90","evidence_sessions":["answer_c9d35c00_1","answer_c9d35c00_2"],"is_certain":true} +{"case_id":"gpt4_e061b84f","evidence_sessions":["answer_8c64ce25_2","answer_8c64ce25_1","answer_8c64ce25_3"],"is_certain":true} +{"case_id":"370a8ff4","evidence_sessions":["answer_61d1be50_1","answer_61d1be50_2"],"is_certain":true} +{"case_id":"gpt4_d6585ce8","evidence_sessions":["answer_f999b05b_1","answer_f999b05b_4","answer_f999b05b_2","answer_f999b05b_5","answer_f999b05b_3"],"is_certain":true} +{"case_id":"gpt4_4ef30696","evidence_sessions":["answer_f964cea3_1","answer_f964cea3_2"],"is_certain":true} +{"case_id":"gpt4_ec93e27f","evidence_sessions":["answer_e3aa84c4_2","answer_e3aa84c4_1"],"is_certain":true} +{"case_id":"6e984301","evidence_sessions":["answer_88841f26_1","answer_88841f26_2"],"is_certain":true} +{"case_id":"8077ef71","evidence_sessions":["answer_0dd54b7c_1"],"is_certain":true} +{"case_id":"gpt4_f420262c","evidence_sessions":["answer_d8a1af6b_1","answer_d8a1af6b_2","answer_d8a1af6b_3","answer_d8a1af6b_4","answer_d8a1af6b_5"],"is_certain":true} +{"case_id":"gpt4_8e165409","evidence_sessions":["answer_d97db962_1","answer_d97db962_2"],"is_certain":true} +{"case_id":"gpt4_74aed68e","evidence_sessions":["answer_aed8cf17_1","answer_aed8cf17_2"],"is_certain":true} +{"case_id":"bcbe585f","evidence_sessions":["answer_43ba3965_1"],"is_certain":true} +{"case_id":"gpt4_21adecb5","evidence_sessions":["answer_1e2369c9_1","answer_1e2369c9_2"],"is_certain":true} +{"case_id":"5e1b23de","evidence_sessions":["answer_c18d480b_1"],"is_certain":true} +{"case_id":"gpt4_98f46fc6","evidence_sessions":["answer_5850de18_2","answer_5850de18_1"],"is_certain":true} +{"case_id":"gpt4_af6db32f","evidence_sessions":["answer_184c8f56_1"],"is_certain":true} +{"case_id":"eac54adc","evidence_sessions":["answer_0d4d0347_1","answer_0d4d0347_2"],"is_certain":true} +{"case_id":"gpt4_7ddcf75f","evidence_sessions":["answer_ad8a76cb_1"],"is_certain":true} +{"case_id":"gpt4_a2d1d1f6","evidence_sessions":["answer_f6d6e33f_1"],"is_certain":true} +{"case_id":"gpt4_85da3956","evidence_sessions":["answer_581ab834_1"],"is_certain":true} +{"case_id":"gpt4_b0863698","evidence_sessions":["answer_550bb2d1_1"],"is_certain":true} +{"case_id":"gpt4_68e94287","evidence_sessions":["answer_9793daa3_2","answer_9793daa3_1"],"is_certain":true} +{"case_id":"gpt4_e414231e","evidence_sessions":["answer_e28c1f0d_1","answer_e28c1f0d_2"],"is_certain":true} +{"case_id":"gpt4_7ca326fa","evidence_sessions":["answer_cf021b36_1","answer_cf021b36_2","answer_cf021b36_3"],"is_certain":true} +{"case_id":"gpt4_7bc6cf22","evidence_sessions":["answer_e22d6aef_1"],"is_certain":true} +{"case_id":"2ebe6c92","evidence_sessions":["answer_c9d35c00_1","answer_c9d35c00_2"],"is_certain":true} +{"case_id":"gpt4_e061b84g","evidence_sessions":["answer_8c64ce26_2","answer_8c64ce26_1","answer_8c64ce26_3"],"is_certain":true} +{"case_id":"71017277","evidence_sessions":["answer_0b4a8adc_1"],"is_certain":true} +{"case_id":"b46e15ee","evidence_sessions":["answer_4bfcc251_4","answer_4bfcc251_3","answer_4bfcc251_2","answer_4bfcc251_1"],"is_certain":true} +{"case_id":"gpt4_d6585ce9","evidence_sessions":["answer_f999b05c_1","answer_f999b05c_4","answer_f999b05c_2","answer_f999b05c_5","answer_f999b05c_3"],"is_certain":true} +{"case_id":"gpt4_1e4a8aec","evidence_sessions":["answer_16bd5ea6_1","answer_16bd5ea6_2"],"is_certain":true} +{"case_id":"gpt4_f420262d","evidence_sessions":["answer_d8a1af6c_1","answer_d8a1af6c_2","answer_d8a1af6c_5","answer_d8a1af6c_4","answer_d8a1af6c_3"],"is_certain":false} +{"case_id":"gpt4_59149c78","evidence_sessions":["answer_d00ba6d1_2"],"is_certain":true} +{"case_id":"gpt4_e414231f","evidence_sessions":["answer_e28c1f0e_1","answer_e28c1f0e_2"],"is_certain":true} +{"case_id":"gpt4_4929293b","evidence_sessions":["answer_add9b013_2","answer_add9b013_1"],"is_certain":true} +{"case_id":"gpt4_468eb064","evidence_sessions":["answer_9b09d95b_1"],"is_certain":true} +{"case_id":"gpt4_fa19884d","evidence_sessions":["answer_ff201787_1","answer_ff201787_2"],"is_certain":true} +{"case_id":"9a707b82","evidence_sessions":["answer_dba89488_2","answer_dba89488_1"],"is_certain":true} +{"case_id":"eac54add","evidence_sessions":["answer_0d4d0348_1","answer_0d4d0348_2"],"is_certain":true} +{"case_id":"4dfccbf8","evidence_sessions":["answer_4bebc783_1","answer_4bebc783_2"],"is_certain":true} +{"case_id":"0bc8ad93","evidence_sessions":["answer_f4ea84fc_3","answer_f4ea84fc_2","answer_f4ea84fc_1"],"is_certain":false} +{"case_id":"6e984302","evidence_sessions":["answer_88841f27_2"],"is_certain":true} +{"case_id":"gpt4_8279ba03","evidence_sessions":["answer_56521e66_1"],"is_certain":true} +{"case_id":"gpt4_b5700ca0","evidence_sessions":["answer_a17423e8_1"],"is_certain":true} +{"case_id":"gpt4_68e94288","evidence_sessions":["answer_9793daa4_2","answer_9793daa4_1"],"is_certain":true} +{"case_id":"gpt4_2655b836","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_2487a7cb","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_76048e76","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_2312f94c","evidence_sessions":[],"is_certain":true} +{"case_id":"0bb5a684","evidence_sessions":[],"is_certain":true} +{"case_id":"08f4fc43","evidence_sessions":["answer_6ea1541e_2","answer_6ea1541e_1"],"is_certain":true} +{"case_id":"2c63a862","evidence_sessions":["answer_d39b7977_1","answer_d39b7977_2"],"is_certain":true} +{"case_id":"gpt4_385a5000","evidence_sessions":[],"is_certain":true} +{"case_id":"2a1811e2","evidence_sessions":[],"is_certain":true} +{"case_id":"bbf86515","evidence_sessions":["answer_b3763b6b_1","answer_b3763b6b_2"],"is_certain":true} +{"case_id":"gpt4_5dcc0aab","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_0b2f1d21","evidence_sessions":[],"is_certain":true} +{"case_id":"f0853d11","evidence_sessions":["answer_932ea3bf_2","answer_932ea3bf_1"],"is_certain":true} +{"case_id":"gpt4_6ed717ea","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_70e84552","evidence_sessions":[],"is_certain":true} +{"case_id":"a3838d2b","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_93159ced","evidence_sessions":["answer_e5131a1b_2","answer_e5131a1b_1"],"is_certain":true} +{"case_id":"gpt4_2d58bcd6","evidence_sessions":["answer_3e11e0ae_1","answer_3e11e0ae_2"],"is_certain":true} +{"case_id":"gpt4_65aabe59","evidence_sessions":[],"is_certain":true} +{"case_id":"982b5123","evidence_sessions":[],"is_certain":true} +{"case_id":"b9cfe692","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_4edbafa2","evidence_sessions":[],"is_certain":true} +{"case_id":"c8090214","evidence_sessions":["answer_70dc7d08_2","answer_70dc7d08_1"],"is_certain":true} +{"case_id":"gpt4_483dd43c","evidence_sessions":[],"is_certain":true} +{"case_id":"e4e14d04","evidence_sessions":[],"is_certain":true} +{"case_id":"c9f37c46","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_2c50253f","evidence_sessions":["answer_9af4e346_1","answer_9af4e346_2"],"is_certain":true} +{"case_id":"dcfa8644","evidence_sessions":["answer_5e3eeb12_2","answer_5e3eeb12_1"],"is_certain":true} +{"case_id":"gpt4_b4a80587","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_9a159967","evidence_sessions":["answer_8a42fedf_2"],"is_certain":true} +{"case_id":"cc6d1ec1","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_8c8961ae","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_d9af6064","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_7de946e7","evidence_sessions":["answer_6a78e959_2","answer_6a78e959_1"],"is_certain":true} +{"case_id":"d01c6aa8","evidence_sessions":[],"is_certain":true} +{"case_id":"993da5e2","evidence_sessions":[],"is_certain":true} +{"case_id":"a3045048","evidence_sessions":["answer_016f6bd4_1"],"is_certain":true} +{"case_id":"gpt4_d31cdae3","evidence_sessions":["answer_8464304d_1","answer_8464304d_2"],"is_certain":true} +{"case_id":"gpt4_cd90e484","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_88806d6e","evidence_sessions":["answer_e60a93ff_1","answer_e60a93ff_2"],"is_certain":true} +{"case_id":"gpt4_4cd9eba1","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_93f6379c","evidence_sessions":["answer_544fe66c_2","answer_544fe66c_3"],"is_certain":true} +{"case_id":"b29f3365","evidence_sessions":["answer_436d4309_2","answer_436d4309_1"],"is_certain":true} +{"case_id":"gpt4_2f56ae70","evidence_sessions":[],"is_certain":true} +{"case_id":"6613b389","evidence_sessions":["answer_aaf71ce2_2","answer_aaf71ce2_1"],"is_certain":true} +{"case_id":"gpt4_78cf46a3","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_0a05b494","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_1a1dc16d","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_2f584639","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_213fd887","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_5438fa52","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_c27434e8","evidence_sessions":["answer_d8e33f5c_1","answer_d8e33f5c_2"],"is_certain":true} +{"case_id":"gpt4_fe651585","evidence_sessions":[],"is_certain":true} +{"case_id":"8c18457d","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_70e84552_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_93159ced_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"982b5123_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"c8090214_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_c27434e8_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"gpt4_fe651585_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"6a1eabeb","evidence_sessions":["answer_a25d4a91_1","answer_a25d4a91_2"],"is_certain":true} +{"case_id":"6aeb4375","evidence_sessions":["answer_3f9693b7_1","answer_3f9693b7_2"],"is_certain":true} +{"case_id":"830ce83f","evidence_sessions":["answer_0b1a0942_2"],"is_certain":true} +{"case_id":"852ce960","evidence_sessions":["answer_3a6f1e82_2"],"is_certain":true} +{"case_id":"945e3d21","evidence_sessions":["answer_6a4f8626_1","answer_6a4f8626_2"],"is_certain":true} +{"case_id":"d7c942c3","evidence_sessions":["answer_eecb10d9_1","answer_eecb10d9_2"],"is_certain":true} +{"case_id":"71315a70","evidence_sessions":["answer_c44b9df4_1","answer_c44b9df4_2"],"is_certain":true} +{"case_id":"89941a93","evidence_sessions":["answer_e1403127_1","answer_e1403127_2"],"is_certain":true} +{"case_id":"ce6d2d27","evidence_sessions":["answer_73540165_2"],"is_certain":true} +{"case_id":"9ea5eabc","evidence_sessions":["answer_02e66dec_1","answer_02e66dec_2"],"is_certain":true} +{"case_id":"07741c44","evidence_sessions":["answer_7e9ad7b4_1","answer_7e9ad7b4_2"],"is_certain":true} +{"case_id":"a1eacc2a","evidence_sessions":["answer_0eb23770_1","answer_0eb23770_2"],"is_certain":true} +{"case_id":"184da446","evidence_sessions":["answer_e2f4f947_1","answer_e2f4f947_2"],"is_certain":true} +{"case_id":"031748ae","evidence_sessions":["answer_8748f791_1","answer_8748f791_2"],"is_certain":true} +{"case_id":"4d6b87c8","evidence_sessions":["answer_766ab8da_1","answer_766ab8da_2"],"is_certain":true} +{"case_id":"0f05491a","evidence_sessions":["answer_d6d2eba8_2"],"is_certain":true} +{"case_id":"08e075c7","evidence_sessions":["answer_cdbe2250_1","answer_cdbe2250_2"],"is_certain":true} +{"case_id":"f9e8c073","evidence_sessions":["answer_b191df5b_1","answer_b191df5b_2"],"is_certain":true} +{"case_id":"41698283","evidence_sessions":["answer_c7ddc051_1","answer_c7ddc051_2"],"is_certain":true} +{"case_id":"2698e78f","evidence_sessions":["answer_9282283d_1","answer_9282283d_2"],"is_certain":true} +{"case_id":"b6019101","evidence_sessions":["answer_67074b4b_1","answer_67074b4b_2"],"is_certain":true} +{"case_id":"45dc21b6","evidence_sessions":["answer_07664d43_1","answer_07664d43_2"],"is_certain":true} +{"case_id":"5a4f22c0","evidence_sessions":["answer_b0f3dfff_1","answer_b0f3dfff_2"],"is_certain":true} +{"case_id":"6071bd76","evidence_sessions":["answer_4dac77cb_1","answer_4dac77cb_2"],"is_certain":true} +{"case_id":"e493bb7c","evidence_sessions":["answer_1a374afa_1","answer_1a374afa_2"],"is_certain":true} +{"case_id":"618f13b2","evidence_sessions":["answer_caf5b52e_1","answer_caf5b52e_2"],"is_certain":true} +{"case_id":"72e3ee87","evidence_sessions":["answer_d7de9a6a_1","answer_d7de9a6a_2"],"is_certain":true} +{"case_id":"c4ea545c","evidence_sessions":["answer_d3bf812b_1","answer_d3bf812b_2"],"is_certain":true} +{"case_id":"01493427","evidence_sessions":["answer_a7b44747_1","answer_a7b44747_2"],"is_certain":true} +{"case_id":"6a27ffc2","evidence_sessions":["answer_77f32504_1","answer_77f32504_2"],"is_certain":true} +{"case_id":"2133c1b5","evidence_sessions":["answer_52382508_1"],"is_certain":true} +{"case_id":"18bc8abd","evidence_sessions":["answer_fff743f5_1","answer_fff743f5_2"],"is_certain":true} +{"case_id":"db467c8c","evidence_sessions":["answer_611b6e83_1","answer_611b6e83_2"],"is_certain":true} +{"case_id":"7a87bd0c","evidence_sessions":["answer_d08a934d_1","answer_d08a934d_2"],"is_certain":true} +{"case_id":"e61a7584","evidence_sessions":["answer_f25c32f5_1","answer_f25c32f5_2"],"is_certain":true} +{"case_id":"1cea1afa","evidence_sessions":["answer_79c395a9_1","answer_79c395a9_2"],"is_certain":true} +{"case_id":"ed4ddc30","evidence_sessions":["answer_babbaccb_1","answer_babbaccb_2"],"is_certain":true} +{"case_id":"8fb83627","evidence_sessions":["answer_966cecbb_1","answer_966cecbb_2"],"is_certain":true} +{"case_id":"b01defab","evidence_sessions":["answer_8c0712af_1","answer_8c0712af_2"],"is_certain":true} +{"case_id":"22d2cb42","evidence_sessions":["answer_bcce0b73_1","answer_bcce0b73_2"],"is_certain":true} +{"case_id":"0e4e4c46","evidence_sessions":["answer_f2f998c7_1","answer_f2f998c7_2"],"is_certain":true} +{"case_id":"4b24c848","evidence_sessions":["answer_2cec623b_1","answer_2cec623b_2"],"is_certain":true} +{"case_id":"7e974930","evidence_sessions":["answer_c9f5693c_1","answer_c9f5693c_2"],"is_certain":true} +{"case_id":"603deb26","evidence_sessions":["answer_8afdebac_1","answer_8afdebac_2"],"is_certain":true} +{"case_id":"59524333","evidence_sessions":["answer_b28f2c7a_2"],"is_certain":true} +{"case_id":"5831f84d","evidence_sessions":["answer_8d63a897_1","answer_8d63a897_2"],"is_certain":true} +{"case_id":"eace081b","evidence_sessions":["answer_8a791264_2"],"is_certain":true} +{"case_id":"affe2881","evidence_sessions":["answer_90de9b4d_1","answer_90de9b4d_2"],"is_certain":true} +{"case_id":"50635ada","evidence_sessions":["answer_dcd74827_1","answer_dcd74827_2"],"is_certain":true} +{"case_id":"e66b632c","evidence_sessions":["answer_ac0140ce_1","answer_ac0140ce_2"],"is_certain":true} +{"case_id":"0ddfec37","evidence_sessions":["answer_a22b654d_1","answer_a22b654d_2"],"is_certain":true} +{"case_id":"f685340e","evidence_sessions":["answer_25df025b_1","answer_25df025b_2"],"is_certain":true} +{"case_id":"cc5ded98","evidence_sessions":["answer_a5b68517_1","answer_a5b68517_2"],"is_certain":true} +{"case_id":"dfde3500","evidence_sessions":["answer_35d6c0be_1","answer_35d6c0be_2"],"is_certain":true} +{"case_id":"69fee5aa","evidence_sessions":["answer_d6028d6e_1","answer_d6028d6e_2"],"is_certain":true} +{"case_id":"7401057b","evidence_sessions":["answer_94650bfa_1","answer_94650bfa_2"],"is_certain":true} +{"case_id":"cf22b7bf","evidence_sessions":["answer_ae3a122b_1","answer_ae3a122b_2"],"is_certain":true} +{"case_id":"a2f3aa27","evidence_sessions":["answer_5126c02d_1","answer_5126c02d_2"],"is_certain":true} +{"case_id":"c7dc5443","evidence_sessions":["answer_0cdbca92_1","answer_0cdbca92_2"],"is_certain":true} +{"case_id":"06db6396","evidence_sessions":["answer_da72b1b4_1","answer_da72b1b4_2"],"is_certain":true} +{"case_id":"3ba21379","evidence_sessions":["answer_cd345582_1","answer_cd345582_2"],"is_certain":true} +{"case_id":"9bbe84a2","evidence_sessions":["answer_c6a0c6c2_1","answer_c6a0c6c2_2"],"is_certain":true} +{"case_id":"10e09553","evidence_sessions":["answer_67be2c38_1","answer_67be2c38_2"],"is_certain":true} +{"case_id":"dad224aa","evidence_sessions":["answer_4a97ae40_2"],"is_certain":true} +{"case_id":"ba61f0b9","evidence_sessions":["answer_f377cda7_2"],"is_certain":true} +{"case_id":"42ec0761","evidence_sessions":["answer_e3892371_1","answer_e3892371_2"],"is_certain":true} +{"case_id":"5c40ec5b","evidence_sessions":["answer_1cb52d0a_1","answer_1cb52d0a_2"],"is_certain":true} +{"case_id":"c6853660","evidence_sessions":["answer_626e93c4_1","answer_626e93c4_2"],"is_certain":true} +{"case_id":"26bdc477","evidence_sessions":["answer_f762ad8d_1","answer_f762ad8d_2"],"is_certain":true} +{"case_id":"0977f2af","evidence_sessions":["answer_3bf5b73b_1","answer_3bf5b73b_2"],"is_certain":true} +{"case_id":"6aeb4375_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"031748ae_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"2698e78f_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"2133c1b5_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"0ddfec37_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"f685340e_abs","evidence_sessions":[],"is_certain":true} +{"case_id":"89941a94","evidence_sessions":["answer_e1403127_1","answer_e1403127_2"],"is_certain":true} +{"case_id":"07741c45","evidence_sessions":["answer_7e9ad7b4_1","answer_7e9ad7b4_2"],"is_certain":true} +{"case_id":"7161e7e2","evidence_sessions":["answer_sharegpt_5Lzox6N_0"],"is_certain":true} +{"case_id":"c4f10528","evidence_sessions":["answer_ultrachat_234453"],"is_certain":true} +{"case_id":"89527b6b","evidence_sessions":["answer_sharegpt_YkWn1Ne_0"],"is_certain":true} +{"case_id":"e9327a54","evidence_sessions":["answer_ultrachat_480665"],"is_certain":true} +{"case_id":"4c36ccef","evidence_sessions":["answer_ultrachat_448704"],"is_certain":true} +{"case_id":"6ae235be","evidence_sessions":["answer_sharegpt_IUWQYGQ_0"],"is_certain":true} +{"case_id":"7e00a6cb","evidence_sessions":["answer_ultrachat_370515"],"is_certain":true} +{"case_id":"1903aded","evidence_sessions":["answer_sharegpt_hA7AkP3_0"],"is_certain":true} +{"case_id":"ceb54acb","evidence_sessions":["answer_sharegpt_cGdjmYo_0"],"is_certain":true} +{"case_id":"f523d9fe","evidence_sessions":["answer_sharegpt_m2xJfjo_0"],"is_certain":true} +{"case_id":"0e5e2d1a","evidence_sessions":["answer_ultrachat_113156"],"is_certain":true} +{"case_id":"fea54f57","evidence_sessions":["answer_ultrachat_187684"],"is_certain":true} +{"case_id":"cc539528","evidence_sessions":["answer_ultrachat_374124"],"is_certain":true} +{"case_id":"dc439ea3","evidence_sessions":["answer_ultrachat_459954"],"is_certain":true} +{"case_id":"18dcd5a5","evidence_sessions":["answer_sharegpt_hn3IS1q_0"],"is_certain":true} +{"case_id":"488d3006","evidence_sessions":["answer_ultrachat_275993"],"is_certain":true} +{"case_id":"58470ed2","evidence_sessions":["answer_sharegpt_U4oCSfU_7"],"is_certain":true} +{"case_id":"8cf51dda","evidence_sessions":["answer_sharegpt_HFMn2ZX_0"],"is_certain":true} +{"case_id":"1d4da289","evidence_sessions":["answer_ultrachat_348449"],"is_certain":true} +{"case_id":"8464fc84","evidence_sessions":["answer_ultrachat_467053"],"is_certain":true} +{"case_id":"8aef76bc","evidence_sessions":["answer_ultrachat_563222"],"is_certain":true} +{"case_id":"71a3fd6b","evidence_sessions":["answer_ultrachat_417348"],"is_certain":true} +{"case_id":"2bf43736","evidence_sessions":["answer_sharegpt_2kpncbX_13"],"is_certain":true} +{"case_id":"70b3e69b","evidence_sessions":["answer_ultrachat_334948"],"is_certain":true} +{"case_id":"8752c811","evidence_sessions":["answer_sharegpt_6pWK9yx_0"],"is_certain":true} +{"case_id":"3249768e","evidence_sessions":["answer_sharegpt_CaxTGYP_0"],"is_certain":true} +{"case_id":"1b9b7252","evidence_sessions":["answer_ultrachat_115151"],"is_certain":true} +{"case_id":"1568498a","evidence_sessions":["answer_sharegpt_d6JJiqH_76"],"is_certain":true} +{"case_id":"6222b6eb","evidence_sessions":["answer_sharegpt_H9PiM5G_0"],"is_certain":true} +{"case_id":"e8a79c70","evidence_sessions":["answer_ultrachat_13075"],"is_certain":true} +{"case_id":"d596882b","evidence_sessions":["answer_ultrachat_252214"],"is_certain":true} +{"case_id":"e3fc4d6e","evidence_sessions":["answer_sharegpt_5m7gg5F_0"],"is_certain":true} +{"case_id":"51b23612","evidence_sessions":["answer_ultrachat_427265"],"is_certain":true} +{"case_id":"3e321797","evidence_sessions":["answer_ultrachat_94624"],"is_certain":true} +{"case_id":"e982271f","evidence_sessions":["answer_ultrachat_195444"],"is_certain":true} +{"case_id":"352ab8bd","evidence_sessions":["answer_sharegpt_NoDZzot_7"],"is_certain":true} +{"case_id":"fca762bc","evidence_sessions":["answer_ultrachat_39395"],"is_certain":true} +{"case_id":"7a8d0b71","evidence_sessions":["answer_sharegpt_i0tMT9q_9"],"is_certain":true} +{"case_id":"a40e080f","evidence_sessions":["answer_ultrachat_269020"],"is_certain":true} +{"case_id":"8b9d4367","evidence_sessions":["answer_ultrachat_289157"],"is_certain":true} +{"case_id":"5809eb10","evidence_sessions":["answer_sharegpt_4aJsGCH_0"],"is_certain":true} +{"case_id":"41275add","evidence_sessions":["answer_sharegpt_81riySf_0"],"is_certain":true} +{"case_id":"4388e9dd","evidence_sessions":["answer_sharegpt_qTi81nS_0"],"is_certain":true} +{"case_id":"4baee567","evidence_sessions":["answer_sharegpt_i9adwQn_0"],"is_certain":true} +{"case_id":"561fabcd","evidence_sessions":["answer_sharegpt_hChsWOp_97"],"is_certain":true} +{"case_id":"b759caee","evidence_sessions":["answer_sharegpt_2BSXlAr_0"],"is_certain":true} +{"case_id":"ac031881","evidence_sessions":["answer_sharegpt_GYqnAhC_190"],"is_certain":true} +{"case_id":"28bcfaac","evidence_sessions":["answer_ultrachat_446979"],"is_certain":true} +{"case_id":"16c90bf4","evidence_sessions":["answer_ultrachat_294807"],"is_certain":true} +{"case_id":"c8f1aeed","evidence_sessions":["answer_ultrachat_519486"],"is_certain":true} +{"case_id":"eaca4986","evidence_sessions":["answer_sharegpt_SS141vi_0"],"is_certain":true} +{"case_id":"c7cf7dfd","evidence_sessions":["answer_ultrachat_456407"],"is_certain":true} +{"case_id":"e48988bc","evidence_sessions":["answer_ultrachat_174360"],"is_certain":true} +{"case_id":"1de5cff2","evidence_sessions":["answer_ultrachat_440262"],"is_certain":true} +{"case_id":"65240037","evidence_sessions":["answer_ultrachat_403752"],"is_certain":true} +{"case_id":"778164c6","evidence_sessions":["answer_ultrachat_399000"],"is_certain":true} diff --git a/benchmark/longmemeval/run_home.py b/benchmark/longmemeval/run_home.py new file mode 100644 index 00000000..2c357ab2 --- /dev/null +++ b/benchmark/longmemeval/run_home.py @@ -0,0 +1,173 @@ +"""Prepare the LongMemEval Home workspace, or optimize it with --optimize after preparation.""" + +import argparse +import asyncio +import json +import sys +from pathlib import Path + +from pydantic import BaseModel, ConfigDict, Field +import yaml + +# Support direct execution from any working directory as well as module imports. +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +# pylint: disable=wrong-import-position +from reme_home.preparation.spec import PreparationSpec # noqa: E402 +from reme_home.judge import JUDGE_PLUGINS # noqa: E402 +from reme_home.sampling import SamplingPolicy # noqa: E402 + +from reme.application import Application # noqa: E402 +from reme.utils.env_utils import load_env # noqa: E402 + +# pylint: enable=wrong-import-position + +DEFAULT_CONFIG = Path(__file__).with_name("config_home.yaml") + + +class RunHomeConfig(BaseModel): + """Runner settings kept separate from Home's public preparation contract.""" + + model_config = ConfigDict(extra="forbid", strict=True) + workspace_dir: str = Field(min_length=1) + preparation: dict + sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy) + validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False) + max_evaluations: int = Field(default=10, ge=1) + memory_limit_mib: int | None = Field(default=None, ge=0) + + +def project_path(value: str, project_root: Path) -> Path: + """Use repository-relative paths consistently regardless of the caller's cwd.""" + if value.startswith("~"): + raise ValueError("Use absolute or project-relative paths, not home-relative paths") + return (project_root / value).resolve() + + +def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT): + """Resolve source paths and validate explicit Home preparation parameters.""" + raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) + config = RunHomeConfig.model_validate(raw) + config.preparation.setdefault("candidate", {"paths": ["plugins/lme/src"]}) + spec = PreparationSpec.model_validate(config.preparation) + spec.benchmark.source = str(project_path(spec.benchmark.source, project_root)) + if spec.benchmark.evidence_source is not None: + spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root)) + spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root)) + workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root) + return workspace, spec.model_dump(mode="json"), config + + +def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT): + """Return resolved preparation inputs, preserving the preparation-only caller contract.""" + workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root) + return workspace, spec + + +async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, instruction=None): + """Apply the same runner configuration to each preparation or optimization lifecycle.""" + workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir) + load_env(PROJECT_ROOT / ".env", override=False) + memory_options = {} + jobs = {} + if config.memory_limit_mib is not None: + memory_options = { + "candidate_gate_memory_limit_mib": config.memory_limit_mib, + "construction_equivalence_memory_limit_mib": config.memory_limit_mib, + } + for name, backend, wrapper in ( + ("home_optimize", "home_optimize_step", "home_optimizer"), + ("home_diagnose", "home_diagnose_step", "home_diagnostic"), + ("home_case_study", "home_case_study_step", "home_case_study"), + ): + jobs[name] = { + "steps": [ + { + "backend": backend, + "agent_wrapper": wrapper, + "memory_limit_mib": config.memory_limit_mib, + **({"concurrency": 4, "output_token_limit": 2000} if name == "home_case_study" else {}), + }, + ], + } + app = Application( + workspace_dir=str(workspace), + plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]], + components={ + "home.evaluation": { + "default": {"sampling_policy": config.sampling_policy.model_dump(mode="json"), **memory_options}, + }, + "agent_wrapper": { + "home_optimizer": { + "validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes, + "max_evaluations": config.max_evaluations, + }, + }, + }, + jobs=jobs, + service={"backend": "cli"}, + enable_logo=False, + log_to_file=False, + ) + try: + await app.start() + if optimize_workspace: + arguments = {} if instruction is None else {"instruction": instruction} + response = await app.run_job("home_optimize", **arguments) + else: + response = await app.run_job("home_prepare", spec=spec) + if not response.success: + raise RuntimeError(response.answer) + return {"workspace_dir": str(workspace), **response.answer} + finally: + await app.close() + + +async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict: + """Prepare once without running evaluations or optimization.""" + return await _run(config_path, workspace_dir=workspace_dir) + + +async def optimize(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir=None, instruction=None) -> dict: + """Optimize an already prepared workspace using this config's actual model budgets.""" + return await _run(config_path, workspace_dir=workspace_dir, optimize_workspace=True, instruction=instruction) + + +async def run(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir: str | None = None) -> dict: + """Prepare through the same Home Job as the CLI, then close the application.""" + return await prepare(config_path, workspace_dir=workspace_dir) + + +def main(default_config: Path = DEFAULT_CONFIG) -> None: + """Run preparation or explicit local one-shot optimization with the benchmark config.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=default_config, help="Home runner YAML configuration") + parser.add_argument("--workspace-dir", help="Override workspace directory (absolute or project-relative)") + parser.add_argument("--optimize", action="store_true", help="Optimize an already prepared workspace") + parser.add_argument("--instruction", help="Optimization focus; requires --optimize") + args = parser.parse_args() + if args.instruction is not None and not args.optimize: + parser.error("--instruction requires --optimize") + if args.optimize: + result = asyncio.run(optimize(args.config, workspace_dir=args.workspace_dir, instruction=args.instruction)) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return + result = asyncio.run(prepare(args.config, workspace_dir=args.workspace_dir)) + print( + json.dumps( + { + "workspace_dir": result["workspace_dir"], + "case_count": result["dataset"]["case_count"], + "query_count": result["dataset"]["query_count"], + "source_commit": result["code"]["source_commit"], + "initial_commit": result["code"]["initial_commit"], + }, + indent=2, + ), + ) + + +if __name__ == "__main__": + main() diff --git a/benchmark/longmemeval/run_home_eval.py b/benchmark/longmemeval/run_home_eval.py new file mode 100644 index 00000000..93d36f27 --- /dev/null +++ b/benchmark/longmemeval/run_home_eval.py @@ -0,0 +1,226 @@ +"""Evaluate a LongMemEval Home candidate in an independent test workspace.""" + +import argparse +import asyncio +import json +import sys +from pathlib import Path + +from pydantic import Field, model_validator +import yaml + +# Support direct execution from any working directory as well as module imports. +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +# pylint: disable=wrong-import-position +from reme_home.code.repository import resolve_revision, validate_repository # noqa: E402 +from reme_home.evaluation.artifacts import contained, read_json # noqa: E402 +from reme_home.evaluation.component import blocking # noqa: E402 +from reme_home.evaluation.inputs import EvaluationSpec, load_inputs # noqa: E402 +from reme_home.judge import JUDGE_PLUGINS # noqa: E402 +from reme_home.preparation.code import tree_fingerprint # noqa: E402 +from reme_home.preparation.data.models import fingerprint # noqa: E402 +from reme_home.preparation.evaluation_source import EvaluationSource # noqa: E402 +from reme_home.preparation.spec import BenchmarkSpec, DebugSpec, SpecModel # noqa: E402 + +from reme.application import Application # noqa: E402 +from reme.utils.env_utils import load_env # noqa: E402 + +# pylint: enable=wrong-import-position + +DEFAULT_CONFIG = Path(__file__).with_name("config_home_eval.yaml") + + +class TestRunConfig(SpecModel): + """Explicit dataset selection and execution options for one pinned candidate.""" + + source_workspace: str = Field(min_length=1) + test_workspace: str = Field(min_length=1) + code_commit: str = Field(min_length=4) + benchmark: BenchmarkSpec + exclude_training_cases: bool = False + evaluation: EvaluationSpec = Field(default_factory=lambda: EvaluationSpec(record_history=False)) + + @model_validator(mode="after") + def validate_test_mode(self): + """Test runs score every prepared query and never consume optimizer budgets.""" + options = self.evaluation + if any( + ( + options.debug, + options.sampling, + options.record_history, + options.max_evaluations is not None, + options.sampling_policy is not None, + options.case_ids is not None, + options.query_ids is not None, + options.reuse_construction_from is not None, + ), + ): + raise ValueError( + "Test evaluation requires full scoring with record_history=false; select cases in benchmark", + ) + if self.exclude_training_cases and (self.benchmark.name != "longmemeval" or not self.benchmark.all_cases): + raise ValueError("exclude_training_cases requires LongMemEval with all_cases=true") + return self + + +def project_path(value: str, project_root: Path) -> Path: + """Resolve runner paths independently of the launching working directory.""" + if value.startswith("~"): + raise ValueError("Use absolute or project-relative paths, not home-relative paths") + return (project_root / value).resolve() + + +def load_request(config_path, *, commit=None, source_workspace=None, test_workspace=None, project_root=PROJECT_ROOT): + """Validate all source inputs before starting an application or writing test data.""" + raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) + for key, value in ( + ("code_commit", commit), + ("source_workspace", source_workspace), + ("test_workspace", test_workspace), + ): + if value is not None: + raw[key] = value + # Execution settings can be partial while retaining the test-only history default. + raw["evaluation"] = {"record_history": False, **raw.get("evaluation", {})} + config = TestRunConfig.model_validate(raw) + source_path = project_path(config.source_workspace, project_root) + target = project_path(config.test_workspace, project_root) + if target.is_relative_to(source_path) or source_path.is_relative_to(target): + raise ValueError("Test and source workspaces must be separate, non-nested directories") + if target.exists() and ( + not target.is_dir() or (any(target.iterdir()) and not (target / "home/metadata/workspace.json").is_file()) + ): + raise ValueError("Use an empty test workspace or a compatible prepared Home test workspace") + source = EvaluationSource.load(source_path, config.code_commit) + benchmark = config.benchmark.model_copy(deep=True) + if benchmark.name != source.spec.benchmark.name: + raise ValueError("Test benchmark must match the source experiment") + benchmark.source = str(project_path(benchmark.source, project_root).resolve(strict=True)) + if benchmark.evidence_source is not None: + benchmark.evidence_source = str(project_path(benchmark.evidence_source, project_root).resolve(strict=True)) + if config.exclude_training_cases: + items = json.loads(Path(benchmark.source).read_text(encoding="utf-8")) + ids = [str(item["question_id"]) for item in items] + if len(ids) != len(set(ids)): + raise ValueError("Duplicate LongMemEval question IDs") + training = set(source.record["dataset"]["case_ids"]) + if training - set(ids): + raise ValueError("Test dataset is missing training IDs; provide the original full dataset") + benchmark.case_ids = [case_id for case_id in ids if case_id not in training] + benchmark.all_cases = False + benchmark = BenchmarkSpec.model_validate(benchmark.model_dump()) + spec = source.spec.model_copy(deep=True) + spec.benchmark = benchmark + spec.debug = DebugSpec(enabled=False) + return config, target, source, spec + + +def existing_preparation(home, source, spec): + """Reopen only the same request with unchanged code, runtime and normalized data.""" + marker = contained(home, "metadata/preparation.json") + if not marker.exists(): + return None + record = read_json(marker) + expected = {**source.provenance, "request_fingerprint": fingerprint(spec.model_dump(mode="json"))} + if record.get("test_evaluation") != expected: + raise ValueError("Test workspace belongs to a different candidate or dataset; use a new workspace") + domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8")) + if fingerprint(domain) != record["config_fingerprint"]: + raise ValueError("Test preparation configuration has changed") + repository, _ = validate_repository(home, record) + if ( + resolve_revision(repository, "HEAD") != record["code"]["initial_commit"] + or tree_fingerprint(repository) != record["code"]["candidate_fingerprint"] + or tree_fingerprint(contained(home, "code/runtime", directory=True)) != source.provenance["runtime_fingerprint"] + ): + raise ValueError("Prepared test code or runtime has changed") + load_inputs(home, EvaluationSpec(record_history=False)) + return record + + +async def run_evaluation( + config_path, + *, + commit=None, + source_workspace=None, + test_workspace=None, + prepare_only=False, + expected_benchmark=None, +): + """Prepare transactionally, then score through the normal application lifecycle.""" + config, target, source, spec = await blocking( + load_request, + config_path, + commit=commit, + source_workspace=source_workspace, + test_workspace=test_workspace, + ) + if expected_benchmark is not None and config.benchmark.name != expected_benchmark: + raise ValueError(f"This entry point requires benchmark.name={expected_benchmark}") + load_env(PROJECT_ROOT / ".env", override=False) + app = Application( + workspace_dir=str(target), + plugins=["home", JUDGE_PLUGINS[config.benchmark.name]], + service={"backend": "cli"}, + enable_logo=False, + log_to_file=False, + ) + try: + await app.start() + home = target / "home" + prepared = await blocking(existing_preparation, home, source, spec) + if prepared is None: + preparation = app.context.components["home.preparation"]["default"] + prepared = await preparation.prepare(spec=spec.model_dump(mode="json"), code_source=source) + result = { + "test_workspace": str(target), + **source.provenance, + "evaluation_commit_sha": prepared["code"]["initial_commit"], + "case_count": prepared["dataset"]["case_count"], + "query_count": prepared["dataset"]["query_count"], + } + if not prepare_only: + response = await app.run_job( + "home_evaluate", + **config.evaluation.model_dump(mode="json", exclude_unset=True), + ) + if not response.success: + raise RuntimeError(response.answer) + result["evaluation"] = response.answer + return result + finally: + await app.close() + + +async def run(config_path=DEFAULT_CONFIG, **kwargs): + """Score the selected commit in its own Home workspace.""" + return await run_evaluation(config_path, expected_benchmark="longmemeval", **kwargs) + + +def main(): + """Explicit local one-shot entry point, with preparation-only service setup.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + parser.add_argument("--commit", help="Candidate commit SHA or unambiguous hexadecimal prefix") + parser.add_argument("--source-workspace", help="Prepared optimization workspace") + parser.add_argument("--test-workspace", help="Independent test output workspace") + parser.add_argument("--prepare-only", action="store_true", help="Prepare for a later home_evaluate service call") + args = parser.parse_args() + result = asyncio.run( + run( + args.config, + commit=args.commit, + source_workspace=args.source_workspace, + test_workspace=args.test_workspace, + prepare_only=args.prepare_only, + ), + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/benchmark/memlens/run.py b/benchmark/memlens/run.py index 433270fc..41434075 100644 --- a/benchmark/memlens/run.py +++ b/benchmark/memlens/run.py @@ -265,15 +265,14 @@ def collect_usage(target: list | None = None, receipt: Path | None = None): def observe_usage(app) -> None: - """Observe owned clients and distinguish native interrupted replies from answers.""" + """Observe owned clients and reject incomplete agent replies.""" for wrapper in app.context.components.get("agent_wrapper", {}).values(): original_reply = wrapper.reply async def reply(*args, _reply=original_reply, **kwargs): result = await _reply(*args, **kwargs) - last = result.get("last_message") or {} - if last.get("finished_reason") == "interrupted" or last.get("error") is not None: - raise RuntimeError("Native Agent was interrupted; see the application log.") + if result["status"] != "completed": + raise RuntimeError(result["error"] or f"Native Agent reply ended: {result['status']}") return result wrapper.reply = reply diff --git a/benchmark/mobilemem/run.py b/benchmark/mobilemem/run.py index 28962ab4..c406eb6d 100644 --- a/benchmark/mobilemem/run.py +++ b/benchmark/mobilemem/run.py @@ -291,15 +291,14 @@ def observe_usage(app) -> None: def observe_completion(app) -> None: - """Keep native interrupted replies unscored; successful replies are unchanged.""" + """Keep incomplete native replies unscored; completed replies are unchanged.""" for wrapper in app.context.components.get("agent_wrapper", {}).values(): original = wrapper.reply async def reply(inputs, *args, _reply=original, **kwargs): result = await _reply(inputs, *args, **kwargs) - last = result.get("last_message") or {} - if str(last.get("finished_reason")) == "interrupted" or last.get("error") is not None: - raise RuntimeError("Native Agent was interrupted; its state remains saved.") + if result["status"] != "completed": + raise RuntimeError(result["error"] or f"Native Agent reply ended: {result['status']}") return result wrapper.reply = reply diff --git a/benchmark/pibench/bridge_reme.py b/benchmark/pibench/bridge_reme.py index dadd7877..8bc49509 100755 --- a/benchmark/pibench/bridge_reme.py +++ b/benchmark/pibench/bridge_reme.py @@ -787,8 +787,10 @@ class ReMeBridge: result = await self.agent_wrapper.reply(user_message, **reply_kwargs) - reply_text = result.get("result", "") - session_id = result.get("session_id", "") + from reme.steps.evolve._evolve import agent_reply_result_text + + reply_text = agent_reply_result_text(result) + session_id = result["session_id"] if session_id: self.agent_session_id = session_id diff --git a/deploy/benchmark_docker/Dockerfile.base b/deploy/benchmark_docker/Dockerfile.base index b7e3ac81..17d7ece1 100644 --- a/deploy/benchmark_docker/Dockerfile.base +++ b/deploy/benchmark_docker/Dockerfile.base @@ -1,5 +1,5 @@ # Build from repository root: -# docker build -f deploy/benchmark_docker/Dockerfile.base -t reme-benchmark-base:agentscope-2.0.9 . +# docker build -f deploy/benchmark_docker/Dockerfile.base -t reme-benchmark-base:agentscope-2.0.11 . FROM python:3.11-slim ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ @@ -18,7 +18,7 @@ RUN mkdir reme \ && printf '__version__ = "0.0.0"\n' > reme/__init__.py \ && python -m pip install '.[core]' \ && python -m pip uninstall -y reme-ai \ - && python -c 'import agentscope; assert agentscope.__version__ == "2.0.9"' \ + && python -c 'import agentscope; assert agentscope.__version__ == "2.0.11"' \ && rm -rf /tmp/reme-dependencies WORKDIR /workspace diff --git a/deploy/benchmark_docker/Dockerfile.candidate b/deploy/benchmark_docker/Dockerfile.candidate index 62c3b743..fb78f043 100644 --- a/deploy/benchmark_docker/Dockerfile.candidate +++ b/deploy/benchmark_docker/Dockerfile.candidate @@ -1,5 +1,5 @@ # Build from repository root with this Dockerfile-specific ignore file. -ARG BASE_IMAGE=reme-benchmark-base:agentscope-2.0.9 +ARG BASE_IMAGE=reme-benchmark-base:agentscope-2.0.11 FROM ${BASE_IMAGE} COPY . /opt/reme diff --git a/deploy/benchmark_docker/README.md b/deploy/benchmark_docker/README.md index f080be60..6265266e 100644 --- a/deploy/benchmark_docker/README.md +++ b/deploy/benchmark_docker/README.md @@ -2,14 +2,18 @@ This package runs ReMe benchmark jobs directly inside isolated AgentScope Docker workspaces. It does not start the ReMe HTTP service. Use AgentScope -**2.0.9** on the host and in the image. +**2.0.11** on the host and in the image. + +Rebuild the base image when upgrading from an older AgentScope image; +existing images must also use 2.0.11 to pass candidate validation. New runs use +the `reme-benchmark-base:agentscope-2.0.11` tag by default, including Home evaluations. From the repository root: ```bash pip install -e '.[benchmark-docker]' docker build -f deploy/benchmark_docker/Dockerfile.base \ - -t reme-benchmark-base:agentscope-2.0.9 . + -t reme-benchmark-base:agentscope-2.0.11 . ``` `SourceCandidate` uploads an immutable source snapshot and installs the root @@ -57,6 +61,18 @@ case's workspace **as it stood immediately after construction**, or provide `workspace_source` pointing at a host workspace directory or a saved archive. A reused workspace may be queried without additional build jobs. `export_workspace_to` persists a portable snapshot beyond the pool run. +`run(..., on_case_finished=callback)` awaits an async `(plan, result)` callback +after each case releases its worker, allowing incremental host-side persistence +and judging. Final worker-close errors can still be attached afterward; reconcile +returned results before publishing a completed summary. Callback errors abort the +batch and trigger worker cleanup. `fail_fast=True` stops the batch on an execution +failure. An optional `accept_build` predicate allows trusted callers to define +partial-build acceptance; the default requires successful construction. +`CasePlan.export_diagnostics_to` requests a best-effort full archive on execution +failure before resetting a still-accessible worker. Interrupted containers may +only retain a cached build log; failed diagnostic export never replaces the +original error. `CaseResult.failure_stage` and `exception` identify execution +failures separately from ordinary structured build/query results. To build memory once and query it in later runs using the same factory: ```python @@ -137,6 +153,6 @@ explicitly enabled: REME_DOCKER_INTEGRATION=1 pytest tests/integration/test_benchmark_docker.py -v ``` -The host must have AgentScope 2.0.9 and a working Docker daemon. To use an +The host must have AgentScope 2.0.11 and a working Docker daemon. To use an already built base image, set `REME_BENCHMARK_TEST_IMAGE` to its tag; the tests do not remove a caller-supplied image. diff --git a/deploy/benchmark_docker/candidate.py b/deploy/benchmark_docker/candidate.py index e8d28909..05c01667 100644 --- a/deploy/benchmark_docker/candidate.py +++ b/deploy/benchmark_docker/candidate.py @@ -146,7 +146,7 @@ class SourceCandidate: """ snapshot: SourceSnapshot - base_image: str = "reme-benchmark-base:agentscope-2.0.9" + base_image: str = "reme-benchmark-base:agentscope-2.0.11" install_projects: tuple[str, ...] = (".",) def __post_init__(self) -> None: diff --git a/deploy/benchmark_docker/direct_docker_workspace.py b/deploy/benchmark_docker/direct_docker_workspace.py index 1db6e8a0..8c7e04ba 100644 --- a/deploy/benchmark_docker/direct_docker_workspace.py +++ b/deploy/benchmark_docker/direct_docker_workspace.py @@ -1,4 +1,4 @@ -"""Minimal AgentScope 2.0.9 DockerWorkspace for direct ReMe jobs. +"""Minimal AgentScope 2.0.11 DockerWorkspace for direct ReMe jobs. The stock workspace builds and starts an MCP gateway. The benchmark harness uses only DockerBackend's exec/read/write primitives, so building that gateway diff --git a/deploy/benchmark_docker/docker_sandbox.py b/deploy/benchmark_docker/docker_sandbox.py index c7452e00..4c25c256 100644 --- a/deploy/benchmark_docker/docker_sandbox.py +++ b/deploy/benchmark_docker/docker_sandbox.py @@ -7,12 +7,13 @@ from dataclasses import asdict from datetime import datetime, timezone import gzip import io +import inspect import json from pathlib import Path from pathlib import PurePosixPath import re import tarfile -from typing import Any, BinaryIO, Callable, Literal, Protocol +from typing import Any, Awaitable, BinaryIO, Callable, Literal, Protocol import uuid from ._cleanup import cleanup_on_error @@ -239,13 +240,13 @@ def _archive_workspace(source: str | Path) -> bytes: def _default_workspace_builder(workspace_id: str, image: str, env: dict[str, str]) -> Workspace: - """Create the AgentScope 2.0.9 Docker backend lazily.""" + """Create the AgentScope 2.0.11 Docker backend lazily.""" import agentscope from .direct_docker_workspace import DirectDockerWorkspace - if agentscope.__version__ != "2.0.9": - raise RuntimeError(f"benchmark Docker host requires agentscope 2.0.9, got {agentscope.__version__}") + if agentscope.__version__ != "2.0.11": + raise RuntimeError(f"benchmark Docker host requires agentscope 2.0.11, got {agentscope.__version__}") return DirectDockerWorkspace( workspace_id=workspace_id, base_image=image, @@ -272,8 +273,12 @@ class DockerReMeSandboxFactory: plugins: tuple[str, ...] | list[str] = (), command_timeout: float = 1800.0, workspace_builder: WorkspaceBuilder | None = None, + on_sandbox_started: Callable[[], None | Awaitable[None]] | None = None, ) -> None: self.candidate = candidate + self._on_sandbox_started = on_sandbox_started + self._sandbox_started_reported = False + self._sandbox_started_lock = asyncio.Lock() self.env = dict(env or {}) self.config = config self.plugins = tuple(plugins) @@ -309,6 +314,13 @@ class DockerReMeSandboxFactory: ) try: await case.initialize() + if self._on_sandbox_started is not None: + async with self._sandbox_started_lock: + if not self._sandbox_started_reported: + result = self._on_sandbox_started() + if inspect.isawaitable(result): + await result + self._sandbox_started_reported = True except BaseException as exc: await cleanup_on_error([(f"workspace {container_id!r}", workspace.close())], exc) raise @@ -489,9 +501,9 @@ class DockerReMeSandbox: ) result = await self._exec_checked("validate_candidate", [python, "-c", script]) validation = json.loads(_decode(result.stdout)) - if validation.get("agentscope") != "2.0.9": + if validation.get("agentscope") != "2.0.11": raise SandboxCommandError( - f"candidate environment requires agentscope 2.0.9, got {validation.get('agentscope')!r}", + f"candidate environment requires agentscope 2.0.11, got {validation.get('agentscope')!r}", ) candidate_root = PurePosixPath(CANDIDATE_ROOT) diff --git a/deploy/benchmark_docker/pool.py b/deploy/benchmark_docker/pool.py index 7e6f417a..09f867b0 100644 --- a/deploy/benchmark_docker/pool.py +++ b/deploy/benchmark_docker/pool.py @@ -9,6 +9,7 @@ from __future__ import annotations import asyncio from dataclasses import dataclass +from collections.abc import Awaitable, Callable from pathlib import Path from tempfile import TemporaryDirectory from typing import Any @@ -16,6 +17,19 @@ from typing import Any from ._cleanup import finish_cleanup, format_error from .docker_sandbox import DockerReMeSandbox, DockerReMeSandboxFactory from .models import EvaluationQuery, JobRequest +from .worker import _exception_details + + +def _exception_classification(exc: BaseException) -> dict[str, Any]: + """Retain only classification fields, omitting absent status codes and causes.""" + details = _exception_details(exc) + result = {key: details[key] for key in ("type", "module")} + status = details.get("status_code") + if isinstance(status, int) and not isinstance(status, bool): + result["status_code"] = status + if details["causes"]: + result["causes"] = details["causes"] + return result @dataclass(frozen=True, slots=True) @@ -30,6 +44,7 @@ class CasePlan: export_workspace_to: str | Path | None = None export_build_log_to: str | Path | None = None export_evaluation_to: str | Path | None = None + export_diagnostics_to: str | Path | None = None @dataclass(slots=True) @@ -42,6 +57,8 @@ class CaseResult: workspace_snapshot: Path | None = None build_log: str | None = None error: str | None = None + failure_stage: str | None = None + exception: dict[str, Any] | None = None @property def success(self) -> bool: @@ -79,7 +96,14 @@ class BenchmarkDockerPool: self.factory = factory self.concurrency = concurrency - async def run(self, plans: list[CasePlan]) -> list[CaseResult]: + async def run( + self, + plans: list[CasePlan], + *, + on_case_finished: Callable[[CasePlan, CaseResult], Awaitable[None]] | None = None, + accept_build: Callable[[dict[str, Any]], bool] | None = None, + fail_fast: bool = False, + ) -> list[CaseResult]: """Return results in input order, closing every container on exit.""" if not plans: return [] @@ -105,8 +129,7 @@ class BenchmarkDockerPool: # a container. The graph check above rules out deadlocks. tasks: dict[str, asyncio.Task[CaseResult]] = {} - async def execute(plan: CasePlan) -> CaseResult: - result = CaseResult(plan.case_id) + async def execute(plan: CasePlan, result: CaseResult) -> CaseResult: source = plan.workspace_source if plan.reuse_workspace_from is not None: parent = await tasks[plan.reuse_workspace_from] @@ -121,43 +144,83 @@ class BenchmarkDockerPool: return CaseResult(plan.case_id, error=slots.exhaustion_error) worker_results.setdefault(id(worker), []).append(result) + operation = "prepare" try: await worker.reset_case(plan.case_id) if source is not None: + operation = "restore" await worker.upload_workspace(source) if plan.build_jobs: + operation = "construction" try: result.build = await worker.run_build(list(plan.build_jobs)) except Exception as exc: result.error = format_error(exc) + result.failure_stage = operation + result.exception = _exception_classification(exc) await self._retain_failed_build_log(worker, plan, result) return result - if not result.build.get("success"): + accepted = accept_build(result.build) if accept_build else result.build.get("success") + if not accepted: result.error = "build failed" + result.failure_stage = operation await self._retain_failed_build_log(worker, plan, result) return result if plan.export_build_log_to is not None: + operation = "artifacts" await worker.export_build_log(plan.export_build_log_to) # Capture before queries, since answering may change the # workspace. Dependents receive construction state. if plan.export_workspace_to is not None or plan.case_id in reused_case_ids: + operation = "artifacts" destination = plan.export_workspace_to or snapshot_root / f"{plan.case_id}.tar.gz" result.workspace_snapshot = await worker.export_workspace(destination) if plan.queries: + operation = "answer" result.queries = await worker.run_queries(list(plan.queries)) if not result.queries.get("success"): result.error = "query failed" + result.failure_stage = operation if plan.export_evaluation_to is not None: + operation = "artifacts" await worker.export_evaluation(plan.export_evaluation_to) return result except Exception as exc: # Keep other cases running and release this worker. result.error = format_error(exc) + result.failure_stage = operation + result.exception = _exception_classification(exc) return result finally: + if result.error and plan.export_diagnostics_to is not None and worker.reusable: + try: + await worker.export_full(plan.export_diagnostics_to) + except Exception as exc: + result.error += f"; diagnostic export failed: {format_error(exc)}" await self._release_worker(worker, result, slots) + async def run_plan(plan: CasePlan) -> CaseResult: + result = CaseResult(plan.case_id) + try: + result = await execute(plan, result) + except asyncio.CancelledError: + result.error = result.error or "interrupted" + raise + finally: + if on_case_finished is not None: + await on_case_finished(plan, result) + failed = not result.success + if accept_build is not None and result.build is not None: + failed = ( + result.error is not None + or not accept_build(result.build) + or (result.queries is not None and not result.queries.get("success")) + ) + if fail_fast and failed: + raise RuntimeError(f"case {plan.case_id!r} failed: {result.error or 'execution failed'}") + return result + for plan in plans: - tasks[plan.case_id] = asyncio.create_task(execute(plan)) + tasks[plan.case_id] = asyncio.create_task(run_plan(plan)) try: results = await asyncio.gather(*(tasks[plan.case_id] for plan in plans)) finally: diff --git a/deploy/benchmark_docker/worker.py b/deploy/benchmark_docker/worker.py index c77ebaa8..51c88081 100644 --- a/deploy/benchmark_docker/worker.py +++ b/deploy/benchmark_docker/worker.py @@ -40,12 +40,30 @@ def _exception_details(exc: BaseException) -> dict[str, Any]: "message": str(exc), } response = getattr(exc, "response", None) - status_code = getattr(response, "status_code", None) + status_code = getattr(exc, "status_code", None) + if not isinstance(status_code, int): + status_code = getattr(response, "status_code", None) if isinstance(status_code, int): details["status_code"] = status_code error_number = getattr(exc, "errno", None) if isinstance(error_number, int): details["errno"] = error_number + causes = [] + seen = {id(exc)} + current = exc + while True: + current = current.__cause__ or (None if current.__suppress_context__ else current.__context__) + if current is None or id(current) in seen: + break + seen.add(id(current)) + cause = {"type": type(current).__name__, "module": type(current).__module__} + status = getattr(current, "status_code", None) + if not isinstance(status, int): + status = getattr(getattr(current, "response", None), "status_code", None) + if isinstance(status, int) and not isinstance(status, bool): + cause["status_code"] = status + causes.append(cause) + details["causes"] = causes return details @@ -158,6 +176,9 @@ async def _run_job_on_app(app: Any, job: str, arguments: dict[str, Any]) -> dict "metadata": response.metadata, "error": None if response.success else str(response.answer), } + exception = response.metadata.get("reme.job_error") + if not response.success and isinstance(exception, dict): + result["exception"] = exception result["token_usage"] = token_usage result["job_call_counts"] = {name: count for name, count in job_call_counts.items() if count} return result diff --git a/docs/en/framework.md b/docs/en/framework.md index fb6757fa..f2e95200 100644 --- a/docs/en/framework.md +++ b/docs/en/framework.md @@ -557,6 +557,29 @@ Current default components in `reme/config/default.yaml`: | `keyword_index` | `default` | `bm25` | BM25 keyword index. | | `file_store` | `default` | `local` | Combines file_graph and keyword_index; defaults to `embedding_store: ""`. | +All three agent backends execute through `reply_stream()`. `reply()` collects that same stream into an +[`AgentReply`](../../reme/schema/agent_reply.py) dictionary: `session_id`, final text in `result`, +`structured_output`, invocation-scoped `usage`, `status`, `error`, `raw`, and the complete ordered JSON `events`. +Both modes accept `output_schema`. Only `completed` is successful; `failed`, `interrupted`, and `waiting` preserve +incomplete outcomes. Setup/transport errors and caller cancellation propagate. Both modes record reported token +usage once per invocation; unavailable usage remains null. + +[`StreamChunk`](../../reme/schema/stream_chunk.py) uses `start`, append-only `delta`, and `end` operations. +Non-null start/end payloads replace a block's accumulated content; null leaves it unchanged, and an empty string +clears it. Session, turn, message, block, and tool-call IDs correlate updates. `metadata.native` retains complete +provider events, including unknown fields and diagnostic notifications. `reply_end.chunk` carries the outcome +fields above, excluding `events`; `reply_start`/`reply_end` delimit one invocation, while DONE only closes transport. +`collect_reply(stream)` and `replay_reply(reply)` convert between the async stream and JSON result without losing +payloads, order, native metadata, or terminal state. Collection fails if the terminal outcome is missing or repeated. + +`raw` preserves native results: AgentScope's final message in `raw.message` plus an optional +`raw.structured_response`, Claude's `ResultMessage`, or Codex's `TurnResult`. Persist the full AgentReply for exact +replay; native session history supplies item snapshots rather than the original stream's deltas. +Studio reconciles block snapshots, displays structured output and terminal states, and retains live events through +history refreshes. Raw HTTP `stream_chunks()` includes errors and DONE; CLI/text helpers handle errors separately. +Migration: replace `last_message`/`turn` with `result`/`raw`, and consume the terminal outcome instead of concatenating +all stream payloads. Existing text-only producers use the default `delta` operation. + Codex wrappers reference `app_server: default` or `app_server: oauth`. Multiple wrappers can share one server while keeping their own models, instructions and conversations. Omit `session_id` to start a new thread; pass a returned `session_id` to continue it. Different threads can execute concurrently; operations on one thread are serialized. diff --git a/docs/zh/framework.md b/docs/zh/framework.md index 1910b1b1..f84a13ea 100644 --- a/docs/zh/framework.md +++ b/docs/zh/framework.md @@ -538,6 +538,26 @@ flowchart LR | `keyword_index` | `default` | `bm25` | BM25 关键词索引 | | `file_store` | `default` | `local` | 组合 file_graph、keyword_index；默认 `embedding_store: ""` | +三个 agent 后端统一通过 `reply_stream()` 执行，`reply()` 收集同一条流，返回 +[`AgentReply`](../../reme/schema/agent_reply.py) 字典：`session_id`、最终文本 `result`、`structured_output`、 +本次调用的 `usage`、`status`、`error`、`raw` 和完整、有序的 JSON `events`。两种模式均支持 `output_schema`。 +只有 `completed` 表示成功；`failed`、`interrupted` 和 `waiting` 保留未完成的结果。启动、传输异常和调用方的 +取消继续传播。两种模式均按调用记录一次已报告的 token 用量；无法获取用量时为 null。 + +[`StreamChunk`](../../reme/schema/stream_chunk.py) 使用 `start`、追加内容的 `delta` 和 `end` 操作。 +start/end 的非 null 内容替换对应块的累计内容；null 表示不变，空字符串表示清空。会话、调用、消息、内容块和 +工具调用 ID 用于关联更新。`metadata.native` 保留完整原生事件，包括未知字段和诊断通知。 +`reply_end.chunk` 携带上述结果字段，不含 `events`；`reply_start`/`reply_end` 包含一次 agent 调用内的模型与工具 +循环，DONE 仅关闭传输。`collect_reply(stream)` 与 `replay_reply(reply)` 在异步流和 JSON 结果间无损转换，保留 +payload、顺序、原生元数据和终止状态；缺少或重复的终止结果会明确报错。 + +`raw` 保留原生结果：AgentScope 的最终消息在 `raw.message`，额外结构化结果在 `raw.structured_response`； +Claude 保留 `ResultMessage`，Codex 保留 `TurnResult`。需要原样重放时应保存完整 AgentReply；原生会话历史提供 +item 快照，不含原始流的逐次增量。Studio 合并内容快照，展示结构化输出和终止状态，历史刷新保留已有实时事件。 +HTTP `stream_chunks()` 完整返回错误事件和 DONE，CLI 与文本辅助函数单独处理报错。 +迁移：用 `result`/`raw` 替代 `last_message`/`turn`，消费终止结果，不再拼接所有流式 payload。 +纯文本生产者使用默认 `delta` 操作。 + Codex wrapper 通过 `app_server: default` 或 `app_server: oauth` 引用共享组件。多个 wrapper 可以共享同一个进程， 并分别配置模型、指令和会话。不传 `session_id` 时创建新 thread；传回已有 `session_id` 时续聊。 不同 thread 可以并发，同一 thread 的操作串行。 diff --git a/integrations/dsh/README.md b/integrations/dsh/README.md index faac78cc..0fe1be7b 100644 --- a/integrations/dsh/README.md +++ b/integrations/dsh/README.md @@ -4,7 +4,7 @@ This guide explains how to install, configure, and use `@agentscope-ai/reme-dsh-plugin` with DeepSeek Harness (DSH), including memory guidance injection, the `reme_search` tool, automatic memory, daily consolidation, and the ReMe Status page. -The screenshots record an earlier local integration test against DSH `0.1.5-rc.2`; the supported releases are `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, `0.2.1-alpha.4`, and `0.2.1-rc.2`. Both the interface and ReMe guidance are set to English, and the isolated DSH and ReMe workspaces contain only fictional Project Aurora data. No `.env` values, API keys, access tokens, or personal memories appear in the screenshots. +The screenshots record an earlier local integration test against DSH `0.1.5-rc.2`; the supported releases are `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, `0.2.1-alpha.4`, `0.2.1-rc.2`, and `0.2.2-alpha.2`. Both the interface and ReMe guidance are set to English, and the isolated DSH and ReMe workspaces contain only fictional Project Aurora data. No `.env` values, API keys, access tokens, or personal memories appear in the screenshots. ## 1. How the plugin works @@ -26,8 +26,8 @@ The DSH adapter injects **usage guidance**, not every historical memory. Relevan ## 2. Requirements - ReMe is installed and its configuration exposes the `search`, `auto_memory`, and `auto_dream` jobs. -- DeepSeek Harness `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, `0.2.1-alpha.4`, or `0.2.1-rc.2`. -- Node.js `^22.19.0` or `>=24.0.0`, matching the current DSH engine range. +- DeepSeek Harness `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, `0.2.1-alpha.4`, `0.2.1-rc.2`, or `0.2.2-alpha.2`. +- Node.js `^22.19.0` or `>=24.0.0`, matching the current DSH engine range (DSH `0.2.2` needs Node 24.9 or newer). - The browser running DSH can reach the configured ReMe HTTP endpoint. Cross-machine deployments must also allow the DSH browser origin. The default ReMe endpoint is `http://127.0.0.1:2333`. ReMe HTTP does not use API-key authentication, so do not expose it directly to an untrusted network. @@ -110,26 +110,43 @@ If an earlier upgrade already renamed `settings.yaml` to `settings.yaml.imported ![ReMe Memory plugin configuration](./figures/reme-memory-settings.png) -| UI meaning | Configuration key | Default | Description | -| ---------------------- | ---------------------- | ----------------------- | --------------------------------------------------------------- | -| Service URL | `endpoint` | `http://127.0.0.1:2333` | Absolute HTTP(S) URL without credentials, query, or fragment. | -| Guidance language | `language` | `en` | `en` or `zh`; controls guidance injected into new sessions. | -| Default search results | `searchLimit` | `5` | Default `reme_search` result limit, from 1 to 50. | -| Search timeout | `requestTimeoutMs` | `10000` | Search timeout in milliseconds, from 1,000 to 120,000. | -| Automatic memory | `autoMemoryEnabled` | `true` | Capture completed user/assistant turns for `auto_memory`. | -| Exclude subagents | `rootAgentsOnly` | `true` | Inject guidance and capture conversations only for root agents. | -| Submission interval | `autoMemoryInterval` | `5` | Submit after this many completed turns, from 1 to 1,000. | -| Memory consolidation | `autoDreamEnabled` | `true` | Run `auto_dream` on the daily schedule. | -| Consolidation schedule | `dreamCron` | `0 23 * * *` | Five-field cron expression interpreted in `timezone`. | -| Consolidation guidance | `dreamHint` | empty | Optional guidance passed to `auto_dream`. | -| Workspace timezone | `timezone` | `Asia/Shanghai` | IANA timezone used for batching and scheduling. | -| Background timeout | `backgroundTimeoutMs` | `3600000` | Timeout for `auto_memory` and `auto_dream`. | -| Shutdown flush timeout | `shutdownTimeoutMs` | `5000` | Budget for draining background work during shutdown. | -| Retry delay | `autoMemoryRetryMs` | `1000` | Initial retry delay; doubles after each failure. | -| Maximum retry delay | `autoMemoryRetryMaxMs` | `60000` | Upper bound for retry delay. | +| UI meaning | Configuration key | Default | Description | +| ---------------------- | ---------------------- | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| Service URL | `endpoint` | `http://127.0.0.1:2333` | Absolute HTTP(S) URL without credentials, query, or fragment. | +| Auth token (optional) | `authToken` | empty | Bearer token for a self-hosted gateway; set in the profile only (not shown in this form), stored in clear text, trusted networks only. | +| Guidance language | `language` | `en` | `en` or `zh`; controls guidance injected into new sessions. | +| Default search results | `searchLimit` | `5` | Default `reme_search` result limit, from 1 to 50. | +| Search timeout | `requestTimeoutMs` | `10000` | Search timeout in milliseconds, from 1,000 to 120,000. | +| Automatic memory | `autoMemoryEnabled` | `true` | Capture completed user/assistant turns for `auto_memory`. | +| Exclude subagents | `rootAgentsOnly` | `true` | Inject guidance and capture conversations only for root agents. | +| Submission interval | `autoMemoryInterval` | `5` | Submit after this many completed turns, from 1 to 1,000. | +| Memory consolidation | `autoDreamEnabled` | `true` | Run `auto_dream` on the daily schedule. | +| Consolidation schedule | `dreamCron` | `0 23 * * *` | Five-field cron expression interpreted in `timezone`. | +| Consolidation guidance | `dreamHint` | empty | Optional guidance passed to `auto_dream`. | +| Workspace timezone | `timezone` | `Asia/Shanghai` | IANA timezone used for batching and scheduling. | +| Background timeout | `backgroundTimeoutMs` | `3600000` | Timeout for `auto_memory` and `auto_dream`. | +| Shutdown flush timeout | `shutdownTimeoutMs` | `5000` | Budget for draining background work during shutdown. | +| Retry delay | `autoMemoryRetryMs` | `1000` | Initial retry delay; doubles after each failure. | +| Maximum retry delay | `autoMemoryRetryMaxMs` | `60000` | Upper bound for retry delay. | Deployment configuration also supports `REME_URL`, or `REME_HOST` together with `REME_PORT`. The timer-only test option `dreamIntervalMs` is intentionally excluded from user settings. +## 4.1 Environment management (Environment tab) + +The **Environment** tab (inside **Plugins → ReMe bundle → reme-memory → Configure**) covers the setup that +otherwise needs a separate terminal, on the machine that runs DSH: + +| Step | What it does | +| --------------- | ------------------------------------------------------------------------------------------------------------------------- | +| Local Python | Finds PEP 514, `py` launcher, `PATH` and conda interpreters, labels each origin and `reme-ai` state, stores `pythonPath` | +| ReMe dependency | Checks `reme-ai`, the `[core]` extras and the `reme` CLI; installs or upgrades `reme-ai[core]` with a live log and cancel | +| ReMe parameters | Edits the workspace `.env`; secrets stay in that file and are shown masked; provides an LLM connectivity test | +| Service control | Starts and stops the local ReMe service, showing PID, version and log | + +See the [Environment management guide](https://github.com/agentscope-ai/ReMe/blob/main/integrations/dsh/docs/environment.md) +for interpreter discovery and the recommendation rule, dependency installation, `.env` validation, connection +testing, service lifetime, platform support and troubleshooting. + ## 5. Memory context injection On `agent/created`, the plugin injects long-term-memory guidance as native plugin context. Expand **Context injection · reme-memory** in the message flow to inspect both the content and provenance. diff --git a/integrations/dsh/README_ZH.md b/integrations/dsh/README_ZH.md index e7d7cd77..67cf147b 100644 --- a/integrations/dsh/README_ZH.md +++ b/integrations/dsh/README_ZH.md @@ -26,8 +26,8 @@ DSH 启动新会话时，插件向根 Agent 注入一段“如何使用长期记 ## 2. 环境要求 - ReMe Python 服务已安装，且配置中提供 `search`、`auto_memory` 和 `auto_dream` Job。 -- DeepSeek Harness `0.1.7-rc.2`、`0.2.0-rc.2`、`0.2.1-alpha.2`、`0.2.1-alpha.4` 或 `0.2.1-rc.2`。 -- Node.js `^22.19.0` 或 `>=24.0.0`，与当前 DSH 的 engine 范围一致。 +- DeepSeek Harness `0.1.7-rc.2`、`0.2.0-rc.2`、`0.2.1-alpha.2`、`0.2.1-alpha.4`、`0.2.1-rc.2` 或 `0.2.2-alpha.2`。 +- Node.js `^22.19.0` 或 `>=24.0.0`，与当前 DSH 的 engine 范围一致（DSH `0.2.2` 需要 Node 24.9 以上）。 - DSH 页面能够访问 ReMe HTTP 地址；跨机器部署时还要允许 DSH 页面所在的浏览器 Origin。 ReMe HTTP 服务默认监听 `http://127.0.0.1:2333`，不使用 API Key 认证。因此不建议未经网络隔离直接暴露到公网。 @@ -115,26 +115,42 @@ dsh web --no-open --port 3090 截图中的测试配置使用 `http://127.0.0.1:3457`、English 指引、默认搜索数量 5、搜索超时 10 秒，并启用了自动记忆和“Exclude subagents”。完整字段如下： -| 界面含义 | 配置键 | 默认值 | 说明 | -| -------------------- | ---------------------- | ----------------------- | -------------------------------------------------------------------------- | -| 服务地址 | `endpoint` | `http://127.0.0.1:2333` | DSH 访问的绝对 HTTP(S) 地址，不支持 URL 内嵌凭据、查询参数或片段。 | -| 记忆指引语言 | `language` | `en` | `en` 或 `zh`；决定新会话中注入的指引语言。 | -| 默认搜索数量 | `searchLimit` | `5` | `reme_search` 未传 `limit` 时的默认结果上限，范围 1–50。 | -| 搜索超时（毫秒） | `requestTimeoutMs` | `10000` | `search` 请求的超时，范围 1,000–120,000。 | -| 自动记录已完成的对话 | `autoMemoryEnabled` | `true` | 是否捕获完成的用户/助手回合并提交给 `auto_memory`。 | -| 排除子 Agent | `rootAgentsOnly` | `true` | 开启时，只为根 Agent 注入指引并捕获对话。 | -| 自动记忆提交间隔 | `autoMemoryInterval` | `5` | 每完成多少轮对话提交一次，范围 1–1,000。退出时插件会在时间预算内尽力排空。 | -| 自动整理记忆 | `autoDreamEnabled` | `true` | 是否按照每日计划调用 `auto_dream`。 | -| 整理计划 | `dreamCron` | `0 23 * * *` | 五段 cron 表达式，按 `timezone` 解释；默认每天 23:00。 | -| 整理指引 | `dreamHint` | 空 | 可选的整理提示，原样交给 `auto_dream`。 | -| 工作区时区 | `timezone` | `Asia/Shanghai` | 有效的 IANA 时区，用于每日批次和 cron 计划。 | -| 后台任务超时（毫秒） | `backgroundTimeoutMs` | `3600000` | `auto_memory` 和 `auto_dream` 的最大等待时间。 | -| 退出等待（毫秒） | `shutdownTimeoutMs` | `5000` | DSH 退出时等待后台任务排空的时间预算。 | -| 失败重试间隔（毫秒） | `autoMemoryRetryMs` | `1000` | 首次失败后的等待时间，连续失败时翻倍。 | -| 最大重试间隔（毫秒） | `autoMemoryRetryMaxMs` | `60000` | 重试等待时间的上限。 | +| 界面含义 | 配置键 | 默认值 | 说明 | +| -------------------- | ---------------------- | ----------------------- | -------------------------------------------------------------------------------------------------- | +| 服务地址 | `endpoint` | `http://127.0.0.1:2333` | DSH 访问的绝对 HTTP(S) 地址，不支持 URL 内嵌凭据、查询参数或片段。 | +| 服务令牌（可选） | `authToken` | 空 | 自部署网关的 Bearer 令牌；仅通过 profile 配置（表单中不显示），明文保存，仅建议内网/受信环境使用。 | +| 记忆指引语言 | `language` | `en` | `en` 或 `zh`；决定新会话中注入的指引语言。 | +| 默认搜索数量 | `searchLimit` | `5` | `reme_search` 未传 `limit` 时的默认结果上限，范围 1–50。 | +| 搜索超时（毫秒） | `requestTimeoutMs` | `10000` | `search` 请求的超时，范围 1,000–120,000。 | +| 自动记录已完成的对话 | `autoMemoryEnabled` | `true` | 是否捕获完成的用户/助手回合并提交给 `auto_memory`。 | +| 排除子 Agent | `rootAgentsOnly` | `true` | 开启时，只为根 Agent 注入指引并捕获对话。 | +| 自动记忆提交间隔 | `autoMemoryInterval` | `5` | 每完成多少轮对话提交一次，范围 1–1,000。退出时插件会在时间预算内尽力排空。 | +| 自动整理记忆 | `autoDreamEnabled` | `true` | 是否按照每日计划调用 `auto_dream`。 | +| 整理计划 | `dreamCron` | `0 23 * * *` | 五段 cron 表达式，按 `timezone` 解释；默认每天 23:00。 | +| 整理指引 | `dreamHint` | 空 | 可选的整理提示，原样交给 `auto_dream`。 | +| 工作区时区 | `timezone` | `Asia/Shanghai` | 有效的 IANA 时区，用于每日批次和 cron 计划。 | +| 后台任务超时（毫秒） | `backgroundTimeoutMs` | `3600000` | `auto_memory` 和 `auto_dream` 的最大等待时间。 | +| 退出等待（毫秒） | `shutdownTimeoutMs` | `5000` | DSH 退出时等待后台任务排空的时间预算。 | +| 失败重试间隔（毫秒） | `autoMemoryRetryMs` | `1000` | 首次失败后的等待时间，连续失败时翻倍。 | +| 最大重试间隔（毫秒） | `autoMemoryRetryMaxMs` | `60000` | 重试等待时间的上限。 | 部署层还支持 `REME_URL`，或组合使用 `REME_HOST` 与 `REME_PORT`。仅供定时器测试的 `dreamIntervalMs` 不出现在用户设置中。 +## 4.1 环境管理（Environment 标签） + +**插件 → ReMe 插件包 → reme-memory → 配置 → 环境管理** 把「找 Python、装 ReMe、写凭据、启停服务」收进插件本体，无需另开终端； +这些操作都发生在运行 DSH 的那台机器上： + +| 步骤 | 做什么 | +| ---------------- | ------------------------------------------------------------------------------------------------------------ | +| 本地 Python 环境 | 发现 PEP 514 注册表、`py` 启动器、`PATH` 与 conda 解释器，标注来源与 `reme-ai` 状态，选中即写入 `pythonPath` | +| ReMe 依赖 | 检测 `reme-ai`、`[core]` 依赖与 `reme` CLI；可一键安装/升级 `reme-ai[core]`，日志实时可见、可取消 | +| ReMe 参数 | 编辑 workspace 的 `.env`：密钥只保存在该文件、界面仅显示掩码；提供 LLM 连通性测试 | +| 服务控制 | 启停本机 ReMe 服务，显示 PID、版本与日志 | + +解释器发现与推荐规则、依赖安装、`.env` 校验、连接测试、服务生命周期、平台支持与故障排查见 +[环境管理指南](https://github.com/agentscope-ai/ReMe/blob/main/integrations/dsh/docs/environment_ZH.md)。 + ## 5. 普通对话中的 memory 上下文注入 创建一个新会话后，插件监听 DSH 的 `agent/created`，把长期记忆使用规则作为一条原生 plugin context 注入。展开消息流中的 **上下文注入 · reme-memory** 可以直接检查内容与来源元数据。 diff --git a/integrations/dsh/docs/environment.md b/integrations/dsh/docs/environment.md new file mode 100644 index 00000000..484f73c5 --- /dev/null +++ b/integrations/dsh/docs/environment.md @@ -0,0 +1,272 @@ +# Environment management + +The **Environment** tab is the setup surface for a local ReMe install. It finds a Python interpreter on the +machine that runs DSH, installs `reme-ai[core]` into it, writes the workspace `.env`, and starts or stops the +local ReMe service — the work that otherwise needs several terminals. + +Open it from **Plugins → ReMe bundle → reme-memory → Configure → Environment**. + +The tab manages a **loopback** endpoint at the origin root (`http://127.0.0.1:…` or `http://localhost:…`). Any +other service URL — a remote host, or a gateway mounted under a path prefix — is only reported as answering or +not, and hides the local steps. + +Everything the tab reports comes from the DSH **Host**, not from the browser: Python discovery, file access, +the LLM probe and the service health check all run on the Host machine. A browser on another machine therefore +sees the Host's environment, and the hero and the service card can never disagree about it. + +## What each step does + +| Section | Capability | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Local Python | Discovers PEP 514 registry entries, the `py` launcher, `PATH` and conda (base and named environments), labels their origin and `reme-ai` state, and stores the selection in `pythonPath` | +| ReMe dependency | Checks the `reme-ai` version, the `[core]` extras and the `reme` CLI for the selected interpreter; installs or upgrades `reme-ai[core]` with an optional package index, a live log and cancel | +| ReMe parameters | Edits the workspace `.env`; secrets stay in that file and are shown masked. Saving requires a non-empty `EMBEDDING_API_KEY` (use `EMPTY` when vector search is off). Provides an LLM connectivity test | +| Service control | Starts and stops the local ReMe service, showing PID, version and log; the launch injects UTF-8 and `NO_PROXY`, sets `enable_logo=false`, and runs inside the workspace so `.env` discovery works | + +The hero summarises the same state as three chips — service, Python, ReMe — plus the endpoint, the detection +time and the two refresh actions. + +## Python discovery + +ReMe needs **Python 3.11 or newer**. The tab probes these channels on the Host and keeps only interpreters that +answered its verification snippet (a `-c` probe reporting `sys.executable`, the version and the base prefix): + +- Windows: PEP 514 registry entries (`HKLM`/`HKCU`), the `py` launcher list, `PATH`, and `conda env list`. +- macOS and Linux: `PATH` (`which -a`), the well-known `/usr/bin`, `/usr/local`, Homebrew (`/opt/homebrew`), + `~/.pyenv/shims` and `~/.local/bin` locations, and `conda env list`. + +Microsoft Store placeholder links under `WindowsApps` are skipped, because they are shell aliases rather than +interpreters. When conda is present, named environments are listed both under `conda env list` and as +interpreters, and the conda root is reported separately. + +Detection is cached for 60 seconds; **Refresh** (in the hero) probes again. The recommended interpreter is +chosen in this order: + +1. A named conda environment (`base` excluded) that already has a supported `reme-ai`. +2. Another interpreter that already has a supported `reme-ai`. +3. Any named conda environment, which is ready for a ReMe install. +4. The first usable interpreter. + +Hover the **recommended** tag to see which rule applied. Picking another interpreter writes `pythonPath` and +nothing else; leaving it unset keeps following the recommendation. The list shows six rows and expands on +demand, with a filter over version, origin and path. + +## ReMe dependency + +The check asks the selected interpreter three questions and reports them as metrics: + +- `reme-ai` metadata version (`importlib.metadata.version("reme-ai")`), and whether it satisfies the minimum + this integration targets. +- `[core]` extras: whether `import agentscope` works. Supported ReMe releases import AgentScope while loading the + CLI, so it is required to start the service. Install or reinstall `reme-ai[core]` when it is missing. +- `reme` CLI: the console script beside the interpreter, or the Python wrapper fallback used at start time. + +**Install reme-ai[core]** runs `python -m pip install reme-ai[core]`, optionally with a package index (PyPI, +Tsinghua, Aliyun). **Upgrade** adds `-U`; **Reinstall** adds `--force-reinstall`, because pip otherwise reports a +present-but-broken install as "already satisfied". Large wheels are downloaded on first use; expect 5–20 minutes +depending on the network. + +The installation is a Host-owned job: + +- It keeps running when the configuration page closes, and reopening the tab restores it for the same + interpreter. The Host looks the job up by interpreter, so two interpreters installing at the same time each + see their own progress. +- Progress is polled by job id, so a second installation for another interpreter cannot replace what you see. + The install answer is bound to the interpreter that asked for it, so a late response after an interpreter + switch is dropped instead of showing another interpreter's job. +- A successful install re-checks `reme-ai` for that interpreter, so the dependency card and **Start service** + follow the install without a manual **Check**. +- **Cancel** stops pip and its build children, and the interpreter stays locked until the whole managed process + group has exited. If pip exits while a build child keeps running, the group is force-killed and the job waits + for the group to empty. If even that is not confirmed, the job stays _cancelling_ and the interpreter stays + locked, because a second installation must not run while the first is still unaccounted for. The tab keeps + showing the job instead of reporting a completed cancellation. +- The log is written to `$DSH_HOME/reme-memory/install-.log` and its tail is shown in place. Each install + opens a new file and only the newest 20 are kept; the long-lived service log rotates to `service.log.1` once it + passes 5 MB. +- When the plugin unloads, running jobs are stopped through the same path: a graceful signal first, then a + force-kill, and the managed process group is watched until it is empty, so a pip that ignores `SIGTERM` — or a + build child that outlives pip — cannot outlive the Host. + +## Workspace `.env` + +ReMe reads credentials from the `.env` in the workspace directory at service start; the tab never stores them in +DSH settings. + +| Group | Keys | +| --------- | -------------------------------------------------------------------------------------- | +| LLM | `LLM_BACKEND`, `LLM_MODEL_NAME`, `LLM_API_KEY`, `LLM_BASE_URL` | +| Embedding | `EMBEDDING_BACKEND`, `EMBEDDING_MODEL_NAME`, `EMBEDDING_API_KEY`, `EMBEDDING_BASE_URL` | + +- The workspace directory defaults to `~/.reme` and must be absolute; changing it reloads the `.env` it points at. +- API keys are write-only: the Host returns a mask (last four characters) and a blank field means “keep the + stored key”. Leaving a key untouched never rewrites it. +- Emptying a **secret** field and saving keeps the stored key, even when a replacement was typed first and then + cleared: a credential cannot be deleted by emptying its control, so the blank field always means “keep what is + stored”. Emptying a **non-secret** field and saving **clears** it. Only fields you edited are written, so the rest + of the file keeps its comments, unknown keys and order. +- The two backend selections are the non-secret fields with a default: clearing one writes the default + (`LLM_BACKEND=openai`, `EMBEDDING_BACKEND=openai`) instead of an empty value. ReMe applies + `${LLM_BACKEND:-openai}` and `${EMBEDDING_BACKEND:-openai}` only when the variable is absent, so an empty + value would name no backend at all and the service would refuse to start. +- A save writes the edits that were submitted and leaves newer ones alone: typing another field while the save is in + flight keeps that text, and the card says that unsaved edits are still pending. +- Saving validates the whole merged file, not just the values you typed: ReMe 0.4.0.x instantiates the embedding + component unconditionally, so `EMBEDDING_API_KEY` must be non-empty even without vector search (`EMPTY` is the + documented placeholder). A duplicated key is rewritten in every position, and the file keeps its own line + ending. +- The write is atomic and owner-only (0600 on POSIX); the previous file is kept as `.env.bak`, so a failed save + cannot lose unrelated keys. +- **Test LLM connection** runs a one-token request from inside the workspace with the selected interpreter, and + adds a hint for `401`/`403` (key), `404` (model or base URL) and network errors. The request follows the + protocol of the `LLM_BACKEND` in the `.env`: `openai`, `dashscope`, `deepseek` and `moonshot` use the + OpenAI-compatible `/chat/completions` (Bearer auth), `anthropic` uses `/v1/messages` (`x-api-key`), `gemini` + uses `/v1beta/models/:generateContent` (`x-goog-api-key`), `ollama` uses `/api/chat` (a local server + needs no key) and `xai` uses the OpenAI-compatible `/chat/completions` at `api.x.ai`. +- The base URL follows ReMe's own credentials: `LLM_BASE_URL` is read for the OpenAI-compatible providers and for + Anthropic, while Gemini, Ollama and xAI carry their own endpoint and ignore it. A stale value left in the `.env` + therefore cannot send the probe to a host the service never calls, and a configuration ReMe accepts is never + reported as broken. The base URL is used as the provider's own SDK builds it, so a base that already carries the + version suffix fails in the probe exactly as it fails in the service. +- ReMe reaches xAI through its gRPC SDK, so the xAI probe is a REST-level approximation of that channel. +- The probe reads the `.env` on disk, so a configuration that is only picked in the form is not tested before the + file is saved; the button says so while it waits. +- A `.env` that sets `LLM_BACKEND` to an empty value is reported as an incomplete configuration + (`LLM config incomplete (LLM_BACKEND)`) and no request is sent: ReMe would start with no backend at all, so the + probe must not test a configuration the service cannot use. A `.env` without the key is probed with the default + backend, exactly as ReMe would start it. +- A known backend written in the wrong case (`LLM_BACKEND=OpenAI`) is reported as invalid, with the spelling to + use: ReMe's backend registry is case-sensitive and refuses that name. A backend name the plugin has never heard + of is still probed tolerantly — the list here is a snapshot, and ReMe may know more than the plugin does. That + holds however the name looks: only a backend the plugin actually knows can be reported as a wrong spelling. +- The draft, the save message and the LLM-test result belong to the workspace they were produced for: switching + the workspace disposes the section's state, so neither a key typed for one workspace nor a success reported for + it can be carried into another. + +## Connection testing + +**Test connection** in the hero asks the Host to `POST /health_check` on the configured endpoint. A healthy answer +must contain a ReMe health report — a version, a health flag and its components — so a generic HTTP server that +answers `200` with `{}` is not reported as a running ReMe service. The Host reuses the same ReMe client as the rest +of the plugin, so the request, the authentication and the three-second timeout are the ones the service itself is +called with. When the settings carry an `authToken`, the Host sends it as `Authorization: Bearer`, scoped to that +endpoint. + +## Service lifetime + +Starting the service here is equivalent to running `reme start` in a terminal — with one deliberate difference: +**the service outlives DSH**. Closing DSH does not stop it, and stopping it is an explicit action in the tab. + +That is why the plugin does not hand the service to DSH's local subprocess provider, which terminates the +processes it manages when the Host exits. The plugin owns a small service adapter instead: + +- The launch is recorded in `$DSH_HOME/reme-memory/service.json` (pid, endpoint, workspace, command, start time) + **after** the OS acknowledged the spawn, so a missing interpreter leaves no state behind. +- A service is reported as managed — and offered a **Stop** button — only when the record matches the endpoint on + screen, the pid is still alive, its image is still a Python or `reme` executable, **and** its command line still + carries the recorded `workspace_dir` and `service.port`. The command line rules out most recycled pids (a longer + workspace path or another port never matches), though it cannot tell a pid apart from a user who starts the exact + same command by hand. A record for port 2333 never claims a service on port 2444. +- When the evidence cannot be read (localized `tasklist`/`ps`, missing permissions), ownership stays unproven. Start + and Stop both leave the process and the record alone: Stop refuses, and Start reports an error instead of dropping + the record. Discarding it would orphan a live service that the tab could never stop again. The card then offers + **Forget this record**, which deletes only `service.json` and never signals the process; the user confirms what to do + with it themselves. The Host names the record that blocked the action, and **Forget** targets exactly that record — + which may belong to another port, not to the endpoint on screen — together with its pid. When the record changed in + the meantime, the Host refuses instead of dropping the new record silently. +- A service that occupies the port but fails the health check is reported as a conflict; if it is provably this + plugin's service, the tab still offers **Stop**. +- A conflict caused by a **recorded** process whose identity cannot be read is a different case: the process is + never signalled, so the tab offers **Forget this record** instead. The record travels with the status, so the way + out does not depend on a start that the conflict card disables. +- A record left by an older version under a path prefix of the same origin (for example + `127.0.0.1:2333/reme`) describes the **same** local service, because the launch configures only host + and port. When it can be proven to be this plugin's service, such a record is re-keyed to the endpoint on + screen and then managed like any other launch: **Stop** is offered, and **Start** replaces it while it is + not healthy. It is then not reported as a foreign process the tab cannot act on, which would leave the + user with a process to kill by hand. +- **Stop** tree-kills the recorded process (process group on POSIX, `taskkill /T /F` on Windows), waits for it to + exit, escalates (POSIX: `SIGTERM` then `SIGKILL`; Windows: force `taskkill` again), and clears the record only + after the pid and the POSIX process group are gone. A child that outlives its parent still receives the force-kill. + If the group cannot be confirmed empty, its record is retained; a later unverifiable group is never signalled again. + A service started elsewhere stays untouched and is only observed. +- A recorded launch that is still alive but never became healthy is shown as **Last launch not ready** with both + **Stop** and **Start**: Stop clears the record, while Start makes the Host stop a provably owned previous launch or + discard a proven-foreign one. It is not shown as "Starting" unless a start from this tab is actually in flight. +- **Start** is serialized: two requests cannot spawn two services, and a second request observes the launch + already in flight. Before replacing a recorded launch that never became healthy, Start only kills it when + ownership is proven; an unproven record is kept and reported, never silently dropped. The selected interpreter's + dependencies are checked before stopping the previous service. + Status reads before spawn acknowledgement keep the launch in flight. If a newer service record appears during + probing or spawning, it is preserved and any unrecorded new child is stopped before reporting failure. + Service-record read errors abort the operation instead of being treated as an absent record. +- **Auto-start** (`autoStart`, off by default) pulls the service up when DSH starts, if it is not already running. + A refused launch is reported in the DSH Host log. + +Python probes and pip installs are bounded Host operations; the service is the only process that must outlive DSH. + +## Platform support + +| Area | Notes | +| ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| DSH | `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, `0.2.1-alpha.4`, `0.2.1-rc.2`, `0.2.2-alpha.2` | +| Node.js | `^22.19.0` or `>=24.0.0`; current DSH releases need Node 24.9 or newer, and the compatibility matrix is validated with Node 24.16 | +| Windows | Tested on Windows 11 with Miniconda: registry/PATH/conda discovery, `tasklist`/`taskkill` for identity and stop, `PYTHONUTF8`/`PYTHONIOENCODING` for pip output | +| macOS / Linux | Discovery, detached spawn, identity lookup and stop are covered by unit tests that run a real child process; still less exercised on real machines than Windows | + +## Troubleshooting + +**No usable Python found.** Install Miniconda or any Python 3.11+, then press **Refresh**. Store placeholders +are filtered by design, so install a real distribution instead of the Store alias. + +**The selected interpreter is not in the list.** The tab shows it with a “not detected” tag and still uses it. +Refresh, or pick a detected interpreter. + +**Installation fails.** Open the log tail: missing wheels, a stale proxy or a restricted network usually show +there. Try a mirror. In a conda environment `pip` works, but a dedicated environment avoids dependency conflicts +with other projects. + +**Saving `.env` is refused.** `LLM_API_KEY` and `EMBEDDING_API_KEY` must both be non-empty in the saved file; +use `EMPTY` for the embedding key when vector search is off. Emptying a secret field never reaches the file, so +the stored key is kept; a required key that is not stored at all still fails the save, and the file is left +unchanged. Only a missing file is treated as a first save; permission or I/O read errors abort the save without +replacing the active file or its backup. Names must start with a letter or underscore and contain only letters, digits and underscores; values must be single-line strings; +invalid updates leave both files unchanged. + +**The service reports “Port occupied”.** Something else listens on the endpoint. Change the port in the service +URL (Memory settings) or stop the other program. + +**The service does not become healthy.** The Start action polls for 120 seconds. If the port is answered but +`/health_check` never passes, the card switches to the conflict state so **Stop** stays available. Use **View log** +(the workspace `service.log`) for the startup trace — a missing `reme-ai`, an invalid `.env`, or a port conflict all +appear there. + +**The service shows “Last launch not ready”.** A launch recorded by the plugin is still alive but never answered +`/health_check` (for example DSH closed while it was booting). Use **Stop** to clear it and start again. If the pid +was recycled by another program, that record is provably foreign: **Start** discards it and launches a fresh service. +If the identity cannot be verified at all, use **Forget this record** — it deletes only the plugin's record and +leaves the process untouched. + +**The service shows as foreign.** It was started outside the plugin, so it is observed but not managed. Stop it +in its own terminal if you want the tab to manage the next launch. + +**Stop is disabled.** The running service does not belong to this endpoint, or its record is gone. Restart it +from the tab to make it managed. + +**LLM test fails.** The result distinguishes `401`/`403` (key), `404` (model name or base URL) and network +errors. The probe reads the workspace `.env`, so save before testing, and it follows the `LLM_BACKEND` there: a +native provider (Anthropic, Gemini, Ollama) is probed with its own protocol, not with an OpenAI-style request. + +## Where the code lives + +```text +integrations/dsh/src/environment/ +├── types.ts shared, browser-safe RPC types +├── env-fields.ts the managed `.env` keys and the rules for updating them +├── dependency.ts what one interpreter can host +├── endpoint-rules.ts the endpoints a local service can be managed on +├── host/ detection, install, service adapter, manager, gateway +└── client/ the tab, one module per step, copy and styles +``` + +The Host modules are the only place that touches processes or files; the client renders what the Host returns. diff --git a/integrations/dsh/docs/environment_ZH.md b/integrations/dsh/docs/environment_ZH.md new file mode 100644 index 00000000..6cba71e9 --- /dev/null +++ b/integrations/dsh/docs/environment_ZH.md @@ -0,0 +1,217 @@ +# 环境管理 + +**Environment（环境管理）** 标签页用于在本机完成 ReMe 的准备工作：找到运行 DSH 那台机器上的 Python、把 +`reme-ai[core]` 装进该解释器、写入 workspace 的 `.env`、并启动或停止本地 ReMe 服务——这些原本需要在多个终端里完成。 + +入口：**插件 → ReMe Bundle → reme-memory → 配置 → 环境管理**。 + +该标签页只管理**根路径上的回环地址**（`http://127.0.0.1:…` 或 `http://localhost:…`）。其他服务地址——远程主机，或挂在 +路径前缀下的网关——只显示该地址是否可用，并隐藏本地步骤。 + +页面上的所有结果都来自 DSH **Host**，而不是浏览器：Python 探测、文件读写、LLM 探测、服务健康检查都在 Host 上执行。因此即使 +浏览器在另一台机器上，看到的也是 Host 的真实环境，顶部概览与服务卡片不会给出不一致的结论。 + +## 每一步做什么 + +| 分组 | 能力 | +| ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| 本地 Python | 探测 PEP 514 注册表、`py` 启动器、`PATH` 与 conda（base 与命名环境），标注来源和 `reme-ai` 状态，选择结果写入 `pythonPath` | +| ReMe 依赖 | 针对所选解释器检查 `reme-ai` 版本、`[core]` 附加依赖与 `reme` CLI；可选用软件源安装或升级 `reme-ai[core]`，显示实时日志并支持取消 | +| ReMe 参数 | 编辑 workspace 的 `.env`；密钥只存在该文件中，界面仅显示掩码。保存要求 `EMBEDDING_API_KEY` 非空（关闭向量检索时用 `EMPTY`）。提供 LLM 连通性测试 | +| 服务控制 | 启动/停止本地 ReMe 服务，显示 PID、版本与日志；启动时注入 UTF-8 与 `NO_PROXY`、设置 `enable_logo=false`，并在 workspace 内运行以保证 `.env` 能被发现 | + +顶部的概览区用三个标签（服务、Python、ReMe）概括同一份状态，并显示服务地址、探测时间与两个刷新操作。 + +## Python 探测 + +ReMe 需要 **Python 3.11 或更新版本**。标签页在 Host 上探测下列来源，只保留能通过验证片段（上报 `sys.executable`、 +版本号与 base prefix 的 `-c` 探测）的解释器： + +- Windows：PEP 514 注册表项（`HKLM`/`HKCU`）、`py` 启动器列表、`PATH`、`conda env list`。 +- macOS / Linux：`PATH`（`which -a`）、常见的 `/usr/bin`、`/usr/local`、Homebrew（`/opt/homebrew`）、 + `~/.pyenv/shims` 与 `~/.local/bin` 路径，以及 `conda env list`。 + +`WindowsApps` 下的 Microsoft Store 占位符会被跳过，因为它们只是别名而不是解释器。装有 conda 时，命名环境会同时出现在 +`conda env list` 与环境列表中，conda 根目录单独上报。 + +探测结果缓存 60 秒，点击概览区的**重新探测**会重新执行。推荐解释器按以下顺序选出： + +1. 已安装受支持 `reme-ai` 的命名 conda 环境（排除 `base`）。 +2. 其他已安装受支持 `reme-ai` 的解释器。 +3. 任意命名 conda 环境（可直接安装 ReMe）。 +4. 第一个可用解释器。 + +把鼠标悬停在**推荐**标签上可以看命中的是哪条规则。选择其它解释器只会写入 `pythonPath`；保持未选择则一直跟随推荐。列表默认 +显示 6 行并可展开，支持按版本、来源、路径筛选。 + +## ReMe 依赖 + +检查会向所选解释器询问三件事，并以指标形式展示： + +- `reme-ai` 元数据版本（`importlib.metadata.version("reme-ai")`），以及是否达到本集成要求的最低版本。 +- `[core]` 附加依赖：`import agentscope` 是否可用。受支持的 ReMe 版本在加载 CLI 时就会导入 AgentScope，因此缺失时 + 无法启动服务；请安装或重装 `reme-ai[core]`。 +- `reme` CLI：解释器旁的命令行入口，或启动时回退使用的 Python 包装。 + +**安装 reme-ai[core]** 执行 `python -m pip install reme-ai[core]`，可附加软件源（PyPI、清华、阿里云）。**升级**追加 +`-U`；**重装**追加 `--force-reinstall`，否则 pip 会把“存在但已损坏”的安装报成 “already satisfied”。首次使用需要下载较大的 +依赖，视网络情况通常 5–20 分钟。 + +安装是 Host 侧的任务： + +- 关闭配置页后仍在运行；重新打开标签页会为同一个解释器恢复进度。Host 按解释器查找正在运行的任务，所以两个解释器同时安装时， + 各自看到的是自己的进度。 +- 进度按任务 ID 轮询，为另一个解释器发起的安装不会顶替当前显示；安装的响应也绑定在发起它的解释器上，切换解释器后迟到的响应会被丢弃。 +- 安装成功后会自动重新检测 `reme-ai`（同一个解释器），依赖卡片与**启动服务**按钮随之更新，无需手动点“检测”。 +- **取消**会同时终止 pip 及其构建子进程，且在**整个受管进程组**真正退出前该解释器保持占用：pip 先退出而构建子进程 + 仍在时，会对进程组强制结束并等待组清空。若连这一步也无法确认，任务仍保持“正在取消”状态、该解释器依然被占用—— + 在第一个安装还没有交代清楚之前，不允许第二个安装开始。页面会继续显示该任务，而不是把它报告成一次已完成的取消。 +- 日志写入 `$DSH_HOME/reme-memory/install-.log`，页面直接显示尾部。每次安装都会新建文件，目录只保留最近 + 20 个；长期运行的服务日志超过 5 MB 后轮转为 `service.log.1`。 +- 插件卸载时，仍在运行的安装走同一条终止路径：先发优雅信号，再强制结束，并对受管进程组持续确认，因此忽略 + `SIGTERM` 的 pip 或比 pip 活得更久的构建子进程都不会活得比 Host 更久。 + +## Workspace `.env` + +ReMe 在服务启动时读取 workspace 目录下的 `.env`；标签页从不把密钥写进 DSH 设置。 + +| 分组 | 键 | +| --------- | -------------------------------------------------------------------------------------- | +| LLM | `LLM_BACKEND`、`LLM_MODEL_NAME`、`LLM_API_KEY`、`LLM_BASE_URL` | +| Embedding | `EMBEDDING_BACKEND`、`EMBEDDING_MODEL_NAME`、`EMBEDDING_API_KEY`、`EMBEDDING_BASE_URL` | + +- workspace 目录默认 `~/.reme`，必须是绝对路径；修改后会重新读取它指向的 `.env`。 +- API Key 是只写的：Host 只返回掩码（末四位），留空表示保留已存的值。没有编辑过的键不会被重写。 +- 清空**密钥**字段并保存会保留已存的密钥——即便先输入过替代值再清空也是如此：凭据无法通过清空输入框来删除， + 因此留空始终表示“保留已存的值”。清空**非密钥**字段并保存即表示**删除**该值。只有编辑过的字段会被写入， + 文件其余部分的注释、未知键与顺序保持不变。 +- 两个后端选择是带默认值的非密钥字段：清空后会写入默认值（`LLM_BACKEND=openai`、`EMBEDDING_BACKEND=openai`）， + 而不是空值。ReMe 只在变量**不存在**时才应用 `${LLM_BACKEND:-openai}` 与 `${EMBEDDING_BACKEND:-openai}`， + 空值等于没有指定任何后端，服务将无法启动。 +- 保存只清除本次提交的那部分编辑，更晚的编辑会被保留：保存进行中再改动其它字段时，那份文本不会丢失，页面也会 + 提示仍有未保存的改动。 +- 保存时会校验合并后的整份文件，而不只是本次输入：ReMe 0.4.0.x 会无条件实例化 embedding 组件，因此即使不使用向量检索 + `EMBEDDING_API_KEY` 也必须非空（`EMPTY` 是文档化的占位值）。重复出现的键会在每一处被改写，文件原有的换行风格也会保留。 +- 写入是原子且仅属主可读写的（POSIX 上为 0600）；旧文件会保留为 `.env.bak`，因此一次失败的保存不会丢掉其它键。 +- **测试 LLM 连接**在 workspace 内、用所选解释器发送一次只需 1 个 token 的请求，并对 `401`/`403` + （密钥）、`404`（模型名或 Base URL）与网络错误给出提示。请求按 `.env` 里的 `LLM_BACKEND` 选择协议： + `openai`、`dashscope`、`deepseek`、`moonshot` 走 OpenAI 兼容的 `/chat/completions`（Bearer 认证）， + `anthropic` 走 `/v1/messages`（`x-api-key`），`gemini` 走 `/v1beta/models/:generateContent` + （`x-goog-api-key`），`ollama` 走 `/api/chat`（本地服务无需密钥），`xai` 走 `api.x.ai` 上 OpenAI 兼容的 + `/chat/completions`。 +- Base URL 的来源与 ReMe 自身的凭据保持一致：`LLM_BASE_URL` 只对 OpenAI 兼容族与 Anthropic 生效，Gemini、Ollama + 与 xAI 自带地址、并不读取该变量。因此 `.env` 里残留的旧值不会把探测引到服务根本不会访问的主机，ReMe 能接受的 + 配置也不会被报成故障。Base URL 的拼接方式与服务商 SDK 完全一致，因此已经带版本后缀的地址在探测与服务中都会 + 同样失败。 +- ReMe 通过 gRPC SDK 连接 xAI，因此 xAI 的探测只是该通道在 REST 层面的近似。 +- 探测读取的是磁盘上的 `.env`，所以只在表单里改过配置而未保存时不会发起测试；等待期间按钮会给出提示。 +- 若 `.env` 把 `LLM_BACKEND` 设为空值，探测会报告配置不完整（`LLM config incomplete (LLM_BACKEND)`）并且不发送 + 任何请求：此时 ReMe 启动时并没有任何后端，探测不能去测试服务根本无法使用的配置。若 `.env` 里没有这个键， + 探测会与 ReMe 一样使用默认后端。 +- 已知后端名写错了大小写（如 `LLM_BACKEND=OpenAI`）会被报告为无效，并给出正确写法：ReMe 的后端注册表区分大小写， + 会拒绝这个名字。而插件从未听说过的后端名仍被宽容地探测——这里的列表只是一份快照，ReMe 可能认识得更多。这条规则与名字 + 长什么样无关：只有插件确实认识的后端才会被判定为大小写写错。 +- 草稿、保存提示与 LLM 测试结果都属于产生它们的 workspace：切换 workspace 会随旧区块一并丢弃，因此为某个 workspace + 输入的密钥、或属于它的一次成功提示，都不会被带到另一个。 + +## 连接测试 + +概览区的**测试连接**让 Host 对配置的服务地址执行 `POST /health_check`。健康响应必须包含可解析的 ReMe 健康报告——版本、 +健康标记与各组件明细——所以只回 `200` 与 `{}` 的普通 HTTP 服务不会被当成运行中的 ReMe。Host 复用插件其余部分使用的 +同一个 ReMe 客户端，因此请求、鉴权与三秒超时都与服务本身被调用时一致。设置里填了 `authToken` 时，Host 会以 +`Authorization: Bearer` 发送，且只作用于该服务地址。 + +## 服务生命周期 + +在这里启动服务等价于在终端执行 `reme start`，但有一点是刻意的差异：**服务会比 DSH 活得更久**。关闭 DSH 不会停止它，停止 +它是在标签页里的一次显式操作。 + +因此本插件没有把服务交给 DSH 的本地子进程提供方——后者会在 Host 退出时终止它托管的进程。插件自带一个小型服务适配器： + +- 启动在操作系统确认 spawn 之后才记录到 `$DSH_HOME/reme-memory/service.json`（pid、地址、workspace、命令行、启动时间）， + 所以解释器缺失时不会留下残留状态。 +- 只有记录与当前地址一致、pid 仍存活、进程映像仍是 Python 或 `reme`，**且命令行仍包含记录的 `workspace_dir` 与 + `service.port`** 时，服务才被判定为“由本插件托管”并给出**停止**按钮。命令行能排除大多数 pid 复用（更长的 + workspace 路径、另一个端口都不会命中），但无法区分“用户照抄同一命令行手工启动”的同一服务。2333 端口上的记录绝不会 + 声称管理 2444 端口上的服务。 +- 证据读不到时（`tasklist`/`ps` 输出本地化、权限不足等）归属保持“未证明”：启动与停止都不会碰进程、也不会丢弃记录—— + 停止会拒绝，启动会返回错误。丢弃记录会让一个仍在运行的进程变成标签页再也停不掉的孤儿；此时卡片会给出**忘记这条记录**， + 它只删除 `service.json`、绝不向进程发信号，由用户自行确认后续处理。Host 会在结果里指明挡住操作的是哪一条记录， + **忘记**针对的正是那条记录（可能是另一个端口上的记录，而不是当前显示的地址），并带上它的 pid：记录在此期间被换掉时， + Host 会拒绝而不是静默丢弃新记录。 +- 服务占用端口但健康检查失败时显示为“端口被占用”；若能证明是本插件启动的，标签页仍给出**停止**按钮。 +- 占用端口的是**已记录**但身份无法验证的进程时情况不同：该进程绝不会被发信号，因此标签页改为给出**忘记这条记录**。 + 这条记录随状态一起返回，所以出路不依赖被“端口被占用”状态禁用的**启动**按钮。 +- 旧版本遗留在同一源（origin）某个路径前缀下（例如 `127.0.0.1:2333/reme`）的记录，描述的是**同一个**本地服务—— + 启动只配置 host 与端口。只要能证明它是本插件启动的，这种记录就会被重新登记到当前显示的地址，随后与其他启动 + 一样被管理：给出**停止**按钮，**启动**在它尚未健康时替换它。它不会再被当成标签页无从处理的外部进程，否则用户 + 只能自己去杀进程。 +- **停止**会树杀记录的进程（POSIX 上按进程组、Windows 上为 `taskkill /T /F`），等待其退出，必要时升级（POSIX： + `SIGTERM` 后 `SIGKILL`；Windows：再次强制 `taskkill`），并**在 pid 及 POSIX 进程组真正消失之后**才清除记录。 + 父进程先退出时，仍会强制结束存活的子进程。无法确认进程组已清空就保留记录；后续无法验证身份的进程组不会再次被发信号。 + 别处启动的服务不受影响，只会被观察。 +- 记录中的进程仍存活但从未就绪时，显示为**上次启动未就绪**，并同时给出**停止**与**启动**：停止负责清记录，启动则让 Host + 停掉可证明归属的旧启动、或丢弃已证明属于外部的记录。只有本标签页发起的启动仍在进行时才会显示“启动中”。 +- **启动**是串行化的：两个请求不会拉起两个服务，后到的请求会观察已在进行的启动。替换“一直没有就绪”的旧记录时，只有 + 归属被证明才会杀掉它；证明不了就保留记录并报错，不会静默丢弃。停止旧服务前会先检查所选解释器的启动依赖。 + 在操作系统确认启动之前，状态查询不会清除启动标记。探测或启动过程中出现更新的服务记录时，会保留新记录， + 并在报告失败前停止尚未登记的新子进程；读取记录失败会中止操作，不会按“没有记录”继续启动。 +- **自动启动**（`autoStart`，默认关闭）会在 DSH 启动时把尚未运行的服务拉起；被拒绝的启动会记入 DSH Host 日志。 + +Python 探测与 pip 安装都是有界的 Host 操作；服务是唯一需要比 DSH 活得更久的进程。 + +## 平台支持 + +| 方面 | 说明 | +| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| DSH | `0.1.7-rc.2`、`0.2.0-rc.2`、`0.2.1-alpha.2`、`0.2.1-alpha.4`、`0.2.1-rc.2`、`0.2.2-alpha.2` | +| Node.js | `^22.19.0` 或 `>=24.0.0`；当前 DSH 需要 Node 24.9 以上，兼容矩阵以 Node 24.16 验证 | +| Windows | 在 Windows 11 + Miniconda 上实测：注册表/PATH/conda 探测、`tasklist`/`taskkill` 身份校验与停止、`PYTHONUTF8`/`PYTHONIOENCODING` 保证 pip 输出不乱码 | +| macOS/Linux | 探测、分离启动、身份查询与停止均有运行真实子进程的单元测试覆盖；真机验证程度仍低于 Windows | + +## 常见问题 + +**未检测到可用的 Python。** 安装 Miniconda 或任意 Python 3.11+，再点**重新探测**。Store 占位符会被过滤，请安装真实的发行版 +而不是 Store 别名。 + +**所选解释器不在列表里。** 标签页会用“不在探测结果中”标注它，但仍然使用它。重新探测，或改选探测到的解释器。 + +**保存 `.env` 时读取失败。** 只有文件不存在才按首次保存处理；权限或 I/O 错误会中止保存，不会替换原文件或备份。变量名须以字母或下划线开头，其余只接受字母、数字和下划线；值必须是单行字符串；无效更新不会改动原文件或备份。 + +**安装失败。** 先看日志尾部：缺少 wheel、代理配置过期或网络受限通常都能看出来，可以换一个软件源。conda 环境里 `pip` 也可 +用，但独立环境更能避免与其它项目的依赖冲突。 + +**保存 `.env` 被拒绝。** 保存后的文件中 `LLM_API_KEY` 与 `EMBEDDING_API_KEY` 都必须非空；关闭向量检索时 embedding 用 +`EMPTY`。清空密钥字段不会写入文件，已存的密钥会被保留；而一个根本没有存过的必需键仍会让保存失败，文件保持原样。 + +**服务显示“端口被占用”。** 该端口被其它程序占用。请在服务地址（记忆设置）中换一个端口，或停止占用它的程序。 + +**服务迟迟未就绪。** 启动会轮询 120 秒；如果端口有响应但 `/health_check` 一直不通过，卡片会切到“端口被占用”状态，**停止**按钮 +始终可用。用**查看日志**（workspace 下的 `service.log`）看启动过程——缺少 `reme-ai`、`.env` 不合法、端口冲突都会体现在那里。 + +**服务显示“上次启动未就绪”。** 本插件记录的进程仍存活，但一直没有通过 `/health_check`（例如启动途中 DSH 被关闭）。点**停止** +清掉它后重新启动。若 pid 已被其它程序复用，这条记录可证明属于外部：点**启动**会丢弃记录并拉起新服务。若完全无法验证身份， +点**忘记这条记录**——它只删除本插件的记录，不会碰进程。 + +**服务显示为外部启动。** 它不是由本插件启动的，因此只被观察而不被管理。若希望在标签页里管理，先在它自己的终端里停止，再从 +这里启动。 + +**停止按钮不可用。** 运行中的服务不属于当前服务地址，或记录已丢失。从标签页重新启动一次即可纳入管理。 + +**LLM 测试失败。** 结果会区分 `401`/`403`（密钥）、`404`（模型名或 Base URL）与网络错误。探测读取的是 workspace 的 +`.env`，所以请先保存再测试；探测协议也取自其中的 `LLM_BACKEND`：原生后端（Anthropic、Gemini、Ollama）用自己的协议 +探测，不会被当成 OpenAI 兼容接口。 + +## 代码位置 + +```text +integrations/dsh/src/environment/ +├── types.ts 浏览器安全的共享 RPC 类型 +├── env-fields.ts 受管的 `.env` 键及其更新规则 +├── dependency.ts 一个解释器能承载什么 +├── endpoint-rules.ts 哪些服务地址可以作为本地服务管理 +├── host/ 探测、安装、服务适配器、manager、gateway +└── client/ 标签页：每个步骤一个模块，以及文案与样式 +``` + +只有 Host 模块会接触进程与文件；客户端只负责渲染 Host 返回的结果。 diff --git a/integrations/dsh/package.json b/integrations/dsh/package.json index 0ec3eaa9..99bd5f00 100644 --- a/integrations/dsh/package.json +++ b/integrations/dsh/package.json @@ -20,6 +20,7 @@ "cordis.patch.yml", "README.md", "README_ZH.md", + "docs", "figures" ], "dsh": { @@ -51,12 +52,12 @@ "typecheck": "tsc -p tsconfig.json --noEmit" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.4 || ^4.0.5-alpha.1", - "@deepseek-ai/dsh-client-ui-primitives": "^0.1.7-rc.2 || ^0.2.0-rc.2 || ^0.2.1-alpha.2", - "@deepseek-ai/dsh-llm": "^0.1.7-rc.2 || ^0.2.0-rc.2 || ^0.2.1-alpha.2", - "@deepseek-ai/dsh-tools": "^0.1.7-rc.2 || ^0.2.0-rc.2 || ^0.2.1-alpha.2", - "@deepseek-ai/dsh-typert-protocol": "^0.1.7-rc.2 || ^0.2.0-rc.2 || ^0.2.1-alpha.2", - "@deepseek-ai/schemastery": "~3.18.4 || ^3.18.5-alpha.1" + "@deepseek-ai/cordis": "~4.0.4 || ^4.0.5-alpha.1 || ~4.0.6-alpha.1", + "@deepseek-ai/dsh-client-ui-primitives": "^0.1.7-rc.2 || ^0.2.0-rc.2 || ^0.2.1-alpha.2 || ^0.2.2-alpha.2", + "@deepseek-ai/dsh-llm": "^0.1.7-rc.2 || ^0.2.0-rc.2 || ^0.2.1-alpha.2 || ^0.2.2-alpha.2", + "@deepseek-ai/dsh-tools": "^0.1.7-rc.2 || ^0.2.0-rc.2 || ^0.2.1-alpha.2 || ^0.2.2-alpha.2", + "@deepseek-ai/dsh-typert-protocol": "^0.1.7-rc.2 || ^0.2.0-rc.2 || ^0.2.1-alpha.2 || ^0.2.2-alpha.2", + "@deepseek-ai/schemastery": "~3.18.4 || ^3.18.5-alpha.1 || ~3.18.6-alpha.1" }, "peerDependenciesMeta": { "@deepseek-ai/cordis": { diff --git a/integrations/dsh/scripts/test-package.mjs b/integrations/dsh/scripts/test-package.mjs index 3e8f3f46..e42b5807 100644 --- a/integrations/dsh/scripts/test-package.mjs +++ b/integrations/dsh/scripts/test-package.mjs @@ -38,24 +38,62 @@ const releases = [ schema: "3.18.5-alpha.2", }, { dsh: "0.2.1-rc.2", cordis: "4.0.5", loader: "1.0.6", schema: "3.18.5" }, + { + dsh: "0.2.2-alpha.2", + cordis: "4.0.6-alpha.1", + loader: "1.0.7-alpha.1", + schema: "3.18.6-alpha.1", + }, ]; -async function run(command, args, cwd) { +// `npm` is `npm.cmd` on Windows, which execFile cannot launch directly; the +// script runs the npm CLI through node instead, using the path npm exported to +// this process. +const npmCommand = process.env.npm_execpath + ? [process.execPath, [process.env.npm_execpath]] + : ["npm", []]; + +async function run(command, args, cwd, label = "command") { try { return await execFileAsync(command, args, { cwd, maxBuffer: 10 * 1024 * 1024, }); } catch (error) { - process.stderr.write(error.stdout || ""); - process.stderr.write(error.stderr || ""); - throw error; + const stdout = String(error.stdout ?? ""); + const stderr = String(error.stderr ?? ""); + const combined = `${stdout}\n${stderr}`; + // The raw TAP buffer is far too large for a CI log and hides which step + // failed. Report the failing cases and a short tail instead. + const failures = combined + .split("\n") + .map((line) => line.trim()) + .filter((line) => /^not ok \d+ - /.test(line) || line.startsWith("✖ ")) + .slice(0, 20); + const tail = combined.trim().split("\n").slice(-40).join("\n"); + process.stderr.write( + `\n[test-package] ${label} failed (exit ${error.code ?? "error"})\n`, + ); + if (failures.length > 0) + process.stderr.write( + `[test-package] failing tests:\n${failures.join("\n")}\n`, + ); + if (tail) process.stderr.write(`[test-package] last output:\n${tail}\n`); + // A short error keeps Node's unhandled-rejection dump readable; the large + // stdout/stderr buffers stay out of it. + const failure = new Error(`${label} failed: ${error.message}`); + failure.code = error.code; + throw failure; } } +/** Run the npm CLI on the platform this check runs on. */ +function runNpm(args, cwd, label = "npm") { + return run(npmCommand[0], [...npmCommand[1], ...args], cwd, label); +} + try { - const { stdout } = await run( - "npm", + const { stdout } = await runNpm( [ "pack", "--json", @@ -64,6 +102,7 @@ try { temporaryDirectory, ], root, + "npm pack", ); const [result] = JSON.parse(stdout); const files = new Set(result.files.map(({ path: file }) => file)); @@ -72,6 +111,7 @@ try { "dist/client.js", "cordis.patch.yml", "README.md", + "docs/environment.md", "figures/reme-status-overview.png", ]) { assert.ok(files.has(file), `missing ${file}`); @@ -112,8 +152,7 @@ try { JSON.stringify({ private: true, type: "module", dependencies }), ); // Install as a consumer with normal peer resolution, then check the shipped artifact. - await run( - "npm", + await runNpm( [ "install", "--ignore-scripts", @@ -123,6 +162,7 @@ try { "--fetch-timeout=30000", ], consumer, + `install DSH ${release.dsh}`, ); await run( process.execPath, @@ -136,6 +176,7 @@ try { `, ], consumer, + `import DSH ${release.dsh}`, ); const installed = path.join(consumer, "node_modules", manifest.name); @@ -154,10 +195,11 @@ try { .map((file) => path.join("tests", file)); const tested = await run( process.execPath, - ["--test", ...testFiles], + ["--test", "--test-reporter=tap", ...testFiles], installed, + `tests DSH ${release.dsh}`, ); - await run("npm", ["run", "build"], installed); + await runNpm(["run", "build"], installed, `build DSH ${release.dsh}`); const count = /# tests (\d+)/.exec(tested.stdout)?.[1]; process.stdout.write( `DSH ${release.dsh}: packed import, source build, and ${count} tests passed\n`, diff --git a/integrations/dsh/src/client/index.tsx b/integrations/dsh/src/client/index.tsx index d14a0151..abe6ad80 100644 --- a/integrations/dsh/src/client/index.tsx +++ b/integrations/dsh/src/client/index.tsx @@ -1,8 +1,19 @@ -import { useState, useSyncExternalStore } from "react"; +import { useEffect, useRef, useState, useSyncExternalStore } from "react"; +import { SettingsFormModel } from "@deepseek-ai/dsh-client-ui-primitives"; +import type { + SettingsFieldState, + SettingsFormShell, +} from "@deepseek-ai/dsh-client-ui-primitives"; import type { ConfigForm } from "@deepseek-ai/dsh-client-ui-settings/client"; import type { ReMeSettings } from "../types.js"; -import { validEndpoint } from "../endpoint.js"; +import { + ReMeEnvironmentTab, + envEn, + envZh, + type EnvTranslator, +} from "../environment/client/tab.js"; +import { environmentStyles } from "../environment/client/styles.js"; import { ReMeStatusPage, statusEn, @@ -10,38 +21,23 @@ import { type StatusRpc, type StatusTranslator, } from "./status-page.js"; +import { SETTINGS_SPECS } from "./settings-specs.js"; import { styles } from "./styles.js"; const NS = "reme.settings"; const STATUS_NS = "reme.status"; +const ENV_NS = "reme.environment"; const SETTINGS_NS = "reme-memory"; -const FIELDS = [ - "endpoint", - "requestTimeoutMs", - "backgroundTimeoutMs", - "shutdownTimeoutMs", - "autoMemoryRetryMs", - "autoMemoryRetryMaxMs", - "autoMemoryEnabled", - "autoMemoryInterval", - "autoDreamEnabled", - "dreamCron", - "dreamHint", - "rootAgentsOnly", - "language", - "searchLimit", - "timezone", -] as const satisfies readonly (keyof ReMeSettings)[]; -type Field = (typeof FIELDS)[number]; type Translator = (key: keyof typeof en) => string; interface ClientContext { get(name: string): { rpc: StatusRpc }; - effect(factory: () => (() => void) | void, label: string): void; + effect(factory: () => void | (() => void), label: string): void; locale: { bind(namespace: typeof NS): Translator; bind(namespace: typeof STATUS_NS): StatusTranslator; + bind(namespace: typeof ENV_NS): EnvTranslator; register( namespace: string, dictionaries: { @@ -63,27 +59,61 @@ interface ClientContext { }; } -interface Draft { - endpoint: string; - requestTimeoutMs: string; - backgroundTimeoutMs: string; - shutdownTimeoutMs: string; - autoMemoryRetryMs: string; - autoMemoryRetryMaxMs: string; - autoMemoryEnabled: boolean; - autoMemoryInterval: string; - autoDreamEnabled: boolean; - dreamTime: string; - dreamHint: string; - rootAgentsOnly: boolean; - language: "en" | "zh"; - searchLimit: string; - timezone: string; +/** One control as the model currently stages it. */ +type FieldState = SettingsFieldState; + +/** + * The slice of DSH's snapshot store this card reads. + * + * `@deepseek-ai/dsh-client-store` is not a dependency of this package, so the + * store's own type is not available here; the card declares the two members it + * uses instead of depending on the package. + */ +interface BoundStore { + getSnapshot(): T; + subscribe(listener: () => void): () => void; +} + +/** Everything the card renders, rebuilt from the model on every publish. */ +interface CardState { + /** Whether the Host still serves this namespace to this client. */ + status: "loading" | "ready" | "unavailable"; + /** What a save would do, and whether one is crossing the wire. */ + shell: SettingsFormShell; + /** Every control's staged state, keyed by field name. */ + fields: Record; +} + +/** + * Rebuild the card's view state from the model. + * + * The model publishes when the scope changes, when a draft is staged, when a + * save starts and when it settles, so this projection is the card's only + * source of truth: what it renders is what the model holds. + * + * The projection carries the model's own state and nothing the settings + * service owns: the store freezes what it is given, and the resolved section + * belongs to the service, which replaces it rather than editing it. + */ +function projectCard( + model: SettingsFormModel, + scope: ConfigForm, +): CardState { + const fields: Record = {}; + for (const spec of SETTINGS_SPECS) + fields[spec.field] = model.field(spec.field); + return { + status: scope.getSnapshot().status, + shell: model.shell(), + fields, + }; } interface ReMeCardProps { scope: ConfigForm; t: Translator; + rpc?: StatusRpc; + envT?: EnvTranslator; view?: "summary" | "page"; } @@ -121,6 +151,8 @@ const en = { retryDelay: "Retry delay (ms)", retryMaxDelay: "Maximum retry delay (ms)", diagnostics: "Service diagnostics", + tabMemory: "Memory settings", + tabEnvironment: "Environment", processMemory: "Process memory", componentMemory: "Component memory", components: "Health components", @@ -196,6 +228,8 @@ const zh: typeof en = { retryDelay: "失败重试间隔（毫秒）", retryMaxDelay: "最大重试间隔（毫秒）", diagnostics: "服务诊断", + tabMemory: "记忆设置", + tabEnvironment: "环境管理", processMemory: "进程内存", componentMemory: "组件内存", components: "健康组件", @@ -250,7 +284,12 @@ export function apply(ctx: ClientContext): void { () => ctx.locale.register(STATUS_NS, { en: statusEn, zh: statusZh }), "remeMemory.statusLocale()", ); + ctx.effect( + () => ctx.locale.register(ENV_NS, { en: envEn, zh: envZh }), + "remeMemory.envLocale()", + ); ctx.effect(() => installStyles(), "remeMemory.settingsStyles()"); + const envT = ctx.locale.bind(ENV_NS); const scope = ctx.configForms.get(SETTINGS_NS); ctx.effect( () => @@ -262,7 +301,12 @@ export function apply(ctx: ClientContext): void { key: "@agentscope-ai/reme-dsh-plugin#reme-memory", label: () => t("title"), locale: NS, - inject: () => ({ scope, t }), + inject: () => ({ + scope, + t, + rpc: ctx.get("connection").rpc, + envT, + }), }, ReMeSettingsCard, ), @@ -291,177 +335,217 @@ export function apply(ctx: ClientContext): void { function ReMeSettingsCard({ scope, t, + rpc, + envT, view, }: ReMeCardProps): JSX.Element | null { - const snapshot = useSyncExternalStore( - (listener) => scope.subscribe(listener), - () => scope.getSnapshot(), + // DSH's form model stages the drafts, marks the ones that are invalid and + // writes every staged edit as one revision-fenced mutation, so this page owns + // only its controls and the field specs behind them. + // + // The model, its actions and the store it publishes to are built once: the + // model owns one subscription to the scope, and `bind` registers one + // listener on the model, so a second build would publish every change twice. + const [form] = useState(() => { + const model = new SettingsFormModel(scope, SETTINGS_SPECS); + return { + model, + actions: model.actions(), + store: model.bind(() => + projectCard(model, scope), + ) as BoundStore, + }; + }); + // The model publishes on a scope change, on every staged edit, when a save + // starts and when it settles. The card reads those publications and nothing + // else, so a staged edit repaints the controls on its own. + const state = useSyncExternalStore( + form.store.subscribe, + form.store.getSnapshot, ); - const [draftOverride, setDraft] = useState<{ - value: Draft; - revision: number | undefined; - }>(); - const [resetAll, setResetAll] = useState(false); - const [saving, setSaving] = useState(false); - const [saveState, setSaveState] = useState<"" | "saved" | "failed">(""); - const value = snapshot.value; - const draft = - draftOverride?.value ?? - (value === undefined ? undefined : draftFrom(value)); + const disposeTimer = useRef | undefined>( + undefined, + ); + const [saved, setSaved] = useState(false); + const [tab, setTab] = useState<"memory" | "environment">("memory"); + useEffect(() => { + // React StrictMode mounts, cleans up and mounts again in development. + // Disposing the model immediately would hand the remounted card a dead + // one, so the cleanup is deferred and cancelled by the next mount. + if (disposeTimer.current !== undefined) { + clearTimeout(disposeTimer.current); + disposeTimer.current = undefined; + } + return () => { + disposeTimer.current = setTimeout(() => { + disposeTimer.current = undefined; + form.model.dispose(); + }, 0); + }; + }, [form.model]); if (view === "summary") return <>{t("description")}; - if (snapshot.status === "unavailable") return null; - const dirty = - resetAll || - (value !== undefined && - draft !== undefined && - !sameDraft(draft, draftFrom(value))); - const validation = draft === undefined ? undefined : parseDraft(draft); + if (state.status === "unavailable") return null; - const update = (field: K, next: Draft[K]) => { - if (value === undefined) return; - setDraft((current) => ({ - value: { ...(current?.value ?? draftFrom(value)), [field]: next }, - revision: current ? current.revision : snapshot.revision, - })); - setResetAll(false); - setSaveState(""); + const { shell, fields } = state; + // The model's own saving state: it is true from the moment the save starts + // until the write settles, so the card keeps no second flag for it. + const saving = shell.saving; + // "The maximum retry delay is at least the initial one" is a relation between + // two fields, which a per-field spec cannot state, so the page keeps it. + const retryMax = (fields["autoMemoryRetryMaxMs"]?.text ?? "").trim(); + const retryBase = (fields["autoMemoryRetryMs"]?.text ?? "").trim(); + const inconsistent = + retryMax !== "" && retryBase !== "" && Number(retryMax) < Number(retryBase); + const blocked = shell.invalid || inconsistent; + const edit = (name: string, text: string): void => { + setSaved(false); + form.actions.edit(name, text); }; - - const save = async () => { - if ( - value === undefined || - draft === undefined || - validation === undefined || - !snapshot.writable || - !dirty || - saving - ) + const save = async (): Promise => { + const current = form.store.getSnapshot().shell; + if (!current.writable || !current.dirty || blocked || current.saving) return; - setSaving(true); - setSaveState(""); - try { - const ops = resetAll - ? FIELDS.map((field) => ({ op: "unset" as const, path: [field] })) - : FIELDS.flatMap((field) => - Object.is(value[field], validation[field]) - ? [] - : [ - { - op: "set" as const, - path: [field], - value: validation[field], - }, - ], - ); - const success = await scope.mutate( - ops, - draftOverride?.revision ?? snapshot.revision, - ); - setSaveState(success ? "saved" : "failed"); - if (success) { - setDraft(undefined); - setResetAll(false); - } - } catch { - setSaveState("failed"); - } finally { - setSaving(false); - } + setSaved(false); + await form.model.save(); + // The model published the settled state before its promise resolved, so the + // store already carries the outcome of the write. + const after = form.store.getSnapshot().shell; + setSaved(!after.failed && !after.dirty); + }; + const reset = (): void => { + setSaved(false); + for (const spec of SETTINGS_SPECS) form.actions.resetField(spec.field); + }; + const discard = (): void => { + setSaved(false); + form.actions.discard(); }; return (
- {dirty ?

{t("unsaved")}

: null} - {snapshot.status === "loading" || - draft === undefined || - value === undefined ? ( -

{t("loading")}

- ) : ( - <> -
- -
- {!snapshot.writable ? ( -

- {t("readOnly")} -

- ) : null} - {validation === undefined ? ( -

- {t("invalid")} -

- ) : null} -
- + +
+
+ {tab === "environment" ? ( + rpc && envT ? ( + + ) : ( +

{t("loading")}

+ ) + ) : state.status === "ready" ? ( + <> + {shell.dirty ? ( +

{t("unsaved")}

+ ) : null} +
- {t("reset")} - - - {saveState === "failed" ? ( - {t("saveFailed")} - ) : null} - {saveState === "saved" ? ( - {t("saved")} - ) : null} - - - -
- - )} + + + {!shell.writable ? ( +

+ {t("readOnly")} +

+ ) : null} + {blocked ? ( +

+ {t("invalid")} +

+ ) : null} +
+ + + {shell.failed ? ( + + {t("saveFailed")} + + ) : null} + {saved && !shell.dirty ? ( + {t("saved")} + ) : null} + + + +
+ + ) : ( +

{t("loading")}

+ )} +
); } +/** The memory-settings controls, all reading and writing through the model. */ function SettingsForm({ - draft, - update, + fields, + edit, t, }: { - draft: Draft; - update: (field: K, value: Draft[K]) => void; + fields: Record; + edit: (name: string, text: string) => void; t: Translator; }): JSX.Element { + const text = (name: string): string => fields[name]?.text ?? ""; + const invalid = (name: string): boolean => fields[name]?.invalid ?? false; + const checked = (name: string): boolean => text(name) === "true"; return ( <>
@@ -469,17 +553,15 @@ function SettingsForm({
update("endpoint", event.target.value)} + value={text("endpoint")} + aria-invalid={invalid("endpoint")} + onChange={(event) => edit("endpoint", event.target.value)} /> - update("autoMemoryInterval", event.target.value) + edit("autoMemoryInterval", event.target.value) } /> update("timezone", event.target.value)} + value={text("timezone")} + aria-invalid={invalid("timezone")} + onChange={(event) => edit("timezone", event.target.value)} />
@@ -545,20 +627,22 @@ function SettingsForm({
update("autoDreamEnabled", value)} + checked={checked("autoDreamEnabled")} + onChange={(next) => + edit("autoDreamEnabled", next ? "true" : "false") + } /> update("dreamTime", event.target.value)} + value={text("dreamCron")} + onChange={(event) => edit("dreamCron", event.target.value)} />