From 53cf655415e94f8475afe42125a961c8a3357f90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=85=B1=E7=89=9B=E8=82=89?= Date: Tue, 3 Nov 2026 12:46:06 +0800 Subject: [PATCH] refactor(home): consolidate Git exports and candidate validation --- .../src/reme_home/agent/candidate_gate.py | 7 +- plugins/home/src/reme_home/code/repository.py | 9 ++ .../home/src/reme_home/evaluation/inputs.py | 58 +--------- .../home/src/reme_home/evaluation/reuse.py | 7 +- plugins/home/src/reme_home/plugin.yaml | 1 - .../home/src/reme_home/preparation/code.py | 109 ++++++++++-------- .../preparation/evaluation_source.py | 17 +-- plugins/home/tests/test_git_export.py | 74 ++++++++++++ 8 files changed, 155 insertions(+), 127 deletions(-) create mode 100644 plugins/home/tests/test_git_export.py diff --git a/plugins/home/src/reme_home/agent/candidate_gate.py b/plugins/home/src/reme_home/agent/candidate_gate.py index 485525fe..ecf706e5 100644 --- a/plugins/home/src/reme_home/agent/candidate_gate.py +++ b/plugins/home/src/reme_home/agent/candidate_gate.py @@ -7,7 +7,7 @@ from uuid import uuid4 from pydantic import Field -from ..code.repository import current_branch, require_clean, resolve_revision, validate_repository, validate_tree +from ..code.repository import current_branch, validate_candidate, validate_repository from ..code.snapshot import export_blob, tree from ..evaluation.artifacts import contained, read_json from ..evaluation.component import blocking @@ -26,10 +26,7 @@ class CandidateGateReport(SpecModel): def admission_context(home): """Called under the workspace lock; retries and the baseline bypass admission.""" - repo, record = validate_repository(home) - require_clean(repo) - commit = resolve_revision(repo, "HEAD") - validate_tree(repo, record, commit) + repo, record, commit = validate_candidate(home) if commit == record["code"]["initial_commit"] or previous_candidate_result(home, commit) is not None: return None ancestors = set(git(repo, "rev-list", commit).splitlines()) diff --git a/plugins/home/src/reme_home/code/repository.py b/plugins/home/src/reme_home/code/repository.py index a02d24b7..3df6a6ce 100644 --- a/plugins/home/src/reme_home/code/repository.py +++ b/plugins/home/src/reme_home/code/repository.py @@ -30,6 +30,15 @@ def validate_repository(home: Path, record: dict | None = None) -> tuple[Path, d return repository, record +def validate_candidate(home: Path) -> tuple[Path, dict, str]: + """Validate a clean candidate HEAD before admission or construction reuse, under the caller's lock.""" + repository, record = validate_repository(home) + require_clean(repository) + revision = resolve_revision(repository, "HEAD") + validate_tree(repository, record, revision) + return repository, record, revision + + def current_branch(repository: Path) -> str: """Reject detached HEAD so every candidate commit remains branch-reachable.""" branch = git(repository, "branch", "--show-current") diff --git a/plugins/home/src/reme_home/evaluation/inputs.py b/plugins/home/src/reme_home/evaluation/inputs.py index 134e73b9..a898f731 100644 --- a/plugins/home/src/reme_home/evaluation/inputs.py +++ b/plugins/home/src/reme_home/evaluation/inputs.py @@ -4,7 +4,6 @@ from pathlib import Path from typing import Literal import shutil -import subprocess import tempfile import tarfile @@ -12,7 +11,7 @@ from pydantic import Field, field_validator, model_validator from deploy.benchmark_docker import SourceSnapshot -from ..preparation.code import git, tree_fingerprint +from ..preparation.code import export_git_tree, git, git_tree, runtime_roots, tree_fingerprint from ..preparation.data.models import CaseSpec, DatasetManifest, fingerprint from .artifacts import contained, file_sha256, inspect_archive, read_json from ..preparation.spec import SpecModel @@ -138,45 +137,11 @@ def snapshot_code(home: Path, record: dict): commit = git(repository, "rev-parse", "--verify", "HEAD^{commit}") branch = "init" if commit == record["code"]["initial_commit"] else git(repository, "branch", "--show-current") entity_id(branch) - # NUL output validates exact paths without interpreting Git quoting rules. - entries = subprocess.check_output( - ["git", "-C", str(repository), "ls-tree", "-r", "-z", commit], - timeout=120, - ).split(b"\0") - parsed = [] - for entry in entries: - if not entry: - continue - header, name = entry.split(b"\t", 1) - mode, kind, oid = header.decode().split() - relative = Path(name.decode()) - if ( - relative.is_absolute() - or ".." in relative.parts - or ".git" in relative.parts - or kind != "blob" - or mode not in {"100644", "100755"} - ): - raise ValueError(f"Candidate must contain contained regular files: {relative}") - if not any( - relative.is_relative_to(path) - for path in [ - Path("reme"), - Path("pyproject.toml"), - Path("README.md"), - Path("LICENSE"), - *[Path("plugins") / p for p in record["code"]["plugins"]], - ] - ): + entries = git_tree(repository, commit) + roots = runtime_roots(record["code"]["plugins"]) + for relative, _, _ in entries: + if not any(relative.is_relative_to(path) for path in roots): raise ValueError(f"Candidate path is outside the prepared runtime projects: {relative}") - parsed.append((relative, mode, oid)) - contents = subprocess.run( - ["git", "-C", str(repository), "cat-file", "--batch"], - input="".join(oid + "\n" for _, _, oid in parsed).encode(), - capture_output=True, - check=True, - timeout=120, - ).stdout with tempfile.TemporaryDirectory( prefix="evaluation-source-", dir=contained(home, "tmp", directory=True), @@ -193,18 +158,7 @@ def snapshot_code(home: Path, record: dict): shutil.rmtree(target) elif target.exists(): target.unlink() - offset = 0 - for relative, mode, oid in parsed: - end = contents.index(b"\n", offset) - found, kind, size = contents[offset:end].decode().split() - if found != oid or kind != "blob": - raise ValueError("Invalid candidate Git object") - offset = end + 1 - target = combined / relative - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(contents[offset : offset + int(size)]) - target.chmod(0o755 if mode == "100755" else 0o644) - offset += int(size) + 1 + export_git_tree(repository, entries, lambda path: combined / path) for project in record["code"]["install_projects"]: if not (combined / project / "pyproject.toml").is_file(): raise ValueError(f"Candidate is missing install project {project}") diff --git a/plugins/home/src/reme_home/evaluation/reuse.py b/plugins/home/src/reme_home/evaluation/reuse.py index 3bc5b9be..ea3c2a91 100644 --- a/plugins/home/src/reme_home/evaluation/reuse.py +++ b/plugins/home/src/reme_home/evaluation/reuse.py @@ -5,7 +5,7 @@ import random import subprocess import tarfile -from ..code.repository import current_branch, require_clean, resolve_revision, validate_repository, validate_tree +from ..code.repository import current_branch, validate_candidate, validate_repository from ..code.snapshot import tree from ..preparation.code import git from ..preparation.data.models import fingerprint @@ -43,10 +43,7 @@ def construction_fingerprints(record, dataset, spec, environment): def reuse_context(home, spec, environment): """Find the nearest valid ancestor before review or sampling, under the workspace lock.""" - repo, record = validate_repository(home) - require_clean(repo) - commit = resolve_revision(repo, "HEAD") - validate_tree(repo, record, commit) + repo, record, commit = validate_candidate(home) identity = {"candidate_commit": commit, "branch_name": current_branch(repo)} previous = previous_candidate_result(home, commit) if previous is not None: diff --git a/plugins/home/src/reme_home/plugin.yaml b/plugins/home/src/reme_home/plugin.yaml index 03b570fe..5021e03b 100644 --- a/plugins/home/src/reme_home/plugin.yaml +++ b/plugins/home/src/reme_home/plugin.yaml @@ -2,7 +2,6 @@ backends: home_optimizer_agent: reme_home.agent.optimizer:OptimizerAgentWrapper home_diagnose_step: reme_home.agent.diagnostic:HomeDiagnoseStep home_optimize_step: reme_home.agent.optimizer:HomeOptimizeStep - home_workflow_agent: reme_home.agent.wrapper:CaseStudyAgentWrapper home_case_study_step: reme_home.agent.case_study:HomeCaseStudyStep home_case_study_agent: reme_home.agent.wrapper:CaseStudyAgentWrapper home_trace_query_evidence_step: reme_home.evaluation.evidence_step:HomeTraceQueryEvidenceStep diff --git a/plugins/home/src/reme_home/preparation/code.py b/plugins/home/src/reme_home/preparation/code.py index 1f9831dd..3f49c5d6 100644 --- a/plugins/home/src/reme_home/preparation/code.py +++ b/plugins/home/src/reme_home/preparation/code.py @@ -1,6 +1,7 @@ """Export trusted source inputs without running their code or installing them.""" import hashlib +from collections.abc import Callable import os from pathlib import Path import shutil @@ -10,7 +11,7 @@ import tomllib from .spec import PreparationSpec, relative_path -def git_bytes(repository: Path, *arguments: str) -> bytes: +def git_bytes(repository: Path, *arguments: str, input_bytes: bytes | None = None) -> bytes: """Run bounded, noninteractive Git commands without a shell or user hooks.""" environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} environment.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, GIT_TERMINAL_PROMPT="0") @@ -29,6 +30,7 @@ def git_bytes(repository: Path, *arguments: str) -> bytes: ], check=True, capture_output=True, + input=input_bytes, env=environment, timeout=120, ) @@ -41,6 +43,59 @@ def git(repository: Path, *arguments: str, strip: bool = True) -> str: return output.strip() if strip else output +def runtime_roots(plugins: list[str]) -> list[Path]: + """Return the core and plugin project roots allowed in prepared sources.""" + return [ + Path("reme"), + Path("pyproject.toml"), + Path("README.md"), + Path("LICENSE"), + *(Path("plugins") / name for name in plugins), + ] + + +def git_tree(repository: Path, revision: str, roots: list[Path] | None = None) -> list[tuple[Path, str, str]]: + """Select regular Git files using exact NUL-delimited paths, retaining modes and object IDs.""" + entries = [] + raw_tree = git_bytes(repository, "ls-tree", "-r", "-z", "--full-tree", revision) + for entry in filter(None, raw_tree.split(b"\0")): + header, raw_name = entry.split(b"\t", 1) + path = Path(relative_path(raw_name.decode("utf-8"))) + if roots is not None and not any(path.is_relative_to(root) for root in roots): + continue + mode, kind, oid = header.decode().split() + if mode not in {"100644", "100755"} or kind != "blob" or ".git" in path.parts: + raise ValueError(f"Git source must contain regular files only: {path}") + entries.append((path, mode, oid)) + return entries + + +def export_git_tree( + repository: Path, + entries: list[tuple[Path, str, str]], + destination: Callable[[Path], Path], +) -> None: + """Batch-export raw blobs without checkout/archive filters or browsing size limits.""" + objects = git_bytes( + repository, + "cat-file", + "--batch", + input_bytes="".join(oid + "\n" for _, _, oid in entries).encode(), + ) + offset = 0 + for path, mode, oid in entries: + boundary = objects.index(b"\n", offset) + found, kind, size = objects[offset:boundary].decode().split() + if found != oid or kind != "blob": + raise ValueError(f"Invalid Git object: {path}") + offset = boundary + 1 + target = destination(path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(objects[offset : offset + int(size)]) + target.chmod(0o755 if mode == "100755" else 0o644) + offset += int(size) + 1 + + def tree_fingerprint(root: Path) -> str: """Hash relative names and file bytes, excluding candidate Git metadata.""" digest = hashlib.sha256() @@ -79,57 +134,11 @@ def prepare_code(spec: PreparationSpec, source: Path, runtime: Path, candidate: is editable. The initial candidate commit supplies the original baseline. """ revision = git(source, "rev-parse", "--verify", "--end-of-options", spec.runtime.source_ref + "^{commit}") - # Read object contents directly, bypassing archive attributes and checkout filters. - environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} - raw_tree = subprocess.run( - ["git", "-C", str(source), "ls-tree", "-r", "-z", "--full-tree", revision], - check=True, - capture_output=True, - timeout=120, - env=environment, - ).stdout - roots = [ - Path("reme"), - Path("pyproject.toml"), - Path("README.md"), - Path("LICENSE"), - *(Path("plugins") / name for name in spec.runtime.plugins), - ] - entries = [] - for entry in raw_tree.split(b"\0"): - if not entry: - continue - header, raw_name = entry.split(b"\t", maxsplit=1) - name = Path(relative_path(raw_name.decode("utf-8"))) - if not any(name.is_relative_to(root) for root in roots): - continue - mode, kind, object_id = header.decode().split() - if mode not in {"100644", "100755"} or kind != "blob": - raise ValueError(f"Runtime source must contain regular files only: {name}") - entries.append((name, mode, object_id)) - objects = subprocess.run( - ["git", "-C", str(source), "cat-file", "--batch"], - input="".join(object_id + "\n" for _, _, object_id in entries).encode(), - check=True, - capture_output=True, - timeout=120, - env=environment, - ).stdout + entries = git_tree(source, revision, runtime_roots(spec.runtime.plugins)) runtime.mkdir() candidate.mkdir() - offset = 0 - for name, mode, object_id in entries: - boundary = objects.index(b"\n", offset) - found, kind, size = objects[offset:boundary].decode().split() - if found != object_id or kind != "blob": - raise ValueError(f"Invalid Git object: {name}") - offset = boundary + 1 - # Keep each selected plugin only in repo, including packaging/config files. - destination = (candidate if name.parts[0] == "plugins" else runtime) / name - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(objects[offset : offset + int(size)]) - destination.chmod(0o755 if mode == "100755" else 0o644) - offset += int(size) + 1 + # Keep each selected plugin only in repo, including packaging/config files. + export_git_tree(source, entries, lambda path: (candidate if path.parts[0] == "plugins" else runtime) / path) if not (runtime / "reme/__init__.py").is_file() or not (runtime / "pyproject.toml").is_file(): raise ValueError("Source revision is not a ReMe project") if not (runtime / "reme/config" / spec.runtime.config).is_file(): diff --git a/plugins/home/src/reme_home/preparation/evaluation_source.py b/plugins/home/src/reme_home/preparation/evaluation_source.py index f2652df1..3def425f 100644 --- a/plugins/home/src/reme_home/preparation/evaluation_source.py +++ b/plugins/home/src/reme_home/preparation/evaluation_source.py @@ -8,11 +8,10 @@ import shutil import yaml from ..code.repository import resolve_revision, validate_repository, validate_tree -from ..code.snapshot import export_blob from ..evaluation.artifacts import contained, read_json -from .code import git, tree_fingerprint +from .code import export_git_tree, git, git_tree, tree_fingerprint from .data.models import fingerprint -from .spec import PreparationSpec, relative_path +from .spec import PreparationSpec @dataclass(frozen=True) @@ -65,17 +64,7 @@ class EvaluationSource: raise ValueError("Source fixed runtime changed during import") repository = contained(self.home, "code/repo", directory=True) candidate.mkdir() - entries = git(repository, "ls-tree", "-r", "-z", "--full-tree", self.commit, strip=False) - for entry in filter(None, entries.split("\0")): - header, name = entry.split("\t", 1) - mode, kind, oid = header.split() - path = Path(relative_path(name)) - if mode not in {"100644", "100755"} or kind != "blob" or ".git" in path.parts: - raise ValueError(f"Candidate must contain regular files only: {name}") - destination = candidate / path - destination.parent.mkdir(parents=True, exist_ok=True) - export_blob(repository, oid, destination) - destination.chmod(0o755 if mode == "100755" else 0o644) + export_git_tree(repository, git_tree(repository, self.commit), lambda path: candidate / path) # The isolated repository's init is the selected snapshot, not the optimizer baseline. git(candidate, "init", "--initial-branch=init", "--template=") git(candidate, "add", "--force", "--all") diff --git a/plugins/home/tests/test_git_export.py b/plugins/home/tests/test_git_export.py new file mode 100644 index 00000000..cdeaa475 --- /dev/null +++ b/plugins/home/tests/test_git_export.py @@ -0,0 +1,74 @@ +"""Git exports preserve raw committed bytes, literal names, and executable modes.""" + +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from reme_home.preparation import code + + +def test_batch_export_bypasses_filters_and_checkout(tmp_path, monkeypatch): + """One batch reads all selected blobs, including binary and large files.""" + repository = tmp_path / "repo" + repository.mkdir() + code.git(repository, "init", "--template=") + contents = { + "space name": b"$Format:%H$\n", + 'quotes"\tand\nnewline': b"\x00binary\xff\n", + "脚本": b"#!/bin/sh\nexit 0\n", + "large": b"x" * 5_000_001, + "empty": b"", + ".gitattributes": b"* export-ignore export-subst filter=blocked\n", + } + for name, data in contents.items(): + (repository / name).write_bytes(data) + (repository / "脚本").chmod(0o755) + code.git(repository, "add", "--all") + code.git(repository, "-c", "user.name=Test", "-c", "user.email=test@localhost", "commit", "-m", "files") + code.git(repository, "config", "filter.blocked.process", "/bin/false") + code.git(repository, "config", "filter.blocked.required", "true") + for name in contents: + (repository / name).write_bytes(b"dirty checkout") + run = Mock(wraps=code.git_bytes) + monkeypatch.setattr(code, "git_bytes", run) + destination = tmp_path / "export" + + code.export_git_tree(repository, code.git_tree(repository, "HEAD"), lambda path: destination / path) + + assert run.call_count == 2 + assert run.call_args.args == (repository, "cat-file", "--batch") + for name, data in contents.items(): + target = destination / name + assert target.read_bytes() == data + assert target.stat().st_mode & 0o777 == (0o755 if name == "脚本" else 0o644) + + +@pytest.mark.parametrize( + "mode,kind,name", + [ + ("120000", "blob", "link"), + ("160000", "commit", "submodule"), + ("100644", "blob", ".git/config"), + ("100644", "blob", "../escape"), + ("100644", "blob", "/escape"), + ("100644", "blob", "~escape"), + ("100644", "blob", "folder\\escape"), + ], +) +def test_tree_rejects_unsafe_paths_and_nonregular_files(monkeypatch, mode, kind, name): + """Reject unsafe object metadata before any file is exported.""" + monkeypatch.setattr(code, "git_bytes", Mock(return_value=f"{mode} {kind} {'a' * 40}\t{name}\0".encode())) + with pytest.raises(ValueError): + code.git_tree(Path("unused"), "HEAD") + + +def test_tree_selection_uses_literal_roots(monkeypatch): + """Ignore out-of-scope links without mistaking similar prefixes for selected roots.""" + raw = ( + f"100644 blob {'a' * 40}\tplugin/file\0" + f"120000 blob {'b' * 40}\tplugin-other/link\0" + f"160000 commit {'c' * 40}\tother\0" + ).encode() + monkeypatch.setattr(code, "git_bytes", Mock(return_value=raw)) + assert code.git_tree(Path("unused"), "HEAD", [Path("plugin")]) == [(Path("plugin/file"), "100644", "a" * 40)]