refactor(home): consolidate Git exports and candidate validation

This commit is contained in:
酱牛肉 2026-11-03 12:46:06 +08:00
parent 6cd9c9c6b2
commit 53cf655415
8 changed files with 155 additions and 127 deletions

View file

@ -7,7 +7,7 @@ from uuid import uuid4
from pydantic import Field
from ..code.repository import current_branch, require_clean, resolve_revision, validate_repository, validate_tree
from ..code.repository import current_branch, validate_candidate, validate_repository
from ..code.snapshot import export_blob, tree
from ..evaluation.artifacts import contained, read_json
from ..evaluation.component import blocking
@ -26,10 +26,7 @@ class CandidateGateReport(SpecModel):
def admission_context(home):
"""Called under the workspace lock; retries and the baseline bypass admission."""
repo, record = validate_repository(home)
require_clean(repo)
commit = resolve_revision(repo, "HEAD")
validate_tree(repo, record, commit)
repo, record, commit = validate_candidate(home)
if commit == record["code"]["initial_commit"] or previous_candidate_result(home, commit) is not None:
return None
ancestors = set(git(repo, "rev-list", commit).splitlines())

View file

@ -30,6 +30,15 @@ def validate_repository(home: Path, record: dict | None = None) -> tuple[Path, d
return repository, record
def validate_candidate(home: Path) -> tuple[Path, dict, str]:
"""Validate a clean candidate HEAD before admission or construction reuse, under the caller's lock."""
repository, record = validate_repository(home)
require_clean(repository)
revision = resolve_revision(repository, "HEAD")
validate_tree(repository, record, revision)
return repository, record, revision
def current_branch(repository: Path) -> str:
"""Reject detached HEAD so every candidate commit remains branch-reachable."""
branch = git(repository, "branch", "--show-current")

View file

@ -4,7 +4,6 @@ from pathlib import Path
from typing import Literal
import shutil
import subprocess
import tempfile
import tarfile
@ -12,7 +11,7 @@ from pydantic import Field, field_validator, model_validator
from deploy.benchmark_docker import SourceSnapshot
from ..preparation.code import git, tree_fingerprint
from ..preparation.code import export_git_tree, git, git_tree, runtime_roots, tree_fingerprint
from ..preparation.data.models import CaseSpec, DatasetManifest, fingerprint
from .artifacts import contained, file_sha256, inspect_archive, read_json
from ..preparation.spec import SpecModel
@ -138,45 +137,11 @@ def snapshot_code(home: Path, record: dict):
commit = git(repository, "rev-parse", "--verify", "HEAD^{commit}")
branch = "init" if commit == record["code"]["initial_commit"] else git(repository, "branch", "--show-current")
entity_id(branch)
# NUL output validates exact paths without interpreting Git quoting rules.
entries = subprocess.check_output(
["git", "-C", str(repository), "ls-tree", "-r", "-z", commit],
timeout=120,
).split(b"\0")
parsed = []
for entry in entries:
if not entry:
continue
header, name = entry.split(b"\t", 1)
mode, kind, oid = header.decode().split()
relative = Path(name.decode())
if (
relative.is_absolute()
or ".." in relative.parts
or ".git" in relative.parts
or kind != "blob"
or mode not in {"100644", "100755"}
):
raise ValueError(f"Candidate must contain contained regular files: {relative}")
if not any(
relative.is_relative_to(path)
for path in [
Path("reme"),
Path("pyproject.toml"),
Path("README.md"),
Path("LICENSE"),
*[Path("plugins") / p for p in record["code"]["plugins"]],
]
):
entries = git_tree(repository, commit)
roots = runtime_roots(record["code"]["plugins"])
for relative, _, _ in entries:
if not any(relative.is_relative_to(path) for path in roots):
raise ValueError(f"Candidate path is outside the prepared runtime projects: {relative}")
parsed.append((relative, mode, oid))
contents = subprocess.run(
["git", "-C", str(repository), "cat-file", "--batch"],
input="".join(oid + "\n" for _, _, oid in parsed).encode(),
capture_output=True,
check=True,
timeout=120,
).stdout
with tempfile.TemporaryDirectory(
prefix="evaluation-source-",
dir=contained(home, "tmp", directory=True),
@ -193,18 +158,7 @@ def snapshot_code(home: Path, record: dict):
shutil.rmtree(target)
elif target.exists():
target.unlink()
offset = 0
for relative, mode, oid in parsed:
end = contents.index(b"\n", offset)
found, kind, size = contents[offset:end].decode().split()
if found != oid or kind != "blob":
raise ValueError("Invalid candidate Git object")
offset = end + 1
target = combined / relative
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(contents[offset : offset + int(size)])
target.chmod(0o755 if mode == "100755" else 0o644)
offset += int(size) + 1
export_git_tree(repository, entries, lambda path: combined / path)
for project in record["code"]["install_projects"]:
if not (combined / project / "pyproject.toml").is_file():
raise ValueError(f"Candidate is missing install project {project}")

View file

@ -5,7 +5,7 @@ import random
import subprocess
import tarfile
from ..code.repository import current_branch, require_clean, resolve_revision, validate_repository, validate_tree
from ..code.repository import current_branch, validate_candidate, validate_repository
from ..code.snapshot import tree
from ..preparation.code import git
from ..preparation.data.models import fingerprint
@ -43,10 +43,7 @@ def construction_fingerprints(record, dataset, spec, environment):
def reuse_context(home, spec, environment):
"""Find the nearest valid ancestor before review or sampling, under the workspace lock."""
repo, record = validate_repository(home)
require_clean(repo)
commit = resolve_revision(repo, "HEAD")
validate_tree(repo, record, commit)
repo, record, commit = validate_candidate(home)
identity = {"candidate_commit": commit, "branch_name": current_branch(repo)}
previous = previous_candidate_result(home, commit)
if previous is not None:

View file

@ -2,7 +2,6 @@ backends:
home_optimizer_agent: reme_home.agent.optimizer:OptimizerAgentWrapper
home_diagnose_step: reme_home.agent.diagnostic:HomeDiagnoseStep
home_optimize_step: reme_home.agent.optimizer:HomeOptimizeStep
home_workflow_agent: reme_home.agent.wrapper:CaseStudyAgentWrapper
home_case_study_step: reme_home.agent.case_study:HomeCaseStudyStep
home_case_study_agent: reme_home.agent.wrapper:CaseStudyAgentWrapper
home_trace_query_evidence_step: reme_home.evaluation.evidence_step:HomeTraceQueryEvidenceStep

View file

@ -1,6 +1,7 @@
"""Export trusted source inputs without running their code or installing them."""
import hashlib
from collections.abc import Callable
import os
from pathlib import Path
import shutil
@ -10,7 +11,7 @@ import tomllib
from .spec import PreparationSpec, relative_path
def git_bytes(repository: Path, *arguments: str) -> bytes:
def git_bytes(repository: Path, *arguments: str, input_bytes: bytes | None = None) -> bytes:
"""Run bounded, noninteractive Git commands without a shell or user hooks."""
environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")}
environment.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, GIT_TERMINAL_PROMPT="0")
@ -29,6 +30,7 @@ def git_bytes(repository: Path, *arguments: str) -> bytes:
],
check=True,
capture_output=True,
input=input_bytes,
env=environment,
timeout=120,
)
@ -41,6 +43,59 @@ def git(repository: Path, *arguments: str, strip: bool = True) -> str:
return output.strip() if strip else output
def runtime_roots(plugins: list[str]) -> list[Path]:
"""Return the core and plugin project roots allowed in prepared sources."""
return [
Path("reme"),
Path("pyproject.toml"),
Path("README.md"),
Path("LICENSE"),
*(Path("plugins") / name for name in plugins),
]
def git_tree(repository: Path, revision: str, roots: list[Path] | None = None) -> list[tuple[Path, str, str]]:
"""Select regular Git files using exact NUL-delimited paths, retaining modes and object IDs."""
entries = []
raw_tree = git_bytes(repository, "ls-tree", "-r", "-z", "--full-tree", revision)
for entry in filter(None, raw_tree.split(b"\0")):
header, raw_name = entry.split(b"\t", 1)
path = Path(relative_path(raw_name.decode("utf-8")))
if roots is not None and not any(path.is_relative_to(root) for root in roots):
continue
mode, kind, oid = header.decode().split()
if mode not in {"100644", "100755"} or kind != "blob" or ".git" in path.parts:
raise ValueError(f"Git source must contain regular files only: {path}")
entries.append((path, mode, oid))
return entries
def export_git_tree(
repository: Path,
entries: list[tuple[Path, str, str]],
destination: Callable[[Path], Path],
) -> None:
"""Batch-export raw blobs without checkout/archive filters or browsing size limits."""
objects = git_bytes(
repository,
"cat-file",
"--batch",
input_bytes="".join(oid + "\n" for _, _, oid in entries).encode(),
)
offset = 0
for path, mode, oid in entries:
boundary = objects.index(b"\n", offset)
found, kind, size = objects[offset:boundary].decode().split()
if found != oid or kind != "blob":
raise ValueError(f"Invalid Git object: {path}")
offset = boundary + 1
target = destination(path)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(objects[offset : offset + int(size)])
target.chmod(0o755 if mode == "100755" else 0o644)
offset += int(size) + 1
def tree_fingerprint(root: Path) -> str:
"""Hash relative names and file bytes, excluding candidate Git metadata."""
digest = hashlib.sha256()
@ -79,57 +134,11 @@ def prepare_code(spec: PreparationSpec, source: Path, runtime: Path, candidate:
is editable. The initial candidate commit supplies the original baseline.
"""
revision = git(source, "rev-parse", "--verify", "--end-of-options", spec.runtime.source_ref + "^{commit}")
# Read object contents directly, bypassing archive attributes and checkout filters.
environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")}
raw_tree = subprocess.run(
["git", "-C", str(source), "ls-tree", "-r", "-z", "--full-tree", revision],
check=True,
capture_output=True,
timeout=120,
env=environment,
).stdout
roots = [
Path("reme"),
Path("pyproject.toml"),
Path("README.md"),
Path("LICENSE"),
*(Path("plugins") / name for name in spec.runtime.plugins),
]
entries = []
for entry in raw_tree.split(b"\0"):
if not entry:
continue
header, raw_name = entry.split(b"\t", maxsplit=1)
name = Path(relative_path(raw_name.decode("utf-8")))
if not any(name.is_relative_to(root) for root in roots):
continue
mode, kind, object_id = header.decode().split()
if mode not in {"100644", "100755"} or kind != "blob":
raise ValueError(f"Runtime source must contain regular files only: {name}")
entries.append((name, mode, object_id))
objects = subprocess.run(
["git", "-C", str(source), "cat-file", "--batch"],
input="".join(object_id + "\n" for _, _, object_id in entries).encode(),
check=True,
capture_output=True,
timeout=120,
env=environment,
).stdout
entries = git_tree(source, revision, runtime_roots(spec.runtime.plugins))
runtime.mkdir()
candidate.mkdir()
offset = 0
for name, mode, object_id in entries:
boundary = objects.index(b"\n", offset)
found, kind, size = objects[offset:boundary].decode().split()
if found != object_id or kind != "blob":
raise ValueError(f"Invalid Git object: {name}")
offset = boundary + 1
# Keep each selected plugin only in repo, including packaging/config files.
destination = (candidate if name.parts[0] == "plugins" else runtime) / name
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(objects[offset : offset + int(size)])
destination.chmod(0o755 if mode == "100755" else 0o644)
offset += int(size) + 1
# Keep each selected plugin only in repo, including packaging/config files.
export_git_tree(source, entries, lambda path: (candidate if path.parts[0] == "plugins" else runtime) / path)
if not (runtime / "reme/__init__.py").is_file() or not (runtime / "pyproject.toml").is_file():
raise ValueError("Source revision is not a ReMe project")
if not (runtime / "reme/config" / spec.runtime.config).is_file():

View file

@ -8,11 +8,10 @@ import shutil
import yaml
from ..code.repository import resolve_revision, validate_repository, validate_tree
from ..code.snapshot import export_blob
from ..evaluation.artifacts import contained, read_json
from .code import git, tree_fingerprint
from .code import export_git_tree, git, git_tree, tree_fingerprint
from .data.models import fingerprint
from .spec import PreparationSpec, relative_path
from .spec import PreparationSpec
@dataclass(frozen=True)
@ -65,17 +64,7 @@ class EvaluationSource:
raise ValueError("Source fixed runtime changed during import")
repository = contained(self.home, "code/repo", directory=True)
candidate.mkdir()
entries = git(repository, "ls-tree", "-r", "-z", "--full-tree", self.commit, strip=False)
for entry in filter(None, entries.split("\0")):
header, name = entry.split("\t", 1)
mode, kind, oid = header.split()
path = Path(relative_path(name))
if mode not in {"100644", "100755"} or kind != "blob" or ".git" in path.parts:
raise ValueError(f"Candidate must contain regular files only: {name}")
destination = candidate / path
destination.parent.mkdir(parents=True, exist_ok=True)
export_blob(repository, oid, destination)
destination.chmod(0o755 if mode == "100755" else 0o644)
export_git_tree(repository, git_tree(repository, self.commit), lambda path: candidate / path)
# The isolated repository's init is the selected snapshot, not the optimizer baseline.
git(candidate, "init", "--initial-branch=init", "--template=")
git(candidate, "add", "--force", "--all")

View file

@ -0,0 +1,74 @@
"""Git exports preserve raw committed bytes, literal names, and executable modes."""
from pathlib import Path
from unittest.mock import Mock
import pytest
from reme_home.preparation import code
def test_batch_export_bypasses_filters_and_checkout(tmp_path, monkeypatch):
"""One batch reads all selected blobs, including binary and large files."""
repository = tmp_path / "repo"
repository.mkdir()
code.git(repository, "init", "--template=")
contents = {
"space name": b"$Format:%H$\n",
'quotes"\tand\nnewline': b"\x00binary\xff\n",
"脚本": b"#!/bin/sh\nexit 0\n",
"large": b"x" * 5_000_001,
"empty": b"",
".gitattributes": b"* export-ignore export-subst filter=blocked\n",
}
for name, data in contents.items():
(repository / name).write_bytes(data)
(repository / "脚本").chmod(0o755)
code.git(repository, "add", "--all")
code.git(repository, "-c", "user.name=Test", "-c", "user.email=test@localhost", "commit", "-m", "files")
code.git(repository, "config", "filter.blocked.process", "/bin/false")
code.git(repository, "config", "filter.blocked.required", "true")
for name in contents:
(repository / name).write_bytes(b"dirty checkout")
run = Mock(wraps=code.git_bytes)
monkeypatch.setattr(code, "git_bytes", run)
destination = tmp_path / "export"
code.export_git_tree(repository, code.git_tree(repository, "HEAD"), lambda path: destination / path)
assert run.call_count == 2
assert run.call_args.args == (repository, "cat-file", "--batch")
for name, data in contents.items():
target = destination / name
assert target.read_bytes() == data
assert target.stat().st_mode & 0o777 == (0o755 if name == "脚本" else 0o644)
@pytest.mark.parametrize(
"mode,kind,name",
[
("120000", "blob", "link"),
("160000", "commit", "submodule"),
("100644", "blob", ".git/config"),
("100644", "blob", "../escape"),
("100644", "blob", "/escape"),
("100644", "blob", "~escape"),
("100644", "blob", "folder\\escape"),
],
)
def test_tree_rejects_unsafe_paths_and_nonregular_files(monkeypatch, mode, kind, name):
"""Reject unsafe object metadata before any file is exported."""
monkeypatch.setattr(code, "git_bytes", Mock(return_value=f"{mode} {kind} {'a' * 40}\t{name}\0".encode()))
with pytest.raises(ValueError):
code.git_tree(Path("unused"), "HEAD")
def test_tree_selection_uses_literal_roots(monkeypatch):
"""Ignore out-of-scope links without mistaking similar prefixes for selected roots."""
raw = (
f"100644 blob {'a' * 40}\tplugin/file\0"
f"120000 blob {'b' * 40}\tplugin-other/link\0"
f"160000 commit {'c' * 40}\tother\0"
).encode()
monkeypatch.setattr(code, "git_bytes", Mock(return_value=raw))
assert code.git_tree(Path("unused"), "HEAD", [Path("plugin")]) == [(Path("plugin/file"), "100644", "a" * 40)]