mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-15 09:11:24 +00:00
refactor(home): consolidate Git exports and candidate validation
This commit is contained in:
parent
6cd9c9c6b2
commit
53cf655415
8 changed files with 155 additions and 127 deletions
|
|
@ -7,7 +7,7 @@ from uuid import uuid4
|
|||
|
||||
from pydantic import Field
|
||||
|
||||
from ..code.repository import current_branch, require_clean, resolve_revision, validate_repository, validate_tree
|
||||
from ..code.repository import current_branch, validate_candidate, validate_repository
|
||||
from ..code.snapshot import export_blob, tree
|
||||
from ..evaluation.artifacts import contained, read_json
|
||||
from ..evaluation.component import blocking
|
||||
|
|
@ -26,10 +26,7 @@ class CandidateGateReport(SpecModel):
|
|||
|
||||
def admission_context(home):
|
||||
"""Called under the workspace lock; retries and the baseline bypass admission."""
|
||||
repo, record = validate_repository(home)
|
||||
require_clean(repo)
|
||||
commit = resolve_revision(repo, "HEAD")
|
||||
validate_tree(repo, record, commit)
|
||||
repo, record, commit = validate_candidate(home)
|
||||
if commit == record["code"]["initial_commit"] or previous_candidate_result(home, commit) is not None:
|
||||
return None
|
||||
ancestors = set(git(repo, "rev-list", commit).splitlines())
|
||||
|
|
|
|||
|
|
@ -30,6 +30,15 @@ def validate_repository(home: Path, record: dict | None = None) -> tuple[Path, d
|
|||
return repository, record
|
||||
|
||||
|
||||
def validate_candidate(home: Path) -> tuple[Path, dict, str]:
|
||||
"""Validate a clean candidate HEAD before admission or construction reuse, under the caller's lock."""
|
||||
repository, record = validate_repository(home)
|
||||
require_clean(repository)
|
||||
revision = resolve_revision(repository, "HEAD")
|
||||
validate_tree(repository, record, revision)
|
||||
return repository, record, revision
|
||||
|
||||
|
||||
def current_branch(repository: Path) -> str:
|
||||
"""Reject detached HEAD so every candidate commit remains branch-reachable."""
|
||||
branch = git(repository, "branch", "--show-current")
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ from pathlib import Path
|
|||
from typing import Literal
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import tarfile
|
||||
|
||||
|
|
@ -12,7 +11,7 @@ from pydantic import Field, field_validator, model_validator
|
|||
|
||||
from deploy.benchmark_docker import SourceSnapshot
|
||||
|
||||
from ..preparation.code import git, tree_fingerprint
|
||||
from ..preparation.code import export_git_tree, git, git_tree, runtime_roots, tree_fingerprint
|
||||
from ..preparation.data.models import CaseSpec, DatasetManifest, fingerprint
|
||||
from .artifacts import contained, file_sha256, inspect_archive, read_json
|
||||
from ..preparation.spec import SpecModel
|
||||
|
|
@ -138,45 +137,11 @@ def snapshot_code(home: Path, record: dict):
|
|||
commit = git(repository, "rev-parse", "--verify", "HEAD^{commit}")
|
||||
branch = "init" if commit == record["code"]["initial_commit"] else git(repository, "branch", "--show-current")
|
||||
entity_id(branch)
|
||||
# NUL output validates exact paths without interpreting Git quoting rules.
|
||||
entries = subprocess.check_output(
|
||||
["git", "-C", str(repository), "ls-tree", "-r", "-z", commit],
|
||||
timeout=120,
|
||||
).split(b"\0")
|
||||
parsed = []
|
||||
for entry in entries:
|
||||
if not entry:
|
||||
continue
|
||||
header, name = entry.split(b"\t", 1)
|
||||
mode, kind, oid = header.decode().split()
|
||||
relative = Path(name.decode())
|
||||
if (
|
||||
relative.is_absolute()
|
||||
or ".." in relative.parts
|
||||
or ".git" in relative.parts
|
||||
or kind != "blob"
|
||||
or mode not in {"100644", "100755"}
|
||||
):
|
||||
raise ValueError(f"Candidate must contain contained regular files: {relative}")
|
||||
if not any(
|
||||
relative.is_relative_to(path)
|
||||
for path in [
|
||||
Path("reme"),
|
||||
Path("pyproject.toml"),
|
||||
Path("README.md"),
|
||||
Path("LICENSE"),
|
||||
*[Path("plugins") / p for p in record["code"]["plugins"]],
|
||||
]
|
||||
):
|
||||
entries = git_tree(repository, commit)
|
||||
roots = runtime_roots(record["code"]["plugins"])
|
||||
for relative, _, _ in entries:
|
||||
if not any(relative.is_relative_to(path) for path in roots):
|
||||
raise ValueError(f"Candidate path is outside the prepared runtime projects: {relative}")
|
||||
parsed.append((relative, mode, oid))
|
||||
contents = subprocess.run(
|
||||
["git", "-C", str(repository), "cat-file", "--batch"],
|
||||
input="".join(oid + "\n" for _, _, oid in parsed).encode(),
|
||||
capture_output=True,
|
||||
check=True,
|
||||
timeout=120,
|
||||
).stdout
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="evaluation-source-",
|
||||
dir=contained(home, "tmp", directory=True),
|
||||
|
|
@ -193,18 +158,7 @@ def snapshot_code(home: Path, record: dict):
|
|||
shutil.rmtree(target)
|
||||
elif target.exists():
|
||||
target.unlink()
|
||||
offset = 0
|
||||
for relative, mode, oid in parsed:
|
||||
end = contents.index(b"\n", offset)
|
||||
found, kind, size = contents[offset:end].decode().split()
|
||||
if found != oid or kind != "blob":
|
||||
raise ValueError("Invalid candidate Git object")
|
||||
offset = end + 1
|
||||
target = combined / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(contents[offset : offset + int(size)])
|
||||
target.chmod(0o755 if mode == "100755" else 0o644)
|
||||
offset += int(size) + 1
|
||||
export_git_tree(repository, entries, lambda path: combined / path)
|
||||
for project in record["code"]["install_projects"]:
|
||||
if not (combined / project / "pyproject.toml").is_file():
|
||||
raise ValueError(f"Candidate is missing install project {project}")
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import random
|
|||
import subprocess
|
||||
import tarfile
|
||||
|
||||
from ..code.repository import current_branch, require_clean, resolve_revision, validate_repository, validate_tree
|
||||
from ..code.repository import current_branch, validate_candidate, validate_repository
|
||||
from ..code.snapshot import tree
|
||||
from ..preparation.code import git
|
||||
from ..preparation.data.models import fingerprint
|
||||
|
|
@ -43,10 +43,7 @@ def construction_fingerprints(record, dataset, spec, environment):
|
|||
|
||||
def reuse_context(home, spec, environment):
|
||||
"""Find the nearest valid ancestor before review or sampling, under the workspace lock."""
|
||||
repo, record = validate_repository(home)
|
||||
require_clean(repo)
|
||||
commit = resolve_revision(repo, "HEAD")
|
||||
validate_tree(repo, record, commit)
|
||||
repo, record, commit = validate_candidate(home)
|
||||
identity = {"candidate_commit": commit, "branch_name": current_branch(repo)}
|
||||
previous = previous_candidate_result(home, commit)
|
||||
if previous is not None:
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ backends:
|
|||
home_optimizer_agent: reme_home.agent.optimizer:OptimizerAgentWrapper
|
||||
home_diagnose_step: reme_home.agent.diagnostic:HomeDiagnoseStep
|
||||
home_optimize_step: reme_home.agent.optimizer:HomeOptimizeStep
|
||||
home_workflow_agent: reme_home.agent.wrapper:CaseStudyAgentWrapper
|
||||
home_case_study_step: reme_home.agent.case_study:HomeCaseStudyStep
|
||||
home_case_study_agent: reme_home.agent.wrapper:CaseStudyAgentWrapper
|
||||
home_trace_query_evidence_step: reme_home.evaluation.evidence_step:HomeTraceQueryEvidenceStep
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""Export trusted source inputs without running their code or installing them."""
|
||||
|
||||
import hashlib
|
||||
from collections.abc import Callable
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
|
|
@ -10,7 +11,7 @@ import tomllib
|
|||
from .spec import PreparationSpec, relative_path
|
||||
|
||||
|
||||
def git_bytes(repository: Path, *arguments: str) -> bytes:
|
||||
def git_bytes(repository: Path, *arguments: str, input_bytes: bytes | None = None) -> bytes:
|
||||
"""Run bounded, noninteractive Git commands without a shell or user hooks."""
|
||||
environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")}
|
||||
environment.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, GIT_TERMINAL_PROMPT="0")
|
||||
|
|
@ -29,6 +30,7 @@ def git_bytes(repository: Path, *arguments: str) -> bytes:
|
|||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
input=input_bytes,
|
||||
env=environment,
|
||||
timeout=120,
|
||||
)
|
||||
|
|
@ -41,6 +43,59 @@ def git(repository: Path, *arguments: str, strip: bool = True) -> str:
|
|||
return output.strip() if strip else output
|
||||
|
||||
|
||||
def runtime_roots(plugins: list[str]) -> list[Path]:
|
||||
"""Return the core and plugin project roots allowed in prepared sources."""
|
||||
return [
|
||||
Path("reme"),
|
||||
Path("pyproject.toml"),
|
||||
Path("README.md"),
|
||||
Path("LICENSE"),
|
||||
*(Path("plugins") / name for name in plugins),
|
||||
]
|
||||
|
||||
|
||||
def git_tree(repository: Path, revision: str, roots: list[Path] | None = None) -> list[tuple[Path, str, str]]:
|
||||
"""Select regular Git files using exact NUL-delimited paths, retaining modes and object IDs."""
|
||||
entries = []
|
||||
raw_tree = git_bytes(repository, "ls-tree", "-r", "-z", "--full-tree", revision)
|
||||
for entry in filter(None, raw_tree.split(b"\0")):
|
||||
header, raw_name = entry.split(b"\t", 1)
|
||||
path = Path(relative_path(raw_name.decode("utf-8")))
|
||||
if roots is not None and not any(path.is_relative_to(root) for root in roots):
|
||||
continue
|
||||
mode, kind, oid = header.decode().split()
|
||||
if mode not in {"100644", "100755"} or kind != "blob" or ".git" in path.parts:
|
||||
raise ValueError(f"Git source must contain regular files only: {path}")
|
||||
entries.append((path, mode, oid))
|
||||
return entries
|
||||
|
||||
|
||||
def export_git_tree(
|
||||
repository: Path,
|
||||
entries: list[tuple[Path, str, str]],
|
||||
destination: Callable[[Path], Path],
|
||||
) -> None:
|
||||
"""Batch-export raw blobs without checkout/archive filters or browsing size limits."""
|
||||
objects = git_bytes(
|
||||
repository,
|
||||
"cat-file",
|
||||
"--batch",
|
||||
input_bytes="".join(oid + "\n" for _, _, oid in entries).encode(),
|
||||
)
|
||||
offset = 0
|
||||
for path, mode, oid in entries:
|
||||
boundary = objects.index(b"\n", offset)
|
||||
found, kind, size = objects[offset:boundary].decode().split()
|
||||
if found != oid or kind != "blob":
|
||||
raise ValueError(f"Invalid Git object: {path}")
|
||||
offset = boundary + 1
|
||||
target = destination(path)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(objects[offset : offset + int(size)])
|
||||
target.chmod(0o755 if mode == "100755" else 0o644)
|
||||
offset += int(size) + 1
|
||||
|
||||
|
||||
def tree_fingerprint(root: Path) -> str:
|
||||
"""Hash relative names and file bytes, excluding candidate Git metadata."""
|
||||
digest = hashlib.sha256()
|
||||
|
|
@ -79,57 +134,11 @@ def prepare_code(spec: PreparationSpec, source: Path, runtime: Path, candidate:
|
|||
is editable. The initial candidate commit supplies the original baseline.
|
||||
"""
|
||||
revision = git(source, "rev-parse", "--verify", "--end-of-options", spec.runtime.source_ref + "^{commit}")
|
||||
# Read object contents directly, bypassing archive attributes and checkout filters.
|
||||
environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")}
|
||||
raw_tree = subprocess.run(
|
||||
["git", "-C", str(source), "ls-tree", "-r", "-z", "--full-tree", revision],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=120,
|
||||
env=environment,
|
||||
).stdout
|
||||
roots = [
|
||||
Path("reme"),
|
||||
Path("pyproject.toml"),
|
||||
Path("README.md"),
|
||||
Path("LICENSE"),
|
||||
*(Path("plugins") / name for name in spec.runtime.plugins),
|
||||
]
|
||||
entries = []
|
||||
for entry in raw_tree.split(b"\0"):
|
||||
if not entry:
|
||||
continue
|
||||
header, raw_name = entry.split(b"\t", maxsplit=1)
|
||||
name = Path(relative_path(raw_name.decode("utf-8")))
|
||||
if not any(name.is_relative_to(root) for root in roots):
|
||||
continue
|
||||
mode, kind, object_id = header.decode().split()
|
||||
if mode not in {"100644", "100755"} or kind != "blob":
|
||||
raise ValueError(f"Runtime source must contain regular files only: {name}")
|
||||
entries.append((name, mode, object_id))
|
||||
objects = subprocess.run(
|
||||
["git", "-C", str(source), "cat-file", "--batch"],
|
||||
input="".join(object_id + "\n" for _, _, object_id in entries).encode(),
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=120,
|
||||
env=environment,
|
||||
).stdout
|
||||
entries = git_tree(source, revision, runtime_roots(spec.runtime.plugins))
|
||||
runtime.mkdir()
|
||||
candidate.mkdir()
|
||||
offset = 0
|
||||
for name, mode, object_id in entries:
|
||||
boundary = objects.index(b"\n", offset)
|
||||
found, kind, size = objects[offset:boundary].decode().split()
|
||||
if found != object_id or kind != "blob":
|
||||
raise ValueError(f"Invalid Git object: {name}")
|
||||
offset = boundary + 1
|
||||
# Keep each selected plugin only in repo, including packaging/config files.
|
||||
destination = (candidate if name.parts[0] == "plugins" else runtime) / name
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_bytes(objects[offset : offset + int(size)])
|
||||
destination.chmod(0o755 if mode == "100755" else 0o644)
|
||||
offset += int(size) + 1
|
||||
# Keep each selected plugin only in repo, including packaging/config files.
|
||||
export_git_tree(source, entries, lambda path: (candidate if path.parts[0] == "plugins" else runtime) / path)
|
||||
if not (runtime / "reme/__init__.py").is_file() or not (runtime / "pyproject.toml").is_file():
|
||||
raise ValueError("Source revision is not a ReMe project")
|
||||
if not (runtime / "reme/config" / spec.runtime.config).is_file():
|
||||
|
|
|
|||
|
|
@ -8,11 +8,10 @@ import shutil
|
|||
import yaml
|
||||
|
||||
from ..code.repository import resolve_revision, validate_repository, validate_tree
|
||||
from ..code.snapshot import export_blob
|
||||
from ..evaluation.artifacts import contained, read_json
|
||||
from .code import git, tree_fingerprint
|
||||
from .code import export_git_tree, git, git_tree, tree_fingerprint
|
||||
from .data.models import fingerprint
|
||||
from .spec import PreparationSpec, relative_path
|
||||
from .spec import PreparationSpec
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -65,17 +64,7 @@ class EvaluationSource:
|
|||
raise ValueError("Source fixed runtime changed during import")
|
||||
repository = contained(self.home, "code/repo", directory=True)
|
||||
candidate.mkdir()
|
||||
entries = git(repository, "ls-tree", "-r", "-z", "--full-tree", self.commit, strip=False)
|
||||
for entry in filter(None, entries.split("\0")):
|
||||
header, name = entry.split("\t", 1)
|
||||
mode, kind, oid = header.split()
|
||||
path = Path(relative_path(name))
|
||||
if mode not in {"100644", "100755"} or kind != "blob" or ".git" in path.parts:
|
||||
raise ValueError(f"Candidate must contain regular files only: {name}")
|
||||
destination = candidate / path
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
export_blob(repository, oid, destination)
|
||||
destination.chmod(0o755 if mode == "100755" else 0o644)
|
||||
export_git_tree(repository, git_tree(repository, self.commit), lambda path: candidate / path)
|
||||
# The isolated repository's init is the selected snapshot, not the optimizer baseline.
|
||||
git(candidate, "init", "--initial-branch=init", "--template=")
|
||||
git(candidate, "add", "--force", "--all")
|
||||
|
|
|
|||
74
plugins/home/tests/test_git_export.py
Normal file
74
plugins/home/tests/test_git_export.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
"""Git exports preserve raw committed bytes, literal names, and executable modes."""
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from reme_home.preparation import code
|
||||
|
||||
|
||||
def test_batch_export_bypasses_filters_and_checkout(tmp_path, monkeypatch):
|
||||
"""One batch reads all selected blobs, including binary and large files."""
|
||||
repository = tmp_path / "repo"
|
||||
repository.mkdir()
|
||||
code.git(repository, "init", "--template=")
|
||||
contents = {
|
||||
"space name": b"$Format:%H$\n",
|
||||
'quotes"\tand\nnewline': b"\x00binary\xff\n",
|
||||
"脚本": b"#!/bin/sh\nexit 0\n",
|
||||
"large": b"x" * 5_000_001,
|
||||
"empty": b"",
|
||||
".gitattributes": b"* export-ignore export-subst filter=blocked\n",
|
||||
}
|
||||
for name, data in contents.items():
|
||||
(repository / name).write_bytes(data)
|
||||
(repository / "脚本").chmod(0o755)
|
||||
code.git(repository, "add", "--all")
|
||||
code.git(repository, "-c", "user.name=Test", "-c", "user.email=test@localhost", "commit", "-m", "files")
|
||||
code.git(repository, "config", "filter.blocked.process", "/bin/false")
|
||||
code.git(repository, "config", "filter.blocked.required", "true")
|
||||
for name in contents:
|
||||
(repository / name).write_bytes(b"dirty checkout")
|
||||
run = Mock(wraps=code.git_bytes)
|
||||
monkeypatch.setattr(code, "git_bytes", run)
|
||||
destination = tmp_path / "export"
|
||||
|
||||
code.export_git_tree(repository, code.git_tree(repository, "HEAD"), lambda path: destination / path)
|
||||
|
||||
assert run.call_count == 2
|
||||
assert run.call_args.args == (repository, "cat-file", "--batch")
|
||||
for name, data in contents.items():
|
||||
target = destination / name
|
||||
assert target.read_bytes() == data
|
||||
assert target.stat().st_mode & 0o777 == (0o755 if name == "脚本" else 0o644)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mode,kind,name",
|
||||
[
|
||||
("120000", "blob", "link"),
|
||||
("160000", "commit", "submodule"),
|
||||
("100644", "blob", ".git/config"),
|
||||
("100644", "blob", "../escape"),
|
||||
("100644", "blob", "/escape"),
|
||||
("100644", "blob", "~escape"),
|
||||
("100644", "blob", "folder\\escape"),
|
||||
],
|
||||
)
|
||||
def test_tree_rejects_unsafe_paths_and_nonregular_files(monkeypatch, mode, kind, name):
|
||||
"""Reject unsafe object metadata before any file is exported."""
|
||||
monkeypatch.setattr(code, "git_bytes", Mock(return_value=f"{mode} {kind} {'a' * 40}\t{name}\0".encode()))
|
||||
with pytest.raises(ValueError):
|
||||
code.git_tree(Path("unused"), "HEAD")
|
||||
|
||||
|
||||
def test_tree_selection_uses_literal_roots(monkeypatch):
|
||||
"""Ignore out-of-scope links without mistaking similar prefixes for selected roots."""
|
||||
raw = (
|
||||
f"100644 blob {'a' * 40}\tplugin/file\0"
|
||||
f"120000 blob {'b' * 40}\tplugin-other/link\0"
|
||||
f"160000 commit {'c' * 40}\tother\0"
|
||||
).encode()
|
||||
monkeypatch.setattr(code, "git_bytes", Mock(return_value=raw))
|
||||
assert code.git_tree(Path("unused"), "HEAD", [Path("plugin")]) == [(Path("plugin/file"), "100644", "a" * 40)]
|
||||
Loading…
Add table
Reference in a new issue