mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-10-23 05:35:30 +00:00
48585f7523
* Add Docker benchmark sandbox and pooled runner * Fix benchmark worker reuse after Docker command timeout * Simplify benchmark worker requests and preserve artifacts * Harden benchmark sandbox reuse and isolate query state * Deduplicate benchmark sandbox cleanup and artifact checks * Report benchmark cleanup failures and preserve diagnostic artifacts * Handle optional Docker dependencies in benchmark unit tests * Keep benchmark Docker dependencies optional in CI * Fix cancellation of thread-pool background jobs * Finish background worker cleanup before propagating cancellation * Revert background worker cancellation changes * Fix benchmark worker plugin defaults before config validation * Stop interrupted benchmark containers before collecting diagnostics * Fix candidate exclusions and benchmark upload serialization * Fix late Docker creation cleanup and preserve query artifacts * Stream benchmark workspace archives and extract plan validation --------- Co-authored-by: jinli.yl <jinli.yl@alibaba-inc.com>
73 lines
2.5 KiB
Python
73 lines
2.5 KiB
Python
"""Small serializable contracts shared by the host and sandbox worker."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class JobResult:
|
|
"""Structured result returned by a direct ReMe job invocation.
|
|
|
|
``token_usage`` contains the input/output/total token delta for each
|
|
configured agent wrapper during this invocation. Missing provider usage is
|
|
represented by ``None`` at the metric level. ``job_call_counts`` contains
|
|
non-zero ReMe job invocation deltas, including nested job tools.
|
|
"""
|
|
|
|
success: bool
|
|
answer: Any = ""
|
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
error: str | None = None
|
|
token_usage: dict[str, dict[str, int | None]] = field(default_factory=dict)
|
|
job_call_counts: dict[str, int] = field(default_factory=dict)
|
|
|
|
@classmethod
|
|
def from_dict(cls, value: dict[str, Any]) -> "JobResult":
|
|
"""Build a result from the worker's JSON response."""
|
|
return cls(
|
|
success=bool(value.get("success")),
|
|
answer=value.get("answer", ""),
|
|
metadata=value.get("metadata") if isinstance(value.get("metadata"), dict) else {},
|
|
token_usage=value.get("token_usage") if isinstance(value.get("token_usage"), dict) else {},
|
|
job_call_counts=value.get("job_call_counts") if isinstance(value.get("job_call_counts"), dict) else {},
|
|
error=value.get("error") if isinstance(value.get("error"), str) else None,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ActionRecord:
|
|
"""Audit record for one command executed inside a case sandbox."""
|
|
|
|
name: str
|
|
exit_code: int
|
|
stdout: str
|
|
stderr: str
|
|
started_at: str
|
|
finished_at: str
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class JobRequest:
|
|
"""One job executed as part of a shared build Application lifecycle."""
|
|
|
|
job: str
|
|
arguments: dict[str, Any] = field(default_factory=dict)
|
|
memory_checkpoint: str | None = None
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class EvaluationQuery:
|
|
"""One answer request plus host-only judging metadata.
|
|
|
|
Only ``query_id``, ``question``, ``answer_arguments`` and ``answer_job``
|
|
cross the candidate sandbox boundary. The remaining fields are retained
|
|
on the trusted host for post-answer evaluation.
|
|
"""
|
|
|
|
query_id: str
|
|
question: str
|
|
golden_answer: Any
|
|
answer_arguments: dict[str, Any] = field(default_factory=dict)
|
|
answer_job: str = "agentic_answer"
|