mirror of
https://github.com/HKUDS/OpenSpace.git
synced 2026-08-28 05:15:00 +00:00
618 lines
19 KiB
Python
618 lines
19 KiB
Python
import asyncio
|
|
import json
|
|
from dataclasses import replace
|
|
from types import SimpleNamespace
|
|
|
|
from openspace.application import OpenSpaceConfig
|
|
from openspace.skill_engine.evidence.types import (
|
|
EvidencePacket,
|
|
EvidenceScope,
|
|
PacketBudget,
|
|
ResourceRef,
|
|
)
|
|
from openspace.skill_engine.evolution.authoring import (
|
|
SkillEvolverAuthoringBackend,
|
|
_normalize_captured_edit_content,
|
|
)
|
|
from openspace.skill_engine.evolution.capture_semantic import (
|
|
CaptureContractSemanticReviewer,
|
|
)
|
|
from openspace.skill_engine.evolution.engine import EvolutionEngine
|
|
from openspace.skill_engine.evolution.validator import (
|
|
EvolutionValidator,
|
|
ValidationResult,
|
|
_has_secondary_frontmatter,
|
|
)
|
|
|
|
|
|
class _FakeClient:
|
|
def __init__(
|
|
self,
|
|
payload: str,
|
|
*,
|
|
stop_reason: str = "stop",
|
|
tool_calls=None,
|
|
) -> None:
|
|
self.payload = payload
|
|
self.stop_reason = stop_reason
|
|
self.tool_calls = tool_calls or []
|
|
self.calls = []
|
|
|
|
async def call_model_with_fallback(self, **kwargs):
|
|
self.calls.append(kwargs)
|
|
return SimpleNamespace(
|
|
assistant_message={"content": self.payload},
|
|
tool_calls=self.tool_calls,
|
|
stop_reason=self.stop_reason,
|
|
)
|
|
|
|
|
|
def _review_payload(**overrides):
|
|
payload = {
|
|
"outcome": "approve",
|
|
"support": "fully_supported",
|
|
"validation_relation": "independent_method",
|
|
"skill_fidelity": "bounded",
|
|
"artifact_quality": "sound",
|
|
"supporting_validation_ref_ids": ["tool:validation"],
|
|
"unsupported_claims": [],
|
|
"reuse_quality": "reusable",
|
|
"safety": "safe",
|
|
"reasons": ["A second decoder produced the same float."],
|
|
}
|
|
payload.update(overrides)
|
|
return payload
|
|
|
|
|
|
def test_semantic_reviewer_approves_bounded_independent_method() -> None:
|
|
client = _FakeClient(json.dumps(_review_payload()))
|
|
reviewer = CaptureContractSemanticReviewer(
|
|
client,
|
|
model="test-model",
|
|
max_tokens=1024,
|
|
)
|
|
|
|
result = asyncio.run(
|
|
reviewer.validate(
|
|
_authoring(),
|
|
_packet(),
|
|
_decision(),
|
|
SimpleNamespace(),
|
|
)
|
|
)
|
|
|
|
assert result["outcome"] == "approve"
|
|
assert "capture_support:fully_supported" in result["warnings"]
|
|
assert [tool.name for tool in client.calls[0]["tools"]] == [
|
|
"submit_capture_review"
|
|
]
|
|
assert client.calls[0]["tool_choice"] == "required"
|
|
assert client.calls[0]["enable_thinking"] is False
|
|
assert client.calls[0]["max_tokens"] == 1024
|
|
prompt = client.calls[0]["messages"][1]["content"]
|
|
assert "A limitation does not" in prompt
|
|
assert "supporting_validation_ref_ids" in prompt
|
|
|
|
|
|
def test_semantic_reviewer_prefers_required_tool_result_over_truncation() -> None:
|
|
payload = _review_payload(
|
|
validation_relation="cross_implementation_comparison",
|
|
reasons=["Both implementations returned the same values."],
|
|
)
|
|
client = _FakeClient(
|
|
"",
|
|
stop_reason="length",
|
|
tool_calls=[
|
|
{
|
|
"id": "call_review",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "submit_capture_review",
|
|
"arguments": json.dumps(payload),
|
|
},
|
|
}
|
|
],
|
|
)
|
|
reviewer = CaptureContractSemanticReviewer(client, model="test-model")
|
|
|
|
result = asyncio.run(
|
|
reviewer.validate(_authoring(), _packet(), _decision(), SimpleNamespace())
|
|
)
|
|
|
|
assert result["outcome"] == "approve"
|
|
assert "capture_support:fully_supported" in result["warnings"]
|
|
|
|
|
|
def test_semantic_review_config_is_fail_closed_by_default(monkeypatch) -> None:
|
|
monkeypatch.setenv(
|
|
"OPENSPACE_EVOLUTION_CAPTURE_SEMANTIC_VALIDATION_ENABLED",
|
|
"true",
|
|
)
|
|
monkeypatch.setenv(
|
|
"OPENSPACE_EVOLUTION_CAPTURE_SEMANTIC_VALIDATION_MAX_TOKENS",
|
|
"1536",
|
|
)
|
|
config = OpenSpaceConfig()
|
|
|
|
assert config.evolution_capture_semantic_validation_enabled is True
|
|
assert config.evolution_capture_semantic_validation_max_tokens == 1536
|
|
|
|
|
|
def test_semantic_reviewer_rejects_same_implementation_self_check() -> None:
|
|
client = _FakeClient(
|
|
json.dumps(
|
|
_review_payload(
|
|
validation_relation="same_implementation",
|
|
reasons=["The validation reran the same detector."],
|
|
)
|
|
)
|
|
)
|
|
reviewer = CaptureContractSemanticReviewer(client, model="test-model")
|
|
|
|
result = asyncio.run(
|
|
reviewer.validate(
|
|
_authoring(),
|
|
_packet(),
|
|
_decision(),
|
|
SimpleNamespace(),
|
|
)
|
|
)
|
|
|
|
assert result["outcome"] == "reject"
|
|
assert "capture_validation_relation:same_implementation" in result["warnings"]
|
|
|
|
|
|
def test_semantic_reviewer_rejects_defective_authored_artifact() -> None:
|
|
client = _FakeClient(
|
|
json.dumps(
|
|
_review_payload(
|
|
artifact_quality="defective",
|
|
reasons=["The helper returns triples but dict() expects pairs."],
|
|
)
|
|
)
|
|
)
|
|
reviewer = CaptureContractSemanticReviewer(client, model="test-model")
|
|
|
|
result = asyncio.run(
|
|
reviewer.validate(_authoring(), _packet(), _decision(), SimpleNamespace())
|
|
)
|
|
|
|
assert result["outcome"] == "reject"
|
|
assert "capture_artifact_quality:defective" in result["warnings"]
|
|
|
|
|
|
def test_semantic_reviewer_rejects_unvalidated_skill_claim() -> None:
|
|
client = _FakeClient(
|
|
json.dumps(
|
|
_review_payload(
|
|
unsupported_claims=[
|
|
"The skill computes all date ranges, but only today was checked."
|
|
],
|
|
reasons=["The validation ref establishes only today's count."],
|
|
)
|
|
)
|
|
)
|
|
reviewer = CaptureContractSemanticReviewer(client, model="test-model")
|
|
|
|
result = asyncio.run(
|
|
reviewer.validate(_authoring(), _packet(), _decision(), SimpleNamespace())
|
|
)
|
|
|
|
assert result["outcome"] == "reject"
|
|
assert any(
|
|
warning.startswith("capture_unsupported_claim:")
|
|
for warning in result["warnings"]
|
|
)
|
|
|
|
|
|
def test_semantic_reviewer_rejects_invalid_validation_ref_binding() -> None:
|
|
client = _FakeClient(
|
|
json.dumps(
|
|
_review_payload(
|
|
supporting_validation_ref_ids=["tool:procedure"],
|
|
reasons=["The procedure ref was incorrectly used as validation."],
|
|
)
|
|
)
|
|
)
|
|
reviewer = CaptureContractSemanticReviewer(client, model="test-model")
|
|
|
|
result = asyncio.run(
|
|
reviewer.validate(_authoring(), _packet(), _decision(), SimpleNamespace())
|
|
)
|
|
|
|
assert result["outcome"] == "reject"
|
|
assert "capture_supporting_validation_refs:invalid" in result["warnings"]
|
|
|
|
|
|
def test_semantic_reviewer_rejects_incomplete_high_impact_skill() -> None:
|
|
client = _FakeClient(
|
|
json.dumps(
|
|
_review_payload(
|
|
safety="unsafe_or_incomplete",
|
|
reasons=["The sanitizer documents known executable bypasses."],
|
|
)
|
|
)
|
|
)
|
|
reviewer = CaptureContractSemanticReviewer(client, model="test-model")
|
|
|
|
result = asyncio.run(
|
|
reviewer.validate(_authoring(), _packet(), _decision(), SimpleNamespace())
|
|
)
|
|
|
|
assert result["outcome"] == "reject"
|
|
assert "capture_safety:unsafe_or_incomplete" in result["warnings"]
|
|
|
|
|
|
def test_semantic_reviewer_rejects_trivial_capture() -> None:
|
|
client = _FakeClient(
|
|
json.dumps(
|
|
_review_payload(
|
|
reuse_quality="trivial",
|
|
reasons=["The artifact only restates one generic command."],
|
|
)
|
|
)
|
|
)
|
|
reviewer = CaptureContractSemanticReviewer(client, model="test-model")
|
|
|
|
result = asyncio.run(
|
|
reviewer.validate(_authoring(), _packet(), _decision(), SimpleNamespace())
|
|
)
|
|
|
|
assert result["outcome"] == "reject"
|
|
assert "capture_reuse_quality:trivial" in result["warnings"]
|
|
|
|
|
|
def test_semantic_reviewer_retries_invalid_structured_response_once() -> None:
|
|
valid_payload = _review_payload(
|
|
reasons=["The independent decoder returned the same value."],
|
|
)
|
|
|
|
class RecoveringClient:
|
|
def __init__(self):
|
|
self.calls = []
|
|
|
|
async def call_model_with_fallback(self, **kwargs):
|
|
self.calls.append(kwargs)
|
|
if len(self.calls) == 1:
|
|
return SimpleNamespace(
|
|
assistant_message={"content": "not json"},
|
|
tool_calls=[],
|
|
stop_reason="stop",
|
|
)
|
|
return SimpleNamespace(
|
|
assistant_message={"content": ""},
|
|
tool_calls=[
|
|
{
|
|
"function": {
|
|
"name": "submit_capture_review",
|
|
"arguments": valid_payload,
|
|
}
|
|
}
|
|
],
|
|
stop_reason="tool_calls",
|
|
)
|
|
|
|
client = RecoveringClient()
|
|
reviewer = CaptureContractSemanticReviewer(client, model="test-model")
|
|
|
|
result = asyncio.run(
|
|
reviewer.validate(_authoring(), _packet(), _decision(), SimpleNamespace())
|
|
)
|
|
|
|
assert result["outcome"] == "approve"
|
|
assert len(client.calls) == 2
|
|
assert "previous response did not satisfy" in client.calls[1]["messages"][1][
|
|
"content"
|
|
]
|
|
|
|
|
|
def test_semantic_reviewer_fails_closed_on_invalid_or_truncated_output() -> None:
|
|
invalid = CaptureContractSemanticReviewer(
|
|
_FakeClient("not json"),
|
|
model="test-model",
|
|
)
|
|
truncated = CaptureContractSemanticReviewer(
|
|
_FakeClient("{}", stop_reason="length"),
|
|
model="test-model",
|
|
)
|
|
|
|
invalid_result = asyncio.run(
|
|
invalid.validate(_authoring(), _packet(), _decision(), SimpleNamespace())
|
|
)
|
|
truncated_result = asyncio.run(
|
|
truncated.validate(_authoring(), _packet(), _decision(), SimpleNamespace())
|
|
)
|
|
|
|
assert invalid_result == {
|
|
"outcome": "reject",
|
|
"warnings": ["capture_semantic_invalid_json"],
|
|
}
|
|
assert truncated_result == {
|
|
"outcome": "reject",
|
|
"warnings": ["capture_semantic_response_truncated"],
|
|
}
|
|
|
|
|
|
def test_async_validator_applies_semantic_rejection() -> None:
|
|
async def reject(*_args):
|
|
return {
|
|
"outcome": "reject",
|
|
"warnings": ["capture_support:partially_supported"],
|
|
}
|
|
|
|
validator = EvolutionValidator(
|
|
semantic_validator=reject,
|
|
semantic_enabled=True,
|
|
)
|
|
base = ValidationResult(
|
|
validation_id="val_test",
|
|
authoring_id="auth_test",
|
|
decision_id="dec_test",
|
|
packet_id="pkt_test",
|
|
outcome="approve",
|
|
)
|
|
validator._run_validation = lambda *_args, **_kwargs: base
|
|
validator._persist_fail_closed = lambda result: result
|
|
|
|
result = asyncio.run(
|
|
validator.validate_async(
|
|
_authoring(),
|
|
_packet(),
|
|
_decision(),
|
|
SimpleNamespace(),
|
|
)
|
|
)
|
|
|
|
assert result.outcome == "reject"
|
|
assert result.checked_by.endswith("+semantic")
|
|
assert result.semantic_warnings == ["capture_support:partially_supported"]
|
|
|
|
|
|
def test_engine_prefers_async_validator_entrypoint() -> None:
|
|
calls = []
|
|
|
|
class Validator:
|
|
async def validate_async(self, *_args):
|
|
calls.append("async")
|
|
return "ok"
|
|
|
|
def validate(self, *_args):
|
|
raise AssertionError("sync validator must not be selected")
|
|
|
|
engine = EvolutionEngine()
|
|
result = asyncio.run(
|
|
engine._call_validator(
|
|
Validator(),
|
|
SimpleNamespace(),
|
|
SimpleNamespace(),
|
|
SimpleNamespace(),
|
|
SimpleNamespace(),
|
|
SimpleNamespace(),
|
|
)
|
|
)
|
|
|
|
assert result == "ok"
|
|
assert calls == ["async"]
|
|
|
|
|
|
def test_semantic_validation_failure_becomes_audit_candidate() -> None:
|
|
class Authoring:
|
|
async def author_from_action_packet(self, _packet):
|
|
return SimpleNamespace(
|
|
status="staged",
|
|
staged_edit=SimpleNamespace(),
|
|
)
|
|
|
|
class PacketBuilder:
|
|
def build_action_packet(self, _scope):
|
|
return SimpleNamespace(packet_type="action")
|
|
|
|
def build_validator_packet(self, _authoring):
|
|
return SimpleNamespace(packet_type="validator")
|
|
|
|
class Validator:
|
|
async def validate_async(self, *_args):
|
|
return ValidationResult(
|
|
validation_id="val_reject",
|
|
authoring_id="auth_test",
|
|
decision_id="dec_test",
|
|
packet_id="pkt_validator",
|
|
outcome="reject",
|
|
semantic_warnings=["capture_support:partially_supported"],
|
|
)
|
|
|
|
class CandidateStore:
|
|
def create_or_merge(self, **kwargs):
|
|
return SimpleNamespace(reason=kwargs["reason"])
|
|
|
|
engine = EvolutionEngine(
|
|
packet_builder=PacketBuilder(),
|
|
authoring_backend=Authoring(),
|
|
validator=Validator(),
|
|
candidate_store=CandidateStore(),
|
|
)
|
|
|
|
outcome = asyncio.run(
|
|
engine._author_validate_commit(
|
|
SimpleNamespace(
|
|
decision_id="dec_test",
|
|
proposed_action="CAPTURED",
|
|
),
|
|
SimpleNamespace(admission_id="adm_test", outcome="direct"),
|
|
SimpleNamespace(packet_type="analysis"),
|
|
SimpleNamespace(job_id="job_test"),
|
|
)
|
|
)
|
|
|
|
assert outcome.committed_action is None
|
|
assert outcome.candidate.reason == "semantic_validation_failed"
|
|
assert outcome.blocked_reason == "validation_failed"
|
|
|
|
|
|
def test_captured_authoring_discards_model_narration_before_frontmatter() -> None:
|
|
content = (
|
|
"I have enough evidence to write the skill.\n\n"
|
|
"---\nname: recover-floats\ndescription: Decode floats.\n---\n"
|
|
"# Recover floats\n"
|
|
)
|
|
|
|
normalized = _normalize_captured_edit_content(content)
|
|
|
|
assert normalized.startswith("---\nname: recover-floats")
|
|
assert "I have enough evidence" not in normalized
|
|
|
|
|
|
def test_captured_authoring_uses_contract_without_audit_tools(tmp_path) -> None:
|
|
class Evolver:
|
|
_model = "test-model"
|
|
|
|
def __init__(self):
|
|
self.available_tools = None
|
|
|
|
async def _run_evolution_loop(self, _prompt, ctx):
|
|
self.available_tools = ctx.available_tools
|
|
return SimpleNamespace(
|
|
edit_content=(
|
|
"---\nname: decode-float\ndescription: Decode one float.\n---\n"
|
|
"# Decode one float\n"
|
|
),
|
|
overlay_fields={},
|
|
overlay_metadata={},
|
|
intent_spec={},
|
|
eval_plan={},
|
|
change_summary="Decode one float.",
|
|
)
|
|
|
|
evolver = Evolver()
|
|
backend = SkillEvolverAuthoringBackend(
|
|
evolver,
|
|
tmp_path / "staging",
|
|
evidence_store=None,
|
|
)
|
|
base_packet = _packet()
|
|
selected_refs = dict(base_packet.selected_refs)
|
|
selected_refs["decision_rationale_ref"] = [
|
|
ResourceRef(
|
|
ref_id="decision:dec_test",
|
|
ref_type="decision_rationale_ref",
|
|
preview="Capture the bounded float decoder.",
|
|
metadata={
|
|
"decision_id": "dec_test",
|
|
"proposed_action": "CAPTURED",
|
|
"reason_summary": "Capture the bounded float decoder.",
|
|
"proposal_contract": _decision().proposal_contract,
|
|
},
|
|
)
|
|
]
|
|
selected_refs["admission_result_ref"] = [
|
|
ResourceRef(
|
|
ref_id="admission:adm_test",
|
|
ref_type="admission_result_ref",
|
|
metadata={
|
|
"admission_id": "adm_test",
|
|
"outcome": "direct",
|
|
"source_validation_passed": True,
|
|
},
|
|
)
|
|
]
|
|
packet = replace(
|
|
base_packet,
|
|
packet_type="action",
|
|
selected_refs=selected_refs,
|
|
instructions={"capture_destination_root": str(tmp_path / "skills")},
|
|
)
|
|
|
|
result = asyncio.run(backend.author_from_action_packet(packet))
|
|
|
|
assert result.status == "staged"
|
|
assert evolver.available_tools == []
|
|
|
|
|
|
def test_secondary_frontmatter_is_rejected_but_fenced_example_is_allowed() -> None:
|
|
malformed = (
|
|
"---\nname: outer\ndescription: Outer.\n---\n"
|
|
"Narration\n---\nname: inner\ndescription: Inner.\n---\n"
|
|
)
|
|
documented = (
|
|
"---\nname: outer\ndescription: Outer.\n---\n"
|
|
"```yaml\n---\nname: example\n---\n```\n"
|
|
)
|
|
|
|
assert _has_secondary_frontmatter(malformed) is True
|
|
assert _has_secondary_frontmatter(documented) is False
|
|
failures = EvolutionValidator()._schema_failures(
|
|
{"SKILL.md": malformed},
|
|
"CAPTURED",
|
|
SimpleNamespace(),
|
|
)
|
|
assert "multiple_frontmatter_blocks" in failures
|
|
|
|
|
|
def _decision():
|
|
return SimpleNamespace(
|
|
proposed_action="CAPTURED",
|
|
proposal_contract={
|
|
"capability": "Decode one float and verify it independently.",
|
|
"preconditions": ["The section is readable."],
|
|
"procedure_refs": ["tool:procedure"],
|
|
"validation_refs": ["tool:validation"],
|
|
"validation_summary": "A second decoder returned the same value.",
|
|
"limitations": [],
|
|
},
|
|
)
|
|
|
|
|
|
def _authoring():
|
|
return SimpleNamespace(
|
|
staged_edit=SimpleNamespace(
|
|
content_snapshot={
|
|
"SKILL.md": (
|
|
"---\nname: decode-float\ndescription: Decode one float.\n---\n"
|
|
"# Decode one float\n"
|
|
)
|
|
}
|
|
)
|
|
)
|
|
|
|
|
|
def _packet() -> EvidencePacket:
|
|
return EvidencePacket(
|
|
packet_id="pkt_test",
|
|
trigger_job_id="trg_test",
|
|
packet_type="validator",
|
|
profile_name="analysis_current_task",
|
|
subprofile="validator:test",
|
|
manifest_watermark=1,
|
|
scope=EvidenceScope(task_id="task_test"),
|
|
selected_refs={
|
|
"tool_event": [
|
|
ResourceRef(
|
|
ref_id="tool:procedure",
|
|
ref_type="tool_event",
|
|
preview="decoder A returned -1.5",
|
|
metadata={
|
|
"status": "success",
|
|
"tool_use_id": "call_a",
|
|
"input_preview": "python decoder_a.py",
|
|
},
|
|
),
|
|
ResourceRef(
|
|
ref_id="tool:validation",
|
|
ref_type="tool_event",
|
|
preview="decoder B returned -1.5",
|
|
metadata={
|
|
"status": "success",
|
|
"tool_use_id": "call_b",
|
|
"input_preview": "objdump then decode manually",
|
|
},
|
|
),
|
|
]
|
|
},
|
|
expanded_snippets=[],
|
|
readable_paths=[],
|
|
instructions={},
|
|
budget=PacketBudget(max_chars=1000, used_chars=100),
|
|
redaction_status="ok",
|
|
build_status="ok",
|
|
missing_ref_types=[],
|
|
)
|