mirror of
https://github.com/HKUDS/OpenSpace.git
synced 2026-09-07 08:25:54 +00:00
466 lines
15 KiB
Python
466 lines
15 KiB
Python
"""Deterministic evidence linkers for quality signals."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Any, Iterable, Mapping, Sequence
|
|
|
|
from openspace.grounding.core.permissions.types import (
|
|
normalize_tool_name_for_rule,
|
|
parse_rule_value,
|
|
)
|
|
from openspace.skill_engine.evidence import ResourceRef
|
|
|
|
|
|
ACTIVE_SKILL_EVENT_TYPES: frozenset[str] = frozenset({"invoked", "applied"})
|
|
FAILED_TOOL_STATUSES: frozenset[str] = frozenset(
|
|
{
|
|
"blocked",
|
|
"cancelled",
|
|
"canceled",
|
|
"denied",
|
|
"error",
|
|
"failed",
|
|
"failure",
|
|
"permission_denied",
|
|
"rejected",
|
|
"timeout",
|
|
"timed_out",
|
|
}
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ToolUseKey:
|
|
session_id: str
|
|
task_id: str
|
|
agent_id: str
|
|
tool_use_id: str
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class SkillContextCandidate:
|
|
skill_id: str
|
|
skill_event: ResourceRef
|
|
skill_record: ResourceRef | None
|
|
skill_file: ResourceRef | None
|
|
link_type: str
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class EvidenceIndexes:
|
|
refs: tuple[ResourceRef, ...]
|
|
tool_events_by_use_id: dict[ToolUseKey, list[ResourceRef]]
|
|
tool_results_by_use_id: dict[ToolUseKey, list[ResourceRef]]
|
|
tool_incidents_by_tool_key: dict[str, list[ResourceRef]]
|
|
quality_records_by_tool_key: dict[str, list[ResourceRef]]
|
|
skill_events_by_skill_id: dict[str, list[ResourceRef]]
|
|
skill_files_by_skill_id: dict[str, list[ResourceRef]]
|
|
skill_records_by_skill_id: dict[str, list[ResourceRef]]
|
|
|
|
@property
|
|
def failed_tool_events(self) -> list[ResourceRef]:
|
|
refs: list[ResourceRef] = []
|
|
for bucket in self.tool_events_by_use_id.values():
|
|
refs.extend(ref for ref in bucket if is_failed_tool_event(ref))
|
|
return sorted(refs, key=ref_sort_key)
|
|
|
|
|
|
def build_evidence_indexes(refs: Iterable[ResourceRef]) -> EvidenceIndexes:
|
|
tool_events_by_use_id: dict[ToolUseKey, list[ResourceRef]] = {}
|
|
tool_results_by_use_id: dict[ToolUseKey, list[ResourceRef]] = {}
|
|
tool_incidents_by_tool_key: dict[str, list[ResourceRef]] = {}
|
|
quality_records_by_tool_key: dict[str, list[ResourceRef]] = {}
|
|
skill_events_by_skill_id: dict[str, list[ResourceRef]] = {}
|
|
skill_files_by_skill_id: dict[str, list[ResourceRef]] = {}
|
|
skill_records_by_skill_id: dict[str, list[ResourceRef]] = {}
|
|
ordered_refs = tuple(sorted(refs, key=ref_sort_key))
|
|
|
|
for ref in ordered_refs:
|
|
if ref.ref_type == "tool_event":
|
|
key = tool_use_key(ref)
|
|
if key.tool_use_id:
|
|
tool_events_by_use_id.setdefault(key, []).append(ref)
|
|
elif ref.ref_type == "tool_result":
|
|
key = tool_use_key(ref)
|
|
if key.tool_use_id:
|
|
tool_results_by_use_id.setdefault(key, []).append(ref)
|
|
elif ref.ref_type == "tool_incident":
|
|
tool_key = ref_tool_key(ref)
|
|
if tool_key:
|
|
tool_incidents_by_tool_key.setdefault(tool_key, []).append(ref)
|
|
elif ref.ref_type == "tool_quality_record":
|
|
tool_key = ref_tool_key(ref)
|
|
if tool_key:
|
|
quality_records_by_tool_key.setdefault(tool_key, []).append(ref)
|
|
elif ref.ref_type == "skill_event":
|
|
skill_id = ref_skill_id(ref)
|
|
if skill_id:
|
|
skill_events_by_skill_id.setdefault(skill_id, []).append(ref)
|
|
elif ref.ref_type == "skill_file":
|
|
skill_id = ref_skill_id(ref)
|
|
if skill_id:
|
|
skill_files_by_skill_id.setdefault(skill_id, []).append(ref)
|
|
elif ref.ref_type == "skill_record":
|
|
skill_id = ref_skill_id(ref)
|
|
if skill_id:
|
|
skill_records_by_skill_id.setdefault(skill_id, []).append(ref)
|
|
|
|
return EvidenceIndexes(
|
|
refs=ordered_refs,
|
|
tool_events_by_use_id=tool_events_by_use_id,
|
|
tool_results_by_use_id=tool_results_by_use_id,
|
|
tool_incidents_by_tool_key=tool_incidents_by_tool_key,
|
|
quality_records_by_tool_key=quality_records_by_tool_key,
|
|
skill_events_by_skill_id=skill_events_by_skill_id,
|
|
skill_files_by_skill_id=skill_files_by_skill_id,
|
|
skill_records_by_skill_id=skill_records_by_skill_id,
|
|
)
|
|
|
|
|
|
def resolve_skill_contexts(
|
|
tool_event: ResourceRef,
|
|
indexes: EvidenceIndexes,
|
|
) -> list[SkillContextCandidate]:
|
|
tool_key = ref_tool_key(tool_event)
|
|
if not tool_key:
|
|
return []
|
|
|
|
candidates: list[SkillContextCandidate] = []
|
|
seen: set[str] = set()
|
|
for skill_id, events in indexes.skill_events_by_skill_id.items():
|
|
active_events = [
|
|
ref
|
|
for ref in events
|
|
if skill_event_type(ref) in ACTIVE_SKILL_EVENT_TYPES
|
|
and same_run_window(tool_event, ref)
|
|
]
|
|
for skill_event in sorted(active_events, key=ref_sort_key):
|
|
link_type = _skill_link_type(tool_event, skill_event, indexes)
|
|
if not link_type or skill_id in seen:
|
|
continue
|
|
seen.add(skill_id)
|
|
candidates.append(
|
|
SkillContextCandidate(
|
|
skill_id=skill_id,
|
|
skill_event=skill_event,
|
|
skill_record=latest_ref(indexes.skill_records_by_skill_id.get(skill_id, [])),
|
|
skill_file=latest_ref(indexes.skill_files_by_skill_id.get(skill_id, [])),
|
|
link_type=link_type,
|
|
)
|
|
)
|
|
return sorted(candidates, key=lambda item: (item.skill_id, item.link_type))
|
|
|
|
|
|
def related_tool_results(
|
|
tool_event: ResourceRef,
|
|
indexes: EvidenceIndexes,
|
|
) -> list[ResourceRef]:
|
|
key = tool_use_key(tool_event)
|
|
refs = indexes.tool_results_by_use_id.get(key, [])
|
|
return sorted(refs, key=ref_sort_key)
|
|
|
|
|
|
def related_tool_incidents(
|
|
tool_event: ResourceRef,
|
|
indexes: EvidenceIndexes,
|
|
) -> list[ResourceRef]:
|
|
tool_key = ref_tool_key(tool_event)
|
|
if not tool_key:
|
|
return []
|
|
refs = [
|
|
ref
|
|
for ref in indexes.tool_incidents_by_tool_key.get(tool_key, [])
|
|
if same_run_window(tool_event, ref)
|
|
and same_tool_attempt(tool_event, ref)
|
|
]
|
|
return sorted(refs, key=ref_sort_key)
|
|
|
|
|
|
def has_representative_tool_evidence(
|
|
tool_key: str,
|
|
indexes: EvidenceIndexes,
|
|
) -> bool:
|
|
if indexes.tool_incidents_by_tool_key.get(tool_key):
|
|
return True
|
|
return any(
|
|
ref_tool_key(ref) == tool_key
|
|
for bucket in indexes.tool_events_by_use_id.values()
|
|
for ref in bucket
|
|
)
|
|
|
|
|
|
def is_failed_tool_event(ref: ResourceRef) -> bool:
|
|
status = str(ref.metadata.get("status") or "").strip().lower()
|
|
if ref.ref_type != "tool_event":
|
|
return False
|
|
if status in FAILED_TOOL_STATUSES:
|
|
return True
|
|
permission_status = str(ref.metadata.get("permission_status") or "").strip().lower()
|
|
return permission_status in {"blocked", "denied", "permission_denied"}
|
|
|
|
|
|
def ref_tool_key(ref: ResourceRef) -> str:
|
|
return _first_text(ref.metadata, "tool_key", "tool_name")
|
|
|
|
|
|
def ref_skill_id(ref: ResourceRef) -> str:
|
|
return _first_text(
|
|
ref.metadata,
|
|
"skill_id",
|
|
"target_skill_id",
|
|
"affected_skill_id",
|
|
)
|
|
|
|
|
|
def skill_event_type(ref: ResourceRef) -> str:
|
|
return _first_text(ref.metadata, "event_type", "lifecycle_event").lower()
|
|
|
|
|
|
def tool_use_key(ref: ResourceRef) -> ToolUseKey:
|
|
return ToolUseKey(
|
|
session_id=str(ref.session_id or ""),
|
|
task_id=str(ref.task_id or ref.metadata.get("task_id") or ""),
|
|
agent_id=str(ref.agent_id or ref.metadata.get("agent_id") or ""),
|
|
tool_use_id=_first_text(
|
|
ref.metadata,
|
|
"tool_use_id",
|
|
"tool_call_id",
|
|
"call_id",
|
|
),
|
|
)
|
|
|
|
|
|
def same_run_window(left: ResourceRef, right: ResourceRef) -> bool:
|
|
for left_value, right_value in (
|
|
(left.session_id, right.session_id),
|
|
(
|
|
left.task_id or left.metadata.get("task_id"),
|
|
right.task_id or right.metadata.get("task_id"),
|
|
),
|
|
):
|
|
if left_value and not right_value:
|
|
return False
|
|
if left_value and right_value and str(left_value) != str(right_value):
|
|
return False
|
|
if left.agent_id and right.agent_id and str(left.agent_id) != str(right.agent_id):
|
|
return False
|
|
return True
|
|
|
|
|
|
def same_tool_attempt(tool_event: ResourceRef, ref: ResourceRef) -> bool:
|
|
event_key = tool_use_key(tool_event)
|
|
ref_key = tool_use_key(ref)
|
|
if event_key.tool_use_id:
|
|
return (
|
|
ref_key.tool_use_id == event_key.tool_use_id
|
|
or tool_event.ref_id in set(ref.raw_backrefs or [])
|
|
)
|
|
return tool_event.ref_id in set(ref.raw_backrefs or [])
|
|
|
|
|
|
def latest_ref(refs: Sequence[ResourceRef]) -> ResourceRef | None:
|
|
if not refs:
|
|
return None
|
|
return sorted(refs, key=ref_sort_key)[-1]
|
|
|
|
|
|
def ref_sort_key(ref: ResourceRef) -> tuple[int, str]:
|
|
return (int(ref.last_seen_watermark or ref.first_seen_watermark or 0), ref.ref_id)
|
|
|
|
|
|
def metadata_values(metadata: Mapping[str, Any], *keys: str) -> set[str]:
|
|
values: set[str] = set()
|
|
for key in keys:
|
|
value = metadata.get(key)
|
|
values.update(_string_values(value))
|
|
nested = metadata.get("metadata")
|
|
if isinstance(nested, Mapping):
|
|
for key in keys:
|
|
values.update(_string_values(nested.get(key)))
|
|
return values
|
|
|
|
|
|
def _skill_link_type(
|
|
tool_event: ResourceRef,
|
|
skill_event: ResourceRef,
|
|
indexes: EvidenceIndexes,
|
|
) -> str:
|
|
skill_id = ref_skill_id(skill_event)
|
|
tool_key = ref_tool_key(tool_event)
|
|
tool_use_id = tool_use_key(tool_event).tool_use_id
|
|
|
|
record = latest_ref(indexes.skill_records_by_skill_id.get(skill_id, []))
|
|
if record is not None:
|
|
dependency_values = metadata_values(
|
|
record.metadata,
|
|
"tool_dependencies",
|
|
"critical_tools",
|
|
"required_tools",
|
|
"allowed_tools",
|
|
"allowed-tools",
|
|
"allowedTools",
|
|
)
|
|
if _tool_matches_any_dependency(tool_key, dependency_values):
|
|
return "skill_dependency"
|
|
|
|
explicit_skill_ids = metadata_values(
|
|
tool_event.metadata,
|
|
"skill_id",
|
|
"skill_ids",
|
|
"active_skill_id",
|
|
"active_skill_ids",
|
|
"invoked_skill_id",
|
|
"invoked_skill_ids",
|
|
)
|
|
if skill_id in explicit_skill_ids or skill_id in set(tool_event.raw_backrefs):
|
|
return "explicit_invocation_scope"
|
|
|
|
tool_backrefs = set(tool_event.raw_backrefs)
|
|
tool_backrefs.update(
|
|
metadata_values(
|
|
tool_event.metadata,
|
|
"skill_event_ref_id",
|
|
"skill_event_ref_ids",
|
|
"skill_event_ref",
|
|
"skill_event_refs",
|
|
"raw_backrefs",
|
|
)
|
|
)
|
|
if skill_event.ref_id in tool_backrefs:
|
|
return "explicit_invocation_scope"
|
|
|
|
if _scope_ids(tool_event.metadata).intersection(_scope_ids(skill_event.metadata)):
|
|
return "explicit_invocation_scope"
|
|
|
|
skill_tool_use_ids = metadata_values(
|
|
skill_event.metadata,
|
|
"tool_use_id",
|
|
"tool_use_ids",
|
|
"tool_call_id",
|
|
"tool_call_ids",
|
|
)
|
|
if tool_use_id and tool_use_id in skill_tool_use_ids:
|
|
return "skill_event_tool_use"
|
|
|
|
skill_backrefs = set(skill_event.raw_backrefs)
|
|
skill_backrefs.update(
|
|
metadata_values(
|
|
skill_event.metadata,
|
|
"tool_event_ref",
|
|
"tool_event_refs",
|
|
"tool_ref",
|
|
"tool_refs",
|
|
"raw_backrefs",
|
|
)
|
|
)
|
|
if tool_event.ref_id in skill_backrefs or (tool_use_id and tool_use_id in skill_backrefs):
|
|
return "skill_event_backref"
|
|
|
|
return ""
|
|
|
|
|
|
def _tool_matches_any_dependency(tool_key: str, dependencies: Iterable[str]) -> bool:
|
|
tool_variants = _tool_identity_variants(tool_key)
|
|
if not tool_variants:
|
|
return False
|
|
for dependency in dependencies:
|
|
if tool_variants.intersection(_tool_identity_variants(dependency)):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _tool_identity_variants(value: Any) -> set[str]:
|
|
raw = str(value or "").strip()
|
|
if not raw:
|
|
return set()
|
|
|
|
candidates: set[str] = {raw}
|
|
try:
|
|
parsed = parse_rule_value(raw).tool_name
|
|
if parsed:
|
|
candidates.add(parsed)
|
|
except Exception:
|
|
head = raw.split("(", 1)[0].strip()
|
|
if head:
|
|
candidates.add(normalize_tool_name_for_rule(head))
|
|
|
|
variants: set[str] = set()
|
|
for candidate in list(candidates):
|
|
candidate = str(candidate or "").strip()
|
|
if not candidate:
|
|
continue
|
|
variants.add(_identity_token(candidate))
|
|
_add_colon_variants(candidate, variants)
|
|
_add_mcp_separator_variants(candidate, variants)
|
|
return {item for item in variants if item}
|
|
|
|
|
|
def _add_colon_variants(value: str, variants: set[str]) -> None:
|
|
parts = [part.strip() for part in value.split(":") if part.strip()]
|
|
if len(parts) == 3:
|
|
backend, server, tool_name = parts
|
|
variants.add(_identity_token(f"{backend}:{server}:{tool_name}"))
|
|
variants.add(_identity_token(f"{backend}:{tool_name}"))
|
|
variants.add(_identity_token(tool_name))
|
|
if backend.lower() == "mcp":
|
|
variants.add(_identity_token(f"mcp__{server}__{tool_name}"))
|
|
variants.add(_identity_token(f"{server}__{tool_name}"))
|
|
variants.add(_identity_token(f"{server}:{tool_name}"))
|
|
elif len(parts) == 2:
|
|
first, second = parts
|
|
variants.add(_identity_token(f"{first}:{second}"))
|
|
variants.add(_identity_token(second))
|
|
|
|
|
|
def _add_mcp_separator_variants(value: str, variants: set[str]) -> None:
|
|
parts = [part.strip() for part in value.split("__") if part.strip()]
|
|
if len(parts) >= 3 and parts[0].lower() == "mcp":
|
|
server = parts[1]
|
|
tool_name = "__".join(parts[2:])
|
|
variants.add(_identity_token(f"mcp:{server}:{tool_name}"))
|
|
variants.add(_identity_token(f"{server}:{tool_name}"))
|
|
variants.add(_identity_token(f"{server}__{tool_name}"))
|
|
variants.add(_identity_token(tool_name))
|
|
elif len(parts) >= 2:
|
|
tool_name = parts[-1]
|
|
server = parts[-2]
|
|
variants.add(_identity_token(f"{server}:{tool_name}"))
|
|
variants.add(_identity_token(tool_name))
|
|
|
|
|
|
def _identity_token(value: str) -> str:
|
|
return " ".join(str(value or "").strip().lower().split())
|
|
|
|
|
|
def _scope_ids(metadata: Mapping[str, Any]) -> set[str]:
|
|
return metadata_values(
|
|
metadata,
|
|
"skill_invocation_scope_id",
|
|
"invocation_scope_id",
|
|
"skill_scope_id",
|
|
"scope_id",
|
|
)
|
|
|
|
|
|
def _first_text(metadata: Mapping[str, Any], *keys: str) -> str:
|
|
for key in keys:
|
|
value = metadata.get(key)
|
|
if value is not None and str(value) != "":
|
|
return str(value)
|
|
return ""
|
|
|
|
|
|
def _string_values(value: Any) -> set[str]:
|
|
if value is None:
|
|
return set()
|
|
if isinstance(value, str):
|
|
return {value} if value else set()
|
|
if isinstance(value, Mapping):
|
|
result: set[str] = set()
|
|
for item in value.values():
|
|
result.update(_string_values(item))
|
|
return result
|
|
if isinstance(value, (list, tuple, set)):
|
|
return {str(item) for item in value if str(item)}
|
|
return {str(value)} if str(value) else set()
|