mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-15 23:31:05 +00:00
* update * refactor(memory): remove unnecessary type check and update error logging * refactor(core): standardize logger import and update agentscope dependency * fix(memory): disable console output and add logging for summarizer component * feat(core): replace OpenAI token counter with custom ReMe token counter - Replace OpenAITokenCounter with ReMeTokenCounter implementation - Add support for HuggingFace mirror and configurable tokenizer - Register ReMeTokenCounter as default token counter in registry - Update config to use hf backend with Qwen2.5-7B-Instruct model refactor(memory): convert token counting methods to async in message handlers - Change count_str_token, stat_message, count_msgs_token to async methods - Update format_msgs_to_str and context_check to use async token counting - Modify _format_tool_result_output to support async token counting - Adjust all dependent methods to await async token counting calls feat(memory): add dialog persistence to in-memory storage - Implement _append_messages_to_dialog for saving messages to JSONL files - Add dialog_path parameter to ReMeInMemoryMemory constructor - Persist messages to daily JSONL files based on timestamp grouping - Update mark_messages_compressed to save and remove compressed messages - Modify clear_content to persist all messages before clearing memory refactor(ops): update token counter type hints and initialization - Change BaseOp to use HuggingFaceTokenCounter instead of TokenCounterBase - Update type annotations for as_token_counter property and parameters - Remove direct token counter injection from Compactor and ContextChecker - Pass as_token_counter parameter through service context mechanism style(logging): improve error logging with exception details - Replace logger.error with logger.exception in browser control tool - Change logger.error to logger.exception in memory get tool error handling - Add proper exception logging with stack trace information chore(config): add token counter configuration to light YAML - Add as_token_counters section with default hf backend configuration - Configure Qwen/Qwen2.5-7B-Instruct model with mirror support enabled - Set up pretrained_model_name_or_path and use_mirror parameters test(context): update context check tests to async implementation - Convert verify_context_check_invariants to async function - Update context check test methods to use async calls - Change stat_message calls to await async implementation - Modify test_empty_messages and test_below_threshold_returns_all to async * feat(core): implement context checking and memory management features * refactor(core): replace direct loguru import with logger utility function * refactor(reme): remove RuntimeContext dependency and simplify context checking * feat(docs): add raw conversation persistence to ReMe framework
248 lines
6.7 KiB
Python
248 lines
6.7 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""Build role snapshot + refs from Playwright aria_snapshot."""
|
|
|
|
import re
|
|
from typing import Any
|
|
|
|
INTERACTIVE_ROLES = frozenset(
|
|
{
|
|
"button",
|
|
"link",
|
|
"textbox",
|
|
"checkbox",
|
|
"radio",
|
|
"combobox",
|
|
"listbox",
|
|
"menuitem",
|
|
"menuitemcheckbox",
|
|
"menuitemradio",
|
|
"option",
|
|
"searchbox",
|
|
"slider",
|
|
"spinbutton",
|
|
"switch",
|
|
"tab",
|
|
"treeitem",
|
|
},
|
|
)
|
|
|
|
CONTENT_ROLES = frozenset(
|
|
{
|
|
"heading",
|
|
"cell",
|
|
"gridcell",
|
|
"columnheader",
|
|
"rowheader",
|
|
"listitem",
|
|
"article",
|
|
"region",
|
|
"main",
|
|
"navigation",
|
|
},
|
|
)
|
|
|
|
STRUCTURAL_ROLES = frozenset(
|
|
{
|
|
"generic",
|
|
"group",
|
|
"list",
|
|
"table",
|
|
"row",
|
|
"rowgroup",
|
|
"grid",
|
|
"treegrid",
|
|
"menu",
|
|
"menubar",
|
|
"toolbar",
|
|
"tablist",
|
|
"tree",
|
|
"directory",
|
|
"document",
|
|
"application",
|
|
"presentation",
|
|
"none",
|
|
},
|
|
)
|
|
|
|
|
|
def _get_indent_level(line: str) -> int:
|
|
m = re.match(r"^(\s*)", line)
|
|
return int(len(m.group(1)) / 2) if m else 0
|
|
|
|
|
|
def _create_tracker() -> dict[str, Any]:
|
|
counts: dict[str, int] = {}
|
|
refs_by_key: dict[str, list[str]] = {}
|
|
|
|
def get_key(role: str, name: str | None) -> str:
|
|
return f"{role}:{name or ''}"
|
|
|
|
def get_next_index(role: str, name: str | None) -> int:
|
|
key = get_key(role, name)
|
|
current = counts.get(key, 0)
|
|
counts[key] = current + 1
|
|
return current
|
|
|
|
def track_ref(role: str, name: str | None, ref: str) -> None:
|
|
key = get_key(role, name)
|
|
refs_by_key.setdefault(key, []).append(ref)
|
|
|
|
def get_duplicate_keys() -> set[str]:
|
|
return {k for k, refs in refs_by_key.items() if len(refs) > 1}
|
|
|
|
return {
|
|
"get_next_index": get_next_index,
|
|
"track_ref": track_ref,
|
|
"get_duplicate_keys": get_duplicate_keys,
|
|
"get_key": get_key,
|
|
}
|
|
|
|
|
|
def _remove_nth_from_non_duplicates(
|
|
refs: dict[str, dict],
|
|
tracker: dict,
|
|
) -> None:
|
|
dup_keys = tracker["get_duplicate_keys"]()
|
|
for _, data in list(refs.items()):
|
|
key = tracker["get_key"](data["role"], data.get("name"))
|
|
if key not in dup_keys and "nth" in data:
|
|
del data["nth"]
|
|
|
|
|
|
def _compact_tree(tree: str) -> str:
|
|
lines = tree.split("\n")
|
|
result = []
|
|
for i, line in enumerate(lines):
|
|
if "[ref=" in line:
|
|
result.append(line)
|
|
continue
|
|
if ":" in line and not line.rstrip().endswith(":"):
|
|
result.append(line)
|
|
continue
|
|
current_indent = _get_indent_level(line)
|
|
has_relevant = False
|
|
for j in range(i + 1, len(lines)):
|
|
if _get_indent_level(lines[j]) <= current_indent:
|
|
break
|
|
if "[ref=" in lines[j]:
|
|
has_relevant = True
|
|
break
|
|
if has_relevant:
|
|
result.append(line)
|
|
return "\n".join(result)
|
|
|
|
|
|
def _process_line( # pylint: disable=too-many-return-statements
|
|
line: str,
|
|
refs: dict[str, dict],
|
|
options: dict[str, Any],
|
|
tracker: dict,
|
|
next_ref: Any,
|
|
) -> str | None:
|
|
depth = _get_indent_level(line)
|
|
max_depth_val = options.get("maxDepth")
|
|
if max_depth_val is not None and depth > max_depth_val:
|
|
return None
|
|
|
|
m = re.match(r'^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$', line)
|
|
if not m:
|
|
return None if options.get("interactive") else line
|
|
|
|
prefix, role_raw, name, suffix = m.groups()
|
|
if role_raw.startswith("/"):
|
|
return None if options.get("interactive") else line
|
|
|
|
role = role_raw.lower()
|
|
is_interactive = role in INTERACTIVE_ROLES
|
|
is_content = role in CONTENT_ROLES
|
|
is_structural = role in STRUCTURAL_ROLES
|
|
|
|
if options.get("interactive") and not is_interactive:
|
|
return None
|
|
if options.get("compact") and is_structural and not name:
|
|
return None
|
|
|
|
should_have_ref = is_interactive or (is_content and name)
|
|
if not should_have_ref:
|
|
return line
|
|
|
|
ref = next_ref()
|
|
nth = tracker["get_next_index"](role, name)
|
|
tracker["track_ref"](role, name, ref)
|
|
refs[ref] = {"role": role, "name": name, "nth": nth}
|
|
|
|
enhanced = f"{prefix}{role_raw}"
|
|
if name:
|
|
enhanced += f' "{name}"'
|
|
enhanced += f" [ref={ref}]"
|
|
if nth is not None and nth > 0:
|
|
enhanced += f" [nth={nth}]"
|
|
if suffix:
|
|
enhanced += suffix
|
|
return enhanced
|
|
|
|
|
|
def build_role_snapshot_from_aria(
|
|
aria_snapshot: str,
|
|
*,
|
|
interactive: bool = False,
|
|
compact: bool = False,
|
|
max_depth: int | None = None,
|
|
) -> tuple[str, dict[str, dict]]:
|
|
"""Build snapshot + refs from Playwright locator.aria_snapshot() output."""
|
|
options = {
|
|
"interactive": interactive,
|
|
"compact": compact,
|
|
"maxDepth": max_depth,
|
|
}
|
|
lines = aria_snapshot.split("\n")
|
|
refs: dict[str, dict] = {}
|
|
tracker = _create_tracker()
|
|
counter = [0]
|
|
|
|
def next_ref() -> str:
|
|
counter[0] += 1
|
|
return f"e{counter[0]}"
|
|
|
|
if options.get("interactive"):
|
|
result_lines = []
|
|
for line in lines:
|
|
depth = _get_indent_level(line)
|
|
max_d = options.get("maxDepth")
|
|
if max_d is not None and depth > max_d:
|
|
continue
|
|
m = re.match(r'^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$', line)
|
|
if not m:
|
|
continue
|
|
_, role_raw, name, suffix = m.groups()
|
|
if role_raw.startswith("/"):
|
|
continue
|
|
role = role_raw.lower()
|
|
if role not in INTERACTIVE_ROLES:
|
|
continue
|
|
ref = next_ref()
|
|
nth = tracker["get_next_index"](role, name)
|
|
tracker["track_ref"](role, name, ref)
|
|
refs[ref] = {"role": role, "name": name, "nth": nth}
|
|
enhanced = f"- {role_raw}"
|
|
if name:
|
|
enhanced += f' "{name}"'
|
|
enhanced += f" [ref={ref}]"
|
|
if nth is not None and nth > 0:
|
|
enhanced += f" [nth={nth}]"
|
|
if "[" in suffix:
|
|
enhanced += suffix
|
|
result_lines.append(enhanced)
|
|
_remove_nth_from_non_duplicates(refs, tracker)
|
|
snapshot = "\n".join(result_lines) or "(no interactive elements)"
|
|
return snapshot, refs
|
|
|
|
result_lines = []
|
|
for line in lines:
|
|
processed = _process_line(line, refs, options, tracker, next_ref)
|
|
if processed is not None:
|
|
result_lines.append(processed)
|
|
_remove_nth_from_non_duplicates(refs, tracker)
|
|
tree = "\n".join(result_lines) or "(empty)"
|
|
snapshot = _compact_tree(tree) if options.get("compact") else tree
|
|
return snapshot, refs
|