From 00018c0ff8d08ae7dd4781ef45d778ea79e5dd27 Mon Sep 17 00:00:00 2001 From: huangsen Date: Tue, 21 Apr 2026 16:39:33 +0800 Subject: [PATCH] ``` feat: add file parser support and search filtering - Add file_parser component with default implementation - Introduce SearchFilter schema for path and tag filtering - Implement filter functionality in BaseFileStore and LocalFileStore - Update file watcher to use parser-based filtering instead of suffix filters - Register new FILE_PARSER component enum - Add test_data directory to gitignore refactor: improve component imports and initialization - Fix relative imports in application.py - Add file_parser import to component init - Initialize registry dict when component type doesn't exist - Remove circular import in HttpService by using string annotation - Update config yaml to use proper component names refactor: enhance file watcher architecture - Replace MdFileWatcher with more flexible FullFileWatcher and LightFileWatcher - Remove suffix-based filtering in favor of parser-based approach - Update BaseFileWatcher to resolve parsers from app context - Remove unused watch_filter method refactor: update ReMe core functionality - Remove memory_path creation - Simplify dream and proactive methods to return empty strings - Update config defaults for HTTP service and component backends docs: update component configuration in paw.yaml - Change service backend from cmd to http - Rename components to use correct singular forms - Add default file parser and file watcher configurations - Set up local file store with default settings ``` --- .gitignore | 3 +- reme2/application.py | 10 +- reme2/component/__init__.py | 4 +- reme2/component/as_token_counter/__init__.py | 2 +- reme2/component/base_step.py | 14 +- reme2/component/component_registry.py | 2 + reme2/component/file_parser/__init__.py | 11 + .../component/file_parser/base_file_parser.py | 41 ++++ .../file_parser/default_file_parser.py | 64 ++++++ reme2/component/file_parser/md_file_parser.py | 55 +++++ reme2/component/file_store/base_file_store.py | 32 ++- .../component/file_store/local_file_store.py | 48 +++-- reme2/component/file_watcher/__init__.py | 6 +- .../file_watcher/base_file_watcher.py | 65 +++--- .../file_watcher/full_file_watcher.py | 48 +++++ .../file_watcher/light_file_watcher.py | 53 +++++ .../component/file_watcher/md_file_watcher.py | 86 -------- reme2/component/service/base_service.py | 1 + reme2/component/service/http_service.py | 7 +- reme2/config/{obsidian.yaml => obsidian.md} | 6 +- reme2/config/paw.yaml | 27 ++- reme2/enumeration/component_enum.py | 2 + reme2/file_based/file_io.py | 45 ++-- reme2/file_based/file_utils.py | 41 ++-- reme2/file_based/memory_search.py | 20 +- reme2/file_based/summarizer.py | 30 +-- reme2/reme.py | 56 ++--- reme2/schema/__init__.py | 2 + reme2/schema/search_filter.py | 36 ++++ tests/test_reme2.py | 201 ++++++++++++++++++ 30 files changed, 749 insertions(+), 269 deletions(-) create mode 100644 reme2/component/file_parser/__init__.py create mode 100644 reme2/component/file_parser/base_file_parser.py create mode 100644 reme2/component/file_parser/default_file_parser.py create mode 100644 reme2/component/file_parser/md_file_parser.py create mode 100644 reme2/component/file_watcher/full_file_watcher.py create mode 100644 reme2/component/file_watcher/light_file_watcher.py delete mode 100644 reme2/component/file_watcher/md_file_watcher.py rename reme2/config/{obsidian.yaml => obsidian.md} (97%) create mode 100644 reme2/schema/search_filter.py create mode 100644 tests/test_reme2.py diff --git a/.gitignore b/.gitignore index b20570b5..fa4adba4 100644 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,5 @@ meta_memory/* **/data/*.json *.db memories/* -.reme/* \ No newline at end of file +.reme/* +/test_data \ No newline at end of file diff --git a/reme2/application.py b/reme2/application.py index c66401dd..3672be46 100644 --- a/reme2/application.py +++ b/reme2/application.py @@ -4,7 +4,7 @@ import asyncio from pathlib import Path from typing import AsyncGenerator -from enumeration import ComponentEnum +from .enumeration import ComponentEnum from .component import BaseComponent, ApplicationContext from .schema import Response, StreamChunk from .utils import execute_stream_task, print_logo, get_logger @@ -122,10 +122,10 @@ class Application(BaseComponent): stream_queue = asyncio.Queue() task = asyncio.create_task(job(stream_queue=stream_queue, app_context=self.context, **kwargs)) async for chunk in execute_stream_task( - stream_queue=stream_queue, - task=task, - task_name=name, - output_format="chunk", + stream_queue=stream_queue, + task=task, + task_name=name, + output_format="chunk", ): assert isinstance(chunk, StreamChunk) yield chunk diff --git a/reme2/component/__init__.py b/reme2/component/__init__.py index 5998242c..1e6c6bfb 100644 --- a/reme2/component/__init__.py +++ b/reme2/component/__init__.py @@ -9,9 +9,9 @@ from .runtime_context import RuntimeContext from . import as_llm from . import as_llm_formatter -from . import as_token_counter from . import client from . import embedding +from . import file_parser from . import file_store from . import file_watcher from . import job @@ -28,9 +28,9 @@ __all__ = [ # base components "as_llm", "as_llm_formatter", - "client", "embedding", + "file_parser", "file_store", "file_watcher", "job", diff --git a/reme2/component/as_token_counter/__init__.py b/reme2/component/as_token_counter/__init__.py index 1678b7f5..1bb4389e 100644 --- a/reme2/component/as_token_counter/__init__.py +++ b/reme2/component/as_token_counter/__init__.py @@ -41,4 +41,4 @@ class EstimatedAsTokenCounter(BaseAsTokenCounter): __all__ = [ "BaseAsTokenCounter", "EstimatedAsTokenCounter", -] \ No newline at end of file +] diff --git a/reme2/component/base_step.py b/reme2/component/base_step.py index e9dc9dd1..7892334c 100644 --- a/reme2/component/base_step.py +++ b/reme2/component/base_step.py @@ -31,13 +31,13 @@ class BaseStep(BaseComponent): return instance def __init__( - self, - name: str = "", - language: str = "", - prompt_dict: dict[str, str] | None = None, - input_mapping: dict[str, str] | None = None, - output_mapping: dict[str, str] | None = None, - **kwargs, + self, + name: str = "", + language: str = "", + prompt_dict: dict[str, str] | None = None, + input_mapping: dict[str, str] | None = None, + output_mapping: dict[str, str] | None = None, + **kwargs, ): """Initialize step configurations.""" super().__init__(**kwargs) diff --git a/reme2/component/component_registry.py b/reme2/component/component_registry.py index 007a5ebe..e5d5c52a 100644 --- a/reme2/component/component_registry.py +++ b/reme2/component/component_registry.py @@ -31,6 +31,8 @@ class ComponentRegistry: raise ValueError("Component name cannot be empty") component_type = cls.component_type + if component_type not in self._registry: + self._registry[component_type] = {} if name in self._registry[component_type]: self.logger.warning(f"Component '{name}' already registered for {component_type}, overwriting") diff --git a/reme2/component/file_parser/__init__.py b/reme2/component/file_parser/__init__.py new file mode 100644 index 00000000..de6491c7 --- /dev/null +++ b/reme2/component/file_parser/__init__.py @@ -0,0 +1,11 @@ +"""File parser implementations for different file formats.""" + +from .base_file_parser import BaseFileParser +from .default_file_parser import DefaultFileParser +from .md_file_parser import MdFileParser + +__all__ = [ + "BaseFileParser", + "DefaultFileParser", + "MdFileParser", +] diff --git a/reme2/component/file_parser/base_file_parser.py b/reme2/component/file_parser/base_file_parser.py new file mode 100644 index 00000000..a7e9d793 --- /dev/null +++ b/reme2/component/file_parser/base_file_parser.py @@ -0,0 +1,41 @@ +"""Abstract base class for file parsers.""" + +from abc import abstractmethod + +from ..base_component import BaseComponent +from ...enumeration import ComponentEnum +from ...schema import FileChunk, FileMetadata + + +class BaseFileParser(BaseComponent): + """Abstract base class for file format parsers. + + Each parser declares which file suffixes it handles and implements + the parse method to produce FileMetadata and FileChunks. + """ + + component_type = ComponentEnum.FILE_PARSER + + suffixes: list[str] = [] + + def __init__(self, chunk_tokens: int = 400, chunk_overlap: int = 80, **kwargs): + super().__init__(**kwargs) + self.chunk_tokens = chunk_tokens + self.chunk_overlap = chunk_overlap + + async def _start(self, app_context=None): + pass + + async def _close(self): + pass + + @abstractmethod + async def parse(self, path: str) -> tuple[FileMetadata, list[FileChunk]]: + """Parse a file into metadata and chunks. + + Args: + path: Absolute path to the file. + + Returns: + Tuple of (FileMetadata, list of FileChunks). + """ diff --git a/reme2/component/file_parser/default_file_parser.py b/reme2/component/file_parser/default_file_parser.py new file mode 100644 index 00000000..53e82629 --- /dev/null +++ b/reme2/component/file_parser/default_file_parser.py @@ -0,0 +1,64 @@ +"""Default file parser for unknown file types.""" + +import asyncio +import hashlib +from pathlib import Path + +from .base_file_parser import BaseFileParser +from ..component_registry import R +from ...schema import FileChunk, FileMetadata +from ...utils import hash_text, chunk_markdown + + +@R.register("default") +class DefaultFileParser(BaseFileParser): + """Fallback parser for unknown file types. + + Attempts to read as text and chunk. If the file is binary, + stores metadata only with no chunks. + """ + + suffixes = [] + + def __init__(self, encoding: str = "utf-8", **kwargs): + super().__init__(**kwargs) + self.encoding = encoding + + async def parse(self, path: str) -> tuple[FileMetadata, list[FileChunk]]: + file_path = Path(path) + + def _read_file(): + stat = file_path.stat() + raw = file_path.read_bytes() + try: + content = raw.decode(self.encoding) + content_hash = hash_text(content) + return stat, content_hash, content + except (UnicodeDecodeError, ValueError): + binary_hash = hashlib.sha256(raw).hexdigest() + return stat, binary_hash, None + + stat, file_hash, content = await asyncio.to_thread(_read_file) + + file_meta = FileMetadata( + hash=file_hash, + mtime_ms=stat.st_mtime * 1000, + size=stat.st_size, + path=str(file_path.absolute()), + content=content, + ) + + chunks: list[FileChunk] = [] + if content: + chunks = ( + chunk_markdown( + content, + file_meta.path, + self.chunk_tokens, + self.chunk_overlap, + ) + or [] + ) + + file_meta.chunk_count = len(chunks) + return file_meta, chunks diff --git a/reme2/component/file_parser/md_file_parser.py b/reme2/component/file_parser/md_file_parser.py new file mode 100644 index 00000000..688a8c04 --- /dev/null +++ b/reme2/component/file_parser/md_file_parser.py @@ -0,0 +1,55 @@ +"""Markdown file parser.""" + +import asyncio +from pathlib import Path + +import frontmatter + +from .base_file_parser import BaseFileParser +from ..component_registry import R +from ...schema import FileChunk, FileMetadata +from ...utils import hash_text, chunk_markdown + + +@R.register("md") +class MdFileParser(BaseFileParser): + """Parser for Markdown files with YAML frontmatter support.""" + + suffixes = [".md", ".markdown"] + + def __init__(self, encoding: str = "utf-8", **kwargs): + super().__init__(**kwargs) + self.encoding = encoding + + async def parse(self, path: str) -> tuple[FileMetadata, list[FileChunk]]: + file_path = Path(path) + + def _read_and_parse(): + raw = file_path.read_text(encoding=self.encoding) + post = frontmatter.loads(raw) + stat = file_path.stat() + return stat, dict(post.metadata), post.content + + stat, metadata, content = await asyncio.to_thread(_read_and_parse) + + file_meta = FileMetadata( + hash=hash_text(content), + mtime_ms=stat.st_mtime * 1000, + size=stat.st_size, + path=str(file_path.absolute()), + content=content, + metadata=metadata, + ) + + chunks = ( + chunk_markdown( + content, + file_meta.path, + self.chunk_tokens, + self.chunk_overlap, + ) + or [] + ) + + file_meta.chunk_count = len(chunks) + return file_meta, chunks diff --git a/reme2/component/file_store/base_file_store.py b/reme2/component/file_store/base_file_store.py index 88db57a2..b1a84cfc 100644 --- a/reme2/component/file_store/base_file_store.py +++ b/reme2/component/file_store/base_file_store.py @@ -7,7 +7,7 @@ from pathlib import Path from ..base_component import BaseComponent from ..embedding import BaseEmbeddingModel from ...enumeration import ComponentEnum -from ...schema import FileChunk, FileMetadata +from ...schema import FileChunk, FileMetadata, SearchFilter class BaseFileStore(BaseComponent): @@ -145,12 +145,36 @@ class BaseFileStore(BaseComponent): async def get_file_chunks(self, path: str) -> list[FileChunk]: """Get all chunks for a file.""" + def _apply_filter( + self, + chunks: list[FileChunk], + search_filter: SearchFilter | None, + file_metadata: dict[str, FileMetadata] | None = None, + ) -> list[FileChunk]: + """Apply search filter to a list of chunks. + + Args: + chunks: Candidate chunks to filter. + search_filter: Filter conditions. + file_metadata: File-level metadata lookup (path -> FileMetadata). + Used for tag filtering since tags are file-level, not chunk-level. + """ + if not search_filter or search_filter.is_empty(): + return chunks + fm = file_metadata or {} + return [c for c in chunks if search_filter.match(c.path, fm[c.path].metadata if c.path in fm else None)] + @abstractmethod - async def vector_search(self, query: str, limit: int) -> list[FileChunk]: + async def vector_search(self, query: str, limit: int, search_filter: SearchFilter | None = None) -> list[FileChunk]: """Perform vector similarity search.""" @abstractmethod - async def keyword_search(self, query: str, limit: int) -> list[FileChunk]: + async def keyword_search( + self, + query: str, + limit: int, + search_filter: SearchFilter | None = None, + ) -> list[FileChunk]: """Perform full-text/keyword search.""" @abstractmethod @@ -160,6 +184,7 @@ class BaseFileStore(BaseComponent): limit: int, vector_weight: float = 0.7, candidate_multiplier: float = 3.0, + search_filter: SearchFilter | None = None, ) -> list[FileChunk]: """Perform hybrid search combining vector and keyword results. @@ -168,6 +193,7 @@ class BaseFileStore(BaseComponent): limit: Maximum number of results. vector_weight: Weight for vector scores (0.0-1.0). candidate_multiplier: Multiplier for candidate pool size. + search_filter: Optional filter for paths/tags. Returns: FileChunk list with score populated, sorted by relevance. diff --git a/reme2/component/file_store/local_file_store.py b/reme2/component/file_store/local_file_store.py index 283968cd..fa410006 100644 --- a/reme2/component/file_store/local_file_store.py +++ b/reme2/component/file_store/local_file_store.py @@ -7,7 +7,7 @@ import numpy as np from .base_file_store import BaseFileStore from ..component_registry import R -from ...schema import FileChunk, FileMetadata +from ...schema import FileChunk, FileMetadata, SearchFilter from ...utils import batch_cosine_similarity @@ -72,9 +72,7 @@ class LocalFileStore(BaseFileStore): async def _save_metadata(self) -> None: """Persist file metadata to JSON file with atomic write.""" - raw = { - path: meta.model_dump(exclude={"content", "metadata"}, mode="json") for path, meta in self._files.items() - } + raw = {path: meta.model_dump(exclude={"content"}, mode="json") for path, meta in self._files.items()} content = json.dumps(raw, indent=2, ensure_ascii=False) temp_path = self._metadata_file.with_suffix(".tmp") try: @@ -97,7 +95,7 @@ class LocalFileStore(BaseFileStore): f"LocalFileStore '{self.store_name}' ready: " f"{len(self._chunks)} chunks, metadata at {self._metadata_file}", ) - await super()._start() + await super()._start(app_context) async def _close(self) -> None: """Flush state to disk and clear memory.""" @@ -111,14 +109,12 @@ class LocalFileStore(BaseFileStore): async def upsert_file(self, file_meta: FileMetadata, chunks: list[FileChunk]) -> None: """Insert or update a file and its chunks.""" - if not chunks: - return - await self.delete_file(file_meta.path) - chunks = await self.get_chunk_embeddings(chunks) - for chunk in chunks: - self._chunks[chunk.id] = chunk + if chunks: + chunks = await self.get_chunk_embeddings(chunks) + for chunk in chunks: + self._chunks[chunk.id] = chunk self._files[file_meta.path] = FileMetadata( hash=file_meta.hash, @@ -126,6 +122,7 @@ class LocalFileStore(BaseFileStore): size=file_meta.size, path=file_meta.path, chunk_count=len(chunks), + metadata=file_meta.metadata, ) async def delete_file(self, path: str) -> None: @@ -170,6 +167,7 @@ class LocalFileStore(BaseFileStore): size=file_meta.size, path=file_meta.path, chunk_count=file_meta.chunk_count, + metadata=file_meta.metadata, ) async def get_file_chunks(self, path: str) -> list[FileChunk]: @@ -180,7 +178,7 @@ class LocalFileStore(BaseFileStore): # -- Search ------------------------------------------------------------- - async def vector_search(self, query: str, limit: int) -> list[FileChunk]: + async def vector_search(self, query: str, limit: int, search_filter: SearchFilter | None = None) -> list[FileChunk]: """Cosine-similarity vector search over in-memory embeddings.""" if not self.vector_enabled or not query: return [] @@ -189,7 +187,11 @@ class LocalFileStore(BaseFileStore): if not query_embedding: return [] - candidates = [c for c in self._chunks.values() if c.embedding] + candidates = self._apply_filter( + [c for c in self._chunks.values() if c.embedding], + search_filter, + self._files, + ) if not candidates: return [] @@ -226,7 +228,12 @@ class LocalFileStore(BaseFileStore): results.sort(key=lambda r: r.score, reverse=True) return results[:limit] - async def keyword_search(self, query: str, limit: int) -> list[FileChunk]: + async def keyword_search( + self, + query: str, + limit: int, + search_filter: SearchFilter | None = None, + ) -> list[FileChunk]: """Keyword search via substring matching.""" if not self.fts_enabled or not query: return [] @@ -239,8 +246,10 @@ class LocalFileStore(BaseFileStore): words_lower = [w.lower() for w in words] n_words = len(words) + filtered_chunks = self._apply_filter(list(self._chunks.values()), search_filter, self._files) + results = [] - for chunk in self._chunks.values(): + for chunk in filtered_chunks: text_lower = chunk.text.lower() match_count = sum(1 for w in words_lower if w in text_lower) if match_count == 0: @@ -271,6 +280,7 @@ class LocalFileStore(BaseFileStore): limit: int, vector_weight: float = 0.7, candidate_multiplier: float = 3.0, + search_filter: SearchFilter | None = None, ) -> list[FileChunk]: """Hybrid search combining vector and keyword results.""" assert 0.0 <= vector_weight <= 1.0 @@ -279,8 +289,8 @@ class LocalFileStore(BaseFileStore): text_weight = 1.0 - vector_weight if self.vector_enabled and self.fts_enabled: - keyword_results = await self.keyword_search(query, candidates) - vector_results = await self.vector_search(query, candidates) + keyword_results = await self.keyword_search(query, candidates, search_filter) + vector_results = await self.vector_search(query, candidates, search_filter) if not keyword_results: return vector_results[:limit] @@ -295,9 +305,9 @@ class LocalFileStore(BaseFileStore): ) return merged[:limit] elif self.vector_enabled: - return await self.vector_search(query, limit) + return await self.vector_search(query, limit, search_filter) elif self.fts_enabled: - return await self.keyword_search(query, limit) + return await self.keyword_search(query, limit, search_filter) return [] @staticmethod diff --git a/reme2/component/file_watcher/__init__.py b/reme2/component/file_watcher/__init__.py index ecea0b28..69adcb01 100644 --- a/reme2/component/file_watcher/__init__.py +++ b/reme2/component/file_watcher/__init__.py @@ -1,9 +1,11 @@ """File watcher implementations for monitoring file system changes.""" from .base_file_watcher import BaseFileWatcher -from .md_file_watcher import MdFileWatcher +from .full_file_watcher import FullFileWatcher +from .light_file_watcher import LightFileWatcher __all__ = [ "BaseFileWatcher", - "MdFileWatcher", + "FullFileWatcher", + "LightFileWatcher", ] diff --git a/reme2/component/file_watcher/base_file_watcher.py b/reme2/component/file_watcher/base_file_watcher.py index 291a0f80..e7d8541c 100644 --- a/reme2/component/file_watcher/base_file_watcher.py +++ b/reme2/component/file_watcher/base_file_watcher.py @@ -7,6 +7,7 @@ from pathlib import Path from watchfiles import Change, awatch from ..base_component import BaseComponent +from ..file_parser import BaseFileParser from ..file_store import BaseFileStore from ...enumeration import ComponentEnum @@ -16,7 +17,7 @@ class BaseFileWatcher(BaseComponent): Provides file monitoring with: - watchfiles integration for efficient change detection - - Suffix-based filtering + - Parser-based file filtering - Auto-restart on failure - Optional index rebuild on start """ @@ -26,34 +27,23 @@ class BaseFileWatcher(BaseComponent): def __init__( self, watch_paths: list[str] | str, - suffix_filters: list[str] | None = None, recursive: bool = False, debounce: int = 2000, chunk_tokens: int = 400, chunk_overlap: int = 80, file_store: str = "default", + default_parser: str | None = None, rebuild_index_on_start: bool = True, poll_delay_ms: int = 2000, **kwargs, ): - """Initialize file watcher configuration. - - Args: - watch_paths: Paths to watch for changes. - suffix_filters: File suffix filters (e.g., ['.py', '.txt']). - recursive: Whether to watch directories recursively. - debounce: Debounce time in milliseconds. - chunk_tokens: Token size for chunking. - chunk_overlap: Overlap size for chunks. - file_store: Name of the file store component. - rebuild_index_on_start: Clear index and rescan files on start. - poll_delay_ms: Polling delay in milliseconds. - """ super().__init__(**kwargs) self._file_store_name: str = file_store + self._default_parser_name: str | None = default_parser self.file_store: BaseFileStore | None = None + self._suffix_to_parser: dict[str, BaseFileParser] = {} + self._default_parser: BaseFileParser | None = None self.watch_paths: list[str] = [watch_paths] if isinstance(watch_paths, str) else watch_paths - self.suffix_filters: list[str] = suffix_filters or [] self.recursive: bool = recursive self.debounce: int = debounce self.chunk_tokens: int = chunk_tokens @@ -68,6 +58,7 @@ class BaseFileWatcher(BaseComponent): """Resolve file_store and start watching task.""" if self._file_store_name: assert app_context is not None, "app_context must be provided" + stores = app_context.components.get(ComponentEnum.FILE_STORE, {}) if self._file_store_name not in stores: raise ValueError(f"File store '{self._file_store_name}' not found.") @@ -76,6 +67,20 @@ class BaseFileWatcher(BaseComponent): raise TypeError(f"Expected BaseFileStore, got {type(store).__name__}") self.file_store = store + parsers = app_context.components.get(ComponentEnum.FILE_PARSER, {}) + for parser in parsers.values(): + if isinstance(parser, BaseFileParser): + for suffix in parser.suffixes: + self._suffix_to_parser[suffix] = parser + + if self._default_parser_name and self._default_parser_name in parsers: + parser = parsers[self._default_parser_name] + if isinstance(parser, BaseFileParser): + self._default_parser = parser + + if not self._suffix_to_parser and not self._default_parser: + self.logger.warning("No file parsers registered") + async def _initialize_and_watch(): if self.rebuild_index_on_start and self.file_store: await self.file_store.clear_all() @@ -100,18 +105,10 @@ class BaseFileWatcher(BaseComponent): self._watch_task = None self._stop_event.clear() self.file_store = None + self._suffix_to_parser.clear() + self._default_parser = None self.logger.info("Stopped watching") - def watch_filter(self, _change: Change, path: str) -> bool: - """Filter files by suffix. Returns True if no filters configured.""" - if not self.suffix_filters: - return True - - for suffix in self.suffix_filters: - if path.endswith("." + suffix.strip(".")): - return True - return False - async def _scan_existing_files(self): """Scan existing files and add them as Change.added.""" if not self.file_store: @@ -127,17 +124,12 @@ class BaseFileWatcher(BaseComponent): continue if watch_path.is_file(): - if self.watch_filter(Change.added, str(watch_path)): - existing_files.add((Change.added, str(watch_path))) + existing_files.add((Change.added, str(watch_path))) elif watch_path.is_dir(): - if self.recursive: - for file_path in watch_path.rglob("*"): - if file_path.is_file() and self.watch_filter(Change.added, str(file_path)): - existing_files.add((Change.added, str(file_path))) - else: - for file_path in watch_path.iterdir(): - if file_path.is_file() and self.watch_filter(Change.added, str(file_path)): - existing_files.add((Change.added, str(file_path))) + iterator = watch_path.rglob("*") if self.recursive else watch_path.iterdir() + for file_path in iterator: + if file_path.is_file(): + existing_files.add((Change.added, str(file_path))) if existing_files: self.logger.info(f"[SCAN_ON_START] Found {len(existing_files)} existing files") @@ -180,7 +172,6 @@ class BaseFileWatcher(BaseComponent): self.logger.info(f"Starting watch on: {valid_paths}") async for changes in awatch( *valid_paths, - watch_filter=self.watch_filter, recursive=self.recursive, debounce=self.debounce, poll_delay_ms=self.poll_delay_ms, diff --git a/reme2/component/file_watcher/full_file_watcher.py b/reme2/component/file_watcher/full_file_watcher.py new file mode 100644 index 00000000..e2ca98ad --- /dev/null +++ b/reme2/component/file_watcher/full_file_watcher.py @@ -0,0 +1,48 @@ +"""Full file watcher for all supported file types.""" + +from pathlib import Path + +from watchfiles import Change + +from .base_file_watcher import BaseFileWatcher +from ..component_registry import R + + +@R.register("full") +class FullFileWatcher(BaseFileWatcher): + """Watches all supported file types and delegates parsing to registered parsers.""" + + async def _on_changes(self, changes: set[tuple[Change, str]]): + """Handle file changes by delegating to appropriate parsers.""" + if not self.file_store: + self.logger.warning("File store not initialized, skipping changes") + return + + for change_type, path in changes: + try: + if change_type in (Change.added, Change.modified): + suffix = Path(path).suffix.lower() + parser = self._suffix_to_parser.get(suffix, self._default_parser) + if not parser: + self.logger.debug(f"No parser available for file type {suffix}: {path}, skipping") + continue + + file_meta, chunks = await parser.parse(path) + await self.file_store.upsert_file(file_meta, chunks) + self.logger.info(f"Upserted {len(chunks)} chunks for {file_meta.path}") + + elif change_type == Change.deleted: + await self.file_store.delete_file(path) + self.logger.info(f"Deleted {path}") + + else: + self.logger.warning(f"Unknown change type: {change_type}") + + self.logger.info(f"File {change_type} changed: {path}") + + except FileNotFoundError: + self.logger.warning(f"File not found: {path}, skipping") + except PermissionError: + self.logger.warning(f"Permission denied: {path}, skipping") + except Exception as e: + self.logger.error(f"Error processing {path}: {e}", exc_info=True) diff --git a/reme2/component/file_watcher/light_file_watcher.py b/reme2/component/file_watcher/light_file_watcher.py new file mode 100644 index 00000000..8333e5c3 --- /dev/null +++ b/reme2/component/file_watcher/light_file_watcher.py @@ -0,0 +1,53 @@ +"""Lightweight file watcher for Markdown files only.""" + +from pathlib import Path + +from watchfiles import Change + +from .base_file_watcher import BaseFileWatcher +from ..component_registry import R + + +@R.register("light") +class LightFileWatcher(BaseFileWatcher): + """Watches only Markdown files and delegates parsing to registered parsers.""" + + async def _on_changes(self, changes: set[tuple[Change, str]]): + """Handle file changes by filtering for Markdown files only.""" + if not self.file_store: + self.logger.warning("File store not initialized, skipping changes") + return + + # Filter changes to only include Markdown files + md_changes = { + (change_type, path) for change_type, path in changes if Path(path).suffix.lower() in [".md", ".markdown"] + } + + for change_type, path in md_changes: + try: + if change_type in (Change.added, Change.modified): + # Use Markdown parser for Markdown files + parser = self._suffix_to_parser.get(".md", self._default_parser) + if not parser: + self.logger.warning(f"No parser available for Markdown file: {path}") + continue + + file_meta, chunks = await parser.parse(path) + await self.file_store.upsert_file(file_meta, chunks) + self.logger.info(f"Upserted {len(chunks)} chunks for {file_meta.path}") + + elif change_type == Change.deleted: + await self.file_store.delete_file(path) + self.logger.info(f"Deleted {path}") + + else: + self.logger.warning(f"Unknown change type: {change_type}") + + self.logger.info(f"Markdown file {change_type} changed: {path}") + + except FileNotFoundError: + self.logger.warning(f"File not found: {path}, skipping") + except PermissionError: + self.logger.warning(f"Permission denied: {path}, skipping") + except Exception as e: + self.logger.error(f"Error processing {path}: {e}", exc_info=True) diff --git a/reme2/component/file_watcher/md_file_watcher.py b/reme2/component/file_watcher/md_file_watcher.py deleted file mode 100644 index e339d975..00000000 --- a/reme2/component/file_watcher/md_file_watcher.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Markdown file watcher for synchronization.""" - -import asyncio -from pathlib import Path - -from watchfiles import Change - -from .base_file_watcher import BaseFileWatcher -from ..component_registry import R -from ...schema import FileMetadata -from ...utils import hash_text, chunk_markdown - - -@R.register("md") -class MdFileWatcher(BaseFileWatcher): - """Markdown file watcher that syncs .md files to memory store.""" - - def __init__(self, encoding: str = "utf-8", **kwargs): - """Initialize Markdown file watcher. - - Args: - encoding: File encoding. - """ - super().__init__(**kwargs) - self.encoding = encoding - - async def _on_changes(self, changes: set[tuple[Change, str]]): - """Handle file changes with full synchronization.""" - if not self.file_store: - self.logger.warning("File store not initialized, skipping changes") - return - - for change_type, path in changes: - try: - if change_type in [Change.added, Change.modified]: - file_meta = await self._build_file_metadata(path) - chunks = ( - chunk_markdown( - file_meta.content, - file_meta.path, - self.chunk_tokens, - self.chunk_overlap, - ) - or [] - ) - if chunks: - chunks = await self.file_store.get_chunk_embeddings(chunks) - file_meta.chunk_count = len(chunks) - - await self.file_store.delete_file(file_meta.path) - self.logger.info(f"delete_file {file_meta.path}") - - await self.file_store.upsert_file(file_meta, chunks) - self.logger.info(f"Upserted {file_meta.chunk_count} chunks for {file_meta.path}") - - elif change_type == Change.deleted: - await self.file_store.delete_file(path) - self.logger.info(f"Deleted {path}") - - else: - self.logger.warning(f"Unknown change type: {change_type}") - - self.logger.info(f"File {change_type} changed: {path}") - - except FileNotFoundError: - self.logger.warning(f"File not found: {path}, skipping") - except PermissionError: - self.logger.warning(f"Permission denied: {path}, skipping") - except Exception as e: - self.logger.error(f"Error processing {path}: {e}", exc_info=True) - - async def _build_file_metadata(self, path: str) -> FileMetadata: - """Build FileMetadata from file path.""" - file_path = Path(path) - - def _read_file_sync(): - return file_path.stat(), file_path.read_text(encoding=self.encoding) - - stat, content = await asyncio.to_thread(_read_file_sync) - return FileMetadata( - hash=hash_text(content), - mtime_ms=stat.st_mtime * 1000, - size=stat.st_size, - path=str(file_path.absolute()), - content=content, - ) diff --git a/reme2/component/service/base_service.py b/reme2/component/service/base_service.py index d70b4939..cb33069e 100644 --- a/reme2/component/service/base_service.py +++ b/reme2/component/service/base_service.py @@ -1,4 +1,5 @@ """Abstract base class for service implementations.""" + from abc import abstractmethod from typing import TYPE_CHECKING diff --git a/reme2/component/service/http_service.py b/reme2/component/service/http_service.py index 4b8f67b1..3f3477bd 100644 --- a/reme2/component/service/http_service.py +++ b/reme2/component/service/http_service.py @@ -27,7 +27,8 @@ class HttpService(BaseService): server-sent events (SSE) for real-time streaming. """ - from ...application import Application + # Removed the circular import - using string annotation instead + # from ...application import Application def __init__(self, host: str = REME_DEFAULT_HOST, port: int = REME_DEFAULT_PORT, **kwargs): """Initialize the HTTP service. @@ -97,7 +98,7 @@ class HttpService(BaseService): else: self._add_job(job) - def build_service(self, app: Application) -> None: + def build_service(self, app: "Application") -> None: """Build the FastAPI application with CORS middleware. Args: @@ -129,7 +130,7 @@ class HttpService(BaseService): ) self.service.post("/health")(lambda: {"status": "healthy"}) - def start_service(self, app: Application) -> None: + def start_service(self, app: "Application") -> None: """Start the HTTP server. Args: diff --git a/reme2/config/obsidian.yaml b/reme2/config/obsidian.md similarity index 97% rename from reme2/config/obsidian.yaml rename to reme2/config/obsidian.md index cd587b32..ca98a27e 100644 --- a/reme2/config/obsidian.yaml +++ b/reme2/config/obsidian.md @@ -6,7 +6,7 @@ reme version reme vault="My Vault" -# daily +# daily daily:path reme daily:xxx @@ -28,12 +28,12 @@ reme list file/path reme search query="search term" limit=10 tag="[]" score=0.1 copy=true # property -reme property:read +reme property:read reme property:update file="My Note" status=done xx=xxx reme property:delete keys="[xxxx, xxxx]" # 全局所有标签 -reme tags +reme tags # show link reme backlinks file="My Note" diff --git a/reme2/config/paw.yaml b/reme2/config/paw.yaml index 20d5c392..0b9c1d07 100644 --- a/reme2/config/paw.yaml +++ b/reme2/config/paw.yaml @@ -3,7 +3,7 @@ log_to_console: true log_to_file: false service: - backend: cmd + backend: http jobs: - backend: base @@ -22,18 +22,19 @@ jobs: backend: xxx components: - as_llms: + as_llm: default: backend: openai model_name: qwen3.6-plus - as_llm_formatters: + as_llm_formatter: default: backend: openai - embedding_models: + embedding_model: default: backend: openai + model_name: text-embedding-v3 dimensions: 1024 use_dimensions: false enable_cache: true @@ -41,15 +42,23 @@ components: max_cache_size: 2000 max_input_length: 8192 - file_stores: + file_parser: + md: + backend: md default: - backend: chroma + backend: default + + file_store: + default: + backend: local embedding_model: default store_name: "reme" + db_path: ".reme/store" - file_watchers: + file_watcher: default: backend: full file_store: default - suffix_filters: [ ".md" ] - recursive: false + default_parser: default + watch_paths: ["./test_data"] + recursive: true diff --git a/reme2/enumeration/component_enum.py b/reme2/enumeration/component_enum.py index 2bf13579..d3426ef7 100644 --- a/reme2/enumeration/component_enum.py +++ b/reme2/enumeration/component_enum.py @@ -24,6 +24,8 @@ class ComponentEnum(str, Enum): EMBEDDING_MODEL = "embedding_model" + FILE_PARSER = "file_parser" + FILE_STORE = "file_store" FILE_WATCHER = "file_watcher" diff --git a/reme2/file_based/file_io.py b/reme2/file_based/file_io.py index a7459440..f86b564a 100644 --- a/reme2/file_based/file_io.py +++ b/reme2/file_based/file_io.py @@ -40,10 +40,10 @@ class FileIO: return str(self.working_dir / file_path) async def read_file( # pylint: disable=too-many-return-statements - self, - file_path: str, - start_line: int | None = None, - end_line: int | None = None, + self, + file_path: str, + start_line: int | None = None, + end_line: int | None = None, ) -> ToolResponse: """Read a file. Relative paths resolve from WORKING_DIR. @@ -138,7 +138,7 @@ class FileIO: ) # Extract selected lines - selected_content = "\n".join(all_lines[s - 1: e]) + selected_content = "\n".join(all_lines[s - 1 : e]) # Apply smart truncation (consistent with shell output format) text = truncate_text_output( @@ -154,14 +154,13 @@ class FileIO: if text == selected_content and e < total: content_bytes = len(text.encode("utf-8")) notice = ( - TRUNCATION_NOTICE_MARKER + - f"\nThe output above was truncated." - f"\nThe full content is saved to the file " - f"and contains {total} lines in total." - f"\nThis excerpt starts at line {s} and " - f"covers the next {content_bytes} bytes." - "\nIf the current content is not enough, " - f"call `read_file` with file_path={file_path} start_line={e + 1} to read more." + TRUNCATION_NOTICE_MARKER + f"\nThe output above was truncated." + f"\nThe full content is saved to the file " + f"and contains {total} lines in total." + f"\nThis excerpt starts at line {s} and " + f"covers the next {content_bytes} bytes." + "\nIf the current content is not enough, " + f"call `read_file` with file_path={file_path} start_line={e + 1} to read more." ) text = text + notice @@ -180,9 +179,9 @@ class FileIO: ) async def write_file( - self, - file_path: str, - content: str, + self, + file_path: str, + content: str, ) -> ToolResponse: """Create or overwrite a file. Relative paths resolve from working_dir. @@ -227,10 +226,10 @@ class FileIO: # pylint: disable=too-many-return-statements async def edit_file( - self, - file_path: str, - old_text: str, - new_text: str, + self, + file_path: str, + old_text: str, + new_text: str, ) -> ToolResponse: """Find-and-replace text in a file. All occurrences of old_text are replaced with new_text. Relative paths resolve from working_dir. @@ -315,9 +314,9 @@ class FileIO: ) async def append_file( - self, - file_path: str, - content: str, + self, + file_path: str, + content: str, ) -> ToolResponse: """Append content to the end of a file. Relative paths resolve from working_dir. diff --git a/reme2/file_based/file_utils.py b/reme2/file_based/file_utils.py index 89981f63..bb775291 100644 --- a/reme2/file_based/file_utils.py +++ b/reme2/file_based/file_utils.py @@ -12,12 +12,12 @@ from ..constants import ( def _truncate_fresh( - text: str, - start_line: int, - total_lines: int, - max_bytes: int, - file_path: str | None, - encoding: str, + text: str, + start_line: int, + total_lines: int, + max_bytes: int, + file_path: str | None, + encoding: str, ) -> str: """Truncate fresh text (no prior truncation marker) by bytes with line integrity. @@ -66,21 +66,20 @@ def _truncate_fresh( return result notice = ( - TRUNCATION_NOTICE_MARKER - + f"\nThe output above was truncated." - f"\nThe full content is saved to the file and contains {total_lines} lines in total." - f"\nThis excerpt starts at line {start_line} and covers the next {max_bytes} bytes." - f"\nIf the current content is not enough, call `read_file` with file_path={file_path or ''} " - f"start_line={read_from} to read more." + TRUNCATION_NOTICE_MARKER + f"\nThe output above was truncated." + f"\nThe full content is saved to the file and contains {total_lines} lines in total." + f"\nThis excerpt starts at line {start_line} and covers the next {max_bytes} bytes." + f"\nIf the current content is not enough, call `read_file` with file_path={file_path or ''} " + f"start_line={read_from} to read more." ) return result + notice def _retruncate( - text: str, - max_bytes: int, - encoding: str, + text: str, + max_bytes: int, + encoding: str, ) -> str: """Re-truncate text that was previously truncated (contains TRUNCATION_NOTICE_MARKER). @@ -133,12 +132,12 @@ def _retruncate( def truncate_text_output( - text: str, - start_line: int = 1, - total_lines: int = 0, - max_bytes: int = DEFAULT_MAX_BYTES, - file_path: str | None = None, - encoding: str = "utf-8", + text: str, + start_line: int = 1, + total_lines: int = 0, + max_bytes: int = DEFAULT_MAX_BYTES, + file_path: str | None = None, + encoding: str = "utf-8", ) -> str: """Truncate file output by bytes with line integrity. diff --git a/reme2/file_based/memory_search.py b/reme2/file_based/memory_search.py index 77cab3a6..647090a4 100644 --- a/reme2/file_based/memory_search.py +++ b/reme2/file_based/memory_search.py @@ -5,6 +5,7 @@ import json from ..component import R from ..component.base_step import BaseStep from ..enumeration import ComponentEnum +from ..schema import SearchFilter @R.register("memory_search") @@ -33,17 +34,26 @@ class MemorySearch(BaseStep): max_results: int = self.context.get("max_results", 5) assert query, "Query cannot be empty" - assert isinstance(min_score, float | int) and 0.0 <= min_score <= 1.0, \ - f"min_score must be between 0 and 1, got {min_score}" - assert isinstance(max_results, int) and max_results > 0, \ - f"max_results must be a positive integer, got {max_results}" + assert ( + isinstance(min_score, float | int) and 0.0 <= min_score <= 1.0 + ), f"min_score must be between 0 and 1, got {min_score}" + assert ( + isinstance(max_results, int) and max_results > 0 + ), f"max_results must be a positive integer, got {max_results}" + + filter_paths: list[str] | None = self.context.get("paths") or None + filter_tags: list[str] | None = self.context.get("tags") or None + exclude_paths: list[str] | None = self.context.get("exclude_paths") or None + search_filter = None + if filter_paths or filter_tags or exclude_paths: + search_filter = SearchFilter(paths=filter_paths, tags=filter_tags, exclude_paths=exclude_paths) - # Use hybrid_search from file_store results = await self.file_store.hybrid_search( query=query, limit=max_results, vector_weight=self.vector_weight, candidate_multiplier=self.candidate_multiplier, + search_filter=search_filter, ) # Filter by min_score diff --git a/reme2/file_based/summarizer.py b/reme2/file_based/summarizer.py index 53284e2f..62335c9b 100644 --- a/reme2/file_based/summarizer.py +++ b/reme2/file_based/summarizer.py @@ -8,6 +8,7 @@ from agentscope.agent import ReActAgent from agentscope.message import Msg from agentscope.token import HuggingFaceTokenCounter from agentscope.tool import Toolkit +from loguru import logger from ..component import BaseStep from ..schema import AsMsgStat, AsBlockStat @@ -17,16 +18,16 @@ class Summarizer(BaseStep): """Summarizer step for summarizing memory messages.""" def __init__( - self, - working_dir: str, - memory_dir: str, - memory_compact_threshold: int, - toolkit: Toolkit | None = None, - console_enabled: bool = False, - timezone: str | None = None, - add_thinking_block: bool = True, - as_token_counter: HuggingFaceTokenCounter | None = None, - **kwargs, + self, + working_dir: str, + memory_dir: str, + memory_compact_threshold: int, + toolkit: Toolkit | None = None, + console_enabled: bool = False, + timezone: str | None = None, + add_thinking_block: bool = True, + as_token_counter: HuggingFaceTokenCounter | None = None, + **kwargs, ): """Initialize the summarizer step. @@ -51,7 +52,6 @@ class Summarizer(BaseStep): self.add_thinking_block: bool = add_thinking_block self._as_token_counter: HuggingFaceTokenCounter | None = as_token_counter - def _get_current_datetime(self) -> datetime.datetime: """Get current datetime with timezone, fallback to local time if timezone is invalid.""" if self.timezone: @@ -227,10 +227,10 @@ class Summarizer(BaseStep): return total async def _format_msgs_to_str( - self, - messages: list[Msg], - memory_compact_threshold: int, - include_thinking: bool = True, + self, + messages: list[Msg], + memory_compact_threshold: int, + include_thinking: bool = True, ) -> str: """Format list of messages to a single formatted string. diff --git a/reme2/reme.py b/reme2/reme.py index 68c0a92a..1cf55749 100644 --- a/reme2/reme.py +++ b/reme2/reme.py @@ -20,20 +20,19 @@ from .utils import run_coro_safely class ReMe(Application): """ReMe memory management application.""" - memory_path = working_path / "memory" - memory_path.mkdir(parents=True, exist_ok=True) + async def summarize( - self, - messages: list[Msg], - as_llm: str | ChatModelBase = "default", - as_llm_formatter: str | FormatterBase = "default", - as_token_counter: str | TokenCounterBase | HuggingFaceTokenCounter = "default", - toolkit: Toolkit | None = None, - language: str = "zh", - max_input_length: float = 128 * 1024, - compact_ratio: float = 0.7, - timezone: str | None = None, - add_thinking_block: bool = True, + self, + messages: list[Msg], + as_llm: str | ChatModelBase = "default", + as_llm_formatter: str | FormatterBase = "default", + as_token_counter: str | TokenCounterBase | HuggingFaceTokenCounter = "default", + toolkit: Toolkit | None = None, + language: str = "zh", + max_input_length: float = 128 * 1024, + compact_ratio: float = 0.7, + timezone: str | None = None, + add_thinking_block: bool = True, ) -> str: """Summarize and compact memory messages. @@ -87,6 +86,7 @@ class ReMe(Application): async def memory_search(self, query: str, max_results: int = 5, min_score: float = 0.1) -> str: """Search memory for relevant entries.""" from .file_based.memory_search import MemorySearch + try: search_step = MemorySearch() self.logger.info(f"Running memory search with {query} {max_results} {min_score}") @@ -95,26 +95,28 @@ class ReMe(Application): return str(e) async def dream( - self, - as_llm: str | ChatModelBase = "default", - as_llm_formatter: str | FormatterBase = "default", - as_token_counter: str | TokenCounterBase = "default", - toolkit: Toolkit | None = None, - language: str = "zh", - timezone: str | None = None, + self, + as_llm: str | ChatModelBase = "default", + as_llm_formatter: str | FormatterBase = "default", + as_token_counter: str | TokenCounterBase = "default", + toolkit: Toolkit | None = None, + language: str = "zh", + timezone: str | None = None, ) -> str: """Process and consolidate memories in background.""" + return "" async def proactive( - self, - as_llm: str | ChatModelBase = "default", - as_llm_formatter: str | FormatterBase = "default", - as_token_counter: str | TokenCounterBase = "default", - toolkit: Toolkit | None = None, - language: str = "zh", - timezone: str | None = None, + self, + as_llm: str | ChatModelBase = "default", + as_llm_formatter: str | FormatterBase = "default", + as_token_counter: str | TokenCounterBase = "default", + toolkit: Toolkit | None = None, + language: str = "zh", + timezone: str | None = None, ) -> str: """Generate proactive memory insights.""" + return "" class ReMeLight(ReMe): diff --git a/reme2/schema/__init__.py b/reme2/schema/__init__.py index 0e6f16ef..7940a1be 100644 --- a/reme2/schema/__init__.py +++ b/reme2/schema/__init__.py @@ -7,6 +7,7 @@ from .file_chunk import FileChunk from .file_metadata import FileMetadata from .request import Request from .response import Response +from .search_filter import SearchFilter from .stream_chunk import StreamChunk __all__ = [ @@ -20,5 +21,6 @@ __all__ = [ "FileMetadata", "Request", "Response", + "SearchFilter", "StreamChunk", ] diff --git a/reme2/schema/search_filter.py b/reme2/schema/search_filter.py new file mode 100644 index 00000000..bc0a2ad5 --- /dev/null +++ b/reme2/schema/search_filter.py @@ -0,0 +1,36 @@ +"""Search filter schema for constraining search results.""" + +from pydantic import BaseModel, Field + + +class SearchFilter(BaseModel): + """Filter conditions for search operations. + + All specified conditions are combined with AND logic. + Within paths/exclude_paths, items are combined with OR logic. + Within tags, items are combined with AND logic (all must match). + """ + + paths: list[str] | None = Field( + default=None, + description="Include only chunks whose path starts with any of these prefixes", + ) + tags: list[str] | None = Field(default=None, description="Include only chunks containing ALL specified tags") + exclude_paths: list[str] | None = Field( + default=None, + description="Exclude chunks whose path starts with any of these prefixes", + ) + + def is_empty(self) -> bool: + return not self.paths and not self.tags and not self.exclude_paths + + def match(self, path: str, metadata: dict | None = None) -> bool: + if self.paths and not any(path.startswith(p) for p in self.paths): + return False + if self.exclude_paths and any(path.startswith(p) for p in self.exclude_paths): + return False + if self.tags: + chunk_tags = set((metadata or {}).get("tags", [])) + if not all(t in chunk_tags for t in self.tags): + return False + return True diff --git a/tests/test_reme2.py b/tests/test_reme2.py new file mode 100644 index 00000000..a9dbe126 --- /dev/null +++ b/tests/test_reme2.py @@ -0,0 +1,201 @@ +import asyncio +import os +import tempfile +from pathlib import Path + +from dotenv import load_dotenv + +load_dotenv() + +from reme2.config.config_parser import _load_yaml +from reme2.reme import ReMe + + +async def test_full_watcher(): + """测试full watcher模式 - 处理所有文件类型""" + print("\n=== 测试 Full Watcher 模式 ===") + + config = _load_yaml("paw") + + # 设置API密钥 + config["components"]["embedding_model"]["default"]["api_key"] = os.environ["EMBEDDING_API_KEY"] + config["components"]["embedding_model"]["default"]["base_url"] = os.environ["EMBEDDING_BASE_URL"] + + config["components"]["as_llm"]["default"]["api_key"] = os.environ["LLM_API_KEY"] + config["components"]["as_llm"]["default"]["base_url"] = os.environ["LLM_BASE_URL"] + config["components"]["as_llm"]["default"]["model_name"] = os.environ["LLM_MODEL_NAME"] + + # 创建临时目录用于测试 + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # 创建测试文件 + md_file = temp_path / "test.md" + txt_file = temp_path / "test.txt" + py_file = temp_path / "test.py" + + md_content = """# Test Markdown File + +This is a test markdown file for full watcher. + +## Section +Some content here about Python programming. +""" + + txt_content = """Plain text file for testing full watcher. +Contains some interesting content about machine learning AI.""" + + py_content = '''""" +Test Python file for full watcher. +Demonstrates Python programming concepts. +""" +def hello_world(): + """A simple function.""" + print("Hello, world!") + # Machine learning related code + return True +''' + + md_file.write_text(md_content) + txt_file.write_text(txt_content) + py_file.write_text(py_content) + + # 更新配置使用临时目录 + config["components"]["file_watcher"]["default"]["watch_paths"] = [str(temp_path)] + config["components"]["file_watcher"]["default"]["recursive"] = False + + reme = ReMe(**config) + + # Start components (embedding, file_store, file_parser, file_watcher) + await reme.start() + + # Wait for watcher to scan existing files + await asyncio.sleep(3) + + # Test: list indexed files + from reme2.enumeration import ComponentEnum + + store = reme.context.components[ComponentEnum.FILE_STORE]["default"] + + files = await store.list_files() + print(f"\n=== Indexed files by Full Watcher ({len(files)}) ===") + for f in files: + meta = await store.get_file_metadata(f) + chunks = await store.get_file_chunks(f) + print(f" {f} — {meta.chunk_count} chunks, metadata={meta.metadata}") + + # Test: keyword search + results = await store.keyword_search("Python programming", limit=5) + print(f"\n=== Keyword search 'Python programming' ({len(results)} results) ===") + for r in results: + print(f" [{r.score:.2f}] {r.path}:{r.start_line}-{r.end_line} — {r.text[:80]}") + + # Test: hybrid search + results = await store.hybrid_search("machine learning AI", limit=5) + print(f"\n=== Hybrid search 'machine learning AI' ({len(results)} results) ===") + for r in results: + print(f" [{r.score:.2f}] {r.path}:{r.start_line}-{r.end_line} — {r.text[:80]}") + + await reme.close() + print("\n=== Full Watcher 测试完成 ===") + + +async def test_light_watcher(): + """测试light watcher模式 - 只处理markdown文件""" + print("\n=== 测试 Light Watcher 模式 ===") + + config = _load_yaml("paw") + + # 设置API密钥 + config["components"]["embedding_model"]["default"]["api_key"] = os.environ["EMBEDDING_API_KEY"] + config["components"]["embedding_model"]["default"]["base_url"] = os.environ["EMBEDDING_BASE_URL"] + + config["components"]["as_llm"]["default"]["api_key"] = os.environ["LLM_API_KEY"] + config["components"]["as_llm"]["default"]["base_url"] = os.environ["LLM_BASE_URL"] + config["components"]["as_llm"]["default"]["model_name"] = os.environ["LLM_MODEL_NAME"] + + # 更改配置使用light watcher + config["components"]["file_watcher"]["default"]["backend"] = "light" + + # 创建临时目录用于测试 + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # 创建测试文件 - 包含markdown和其他类型文件 + md_file = temp_path / "light_test.md" + txt_file = temp_path / "light_test.txt" + py_file = temp_path / "light_test.py" + + md_content = """# Light Test Markdown File + +This is a test markdown file for light watcher. + +## Section +Only markdown files should be processed by light watcher. +""" + + txt_content = "This text file should be ignored by light watcher." + py_content = "# This Python file should be ignored by light watcher" + + md_file.write_text(md_content) + txt_file.write_text(txt_content) + py_file.write_text(py_content) + + # 更新配置使用临时目录 + config["components"]["file_watcher"]["default"]["watch_paths"] = [str(temp_path)] + config["components"]["file_watcher"]["default"]["recursive"] = False + + reme = ReMe(**config) + + # Start components + await reme.start() + + # Wait for watcher to scan existing files + await asyncio.sleep(3) + + # Test: list indexed files - should only contain markdown files + from reme2.enumeration import ComponentEnum + + store = reme.context.components[ComponentEnum.FILE_STORE]["default"] + + files = await store.list_files() + print(f"\n=== Indexed files by Light Watcher ({len(files)}) ===") + for f in files: + # 验证只有markdown文件被索引 + file_path = Path(f) + if file_path.suffix.lower() in [".md", ".markdown"]: + meta = await store.get_file_metadata(f) + chunks = await store.get_file_chunks(f) + print(f" {f} — {meta.chunk_count} chunks, metadata={meta.metadata}") + else: + print(f" ERROR: Non-markdown file indexed: {f}") + + print(f"\nExpected: Only markdown files should be indexed. Actually indexed: {len(files)} files") + + # Test: search + results = await store.keyword_search("light watcher", limit=5) + print(f"\n=== Keyword search 'light watcher' ({len(results)} results) ===") + for r in results: + print(f" [{r.score:.2f}] {r.path}:{r.start_line}-{r.end_line} — {r.text[:80]}") + + await reme.close() + print("\n=== Light Watcher 测试完成 ===") + + +async def main(): + """主函数，运行所有测试""" + print("开始测试ReMe系统的Full和Light Watcher模式") + + try: + await test_full_watcher() + await test_light_watcher() + print("\n✅ 所有测试完成！") + except Exception as e: + print(f"\n❌ 测试失败: {e}") + import traceback + + traceback.print_exc() + + +if __name__ == "__main__": + asyncio.run(main())