From 82337ead336d57d4d01c5a6feb1074e9464d2881 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Sun, 8 Feb 2026 21:37:55 +0800 Subject: [PATCH] feat(file-watcher): add file watching functionality with error handling --- pyproject.toml | 36 +- reme/config/fs.yaml | 8 + reme/core/context/service_context.py | 3 + reme/core/file_watcher/base_file_watcher.py | 27 +- reme/core/memory_store/sqlite_memory_store.py | 40 +- reme/reme_fs.py | 9 +- tests/test_fs_file_watch_integration.py | 523 ++++++++++++++++++ tests/test_memory_store.py | 43 +- 8 files changed, 657 insertions(+), 32 deletions(-) create mode 100644 tests/test_fs_file_watch_integration.py diff --git a/pyproject.toml b/pyproject.toml index 19824b66..df1d919f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,16 +30,42 @@ classifiers = [ "Typing :: Typed", ] -keywords = ["llm", "memory", "experience", "memoryscope", "ai", "mcp", "http"] +keywords = ["llm", "memory", "experience", "memoryscope", "ai", "mcp", "http", "reme", "personal"] dependencies = [ "flowllm[reme]>=0.2.0.10", "sqlite-vec>=0.1.6", "prompt_toolkit>=3.0.52", - "rich>=13.0.0", + "rich>=14.2.0", + "asyncpg>=0.31.0", + "chromadb>=1.3.5", + "dashscope>=1.25.1", + "elasticsearch>=9.2.0", + "fastapi>=0.121.3", + "fastmcp>=2.14.1", + "httpx>=0.28.1", + "litellm>=1.80.0", + "loguru>=0.7.3", + "mcp>=1.25.0", + "numpy>=2.2.6", + "openai>=2.8.1", + "pandas>=2.3.3", + "pydantic>=2.12.4", + "qdrant-client>=1.16.0", + "tavily-python>=0.7.13", + "tiktoken>=0.12.0", + "tqdm>=4.67.1", + "transformers>=4.57.3", + "uvicorn>=0.40.0", + "watchfiles>=1.1.1", + "pyyaml>=6.0.3", ] [project.optional-dependencies] +ray = [ + "ray", +] + dev = [ "jupyter-book", "ghp-import", @@ -50,12 +76,8 @@ dev = [ "pre-commit", ] -token = [ - "flowllm[token]>=0.2.0.10" -] - full = [ - "reme_ai[dev,token]" + "reme_ai[dev,ray]" ] [tool.setuptools.packages.find] diff --git a/reme/config/fs.yaml b/reme/config/fs.yaml index b88583f3..3446890b 100644 --- a/reme/config/fs.yaml +++ b/reme/config/fs.yaml @@ -22,6 +22,14 @@ memory_stores: fts_enabled: true snippet_max_chars: 700 +file_watchers: + default: + backend: full + watch_paths: [".reme", ".reme/memory"] + suffix_filters: [".md"] + recursive: false + scan_on_start: true + token_counters: default: backend: base diff --git a/reme/core/context/service_context.py b/reme/core/context/service_context.py index c0fc57dc..9b6d3895 100644 --- a/reme/core/context/service_context.py +++ b/reme/core/context/service_context.py @@ -219,6 +219,9 @@ class ServiceContext(BaseContext): for _, memory_store in self.memory_stores.items(): await memory_store.close() + for _, file_watcher in self.file_watchers.items(): + await file_watcher.close() + for _, llm in self.llms.items(): await llm.close() diff --git a/reme/core/file_watcher/base_file_watcher.py b/reme/core/file_watcher/base_file_watcher.py index dae61a43..c5313a7b 100644 --- a/reme/core/file_watcher/base_file_watcher.py +++ b/reme/core/file_watcher/base_file_watcher.py @@ -150,17 +150,24 @@ class BaseFileWatcher: logger.warning("No watch paths specified") return - async for changes in awatch( - *self.watch_paths, - watch_filter=self.watch_filter, - recursive=self.recursive, - debounce=self.debounce, - stop_event=self._stop_event, - ): - if self._stop_event.is_set(): - break + try: + async for changes in awatch( + *self.watch_paths, + watch_filter=self.watch_filter, + recursive=self.recursive, + debounce=self.debounce, + stop_event=self._stop_event, + ): + if self._stop_event.is_set(): + break - await self.on_changes(changes) + await self.on_changes(changes) + except FileNotFoundError as e: + # Watch path was deleted, this is expected during cleanup + logger.debug(f"Watch path no longer exists: {e}") + except Exception as e: + # Log other exceptions but don't crash + logger.error(f"Error in watch loop: {e}", exc_info=True) async def _on_changes(self, changes: set[tuple[Change, str]]): """Callback method to handle file changes""" diff --git a/reme/core/memory_store/sqlite_memory_store.py b/reme/core/memory_store/sqlite_memory_store.py index 7e1519b8..e5a32bb6 100644 --- a/reme/core/memory_store/sqlite_memory_store.py +++ b/reme/core/memory_store/sqlite_memory_store.py @@ -558,6 +558,38 @@ class SqliteMemoryStore(BaseMemoryStore): finally: cursor.close() + def _sanitize_fts_query(self, query: str) -> str: + """Sanitize query string for FTS5 search. + + Removes or escapes special characters that have special meaning in FTS5: + - * (prefix match) + - ? (not used in FTS5, but can cause issues) + - " (phrase search, needs escaping) + - : (column filter) + - ^ (start of line anchor, not standard FTS5) + - Other special chars that may interfere + + Args: + query: Raw query string + + Returns: + Sanitized query string safe for FTS5 + """ + if not query: + return "" + + # Remove FTS5 special characters that we don't want users to use + # Keep only alphanumeric, spaces, and some safe punctuation + special_chars = ["*", "?", ":", "^", "(", ")", "[", "]", "{", "}"] + cleaned = query + for char in special_chars: + cleaned = cleaned.replace(char, " ") + + # Normalize whitespace + cleaned = " ".join(cleaned.split()) + + return cleaned + async def keyword_search( self, query: str, @@ -568,14 +600,12 @@ class SqliteMemoryStore(BaseMemoryStore): if not self.fts_available: return [] - # Build FTS5 query - # Split query into tokens and join with OR for better recall - # Individual words are automatically stemmed and matched by FTS5 - cleaned = query.strip() + # Sanitize and prepare query + cleaned = self._sanitize_fts_query(query) if not cleaned: return [] - # Split into words and escape each + # Split into words and escape double quotes for FTS5 phrase matching words = cleaned.split() if not words: return [] diff --git a/reme/reme_fs.py b/reme/reme_fs.py index c113eaa5..c28d9c32 100644 --- a/reme/reme_fs.py +++ b/reme/reme_fs.py @@ -71,14 +71,7 @@ class ReMeFs(Application): default_embedding_model_config=default_embedding_model_config, default_memory_store_config=default_memory_store_config, default_token_counter_config=default_token_counter_config, - default_file_watcher_config=default_file_watcher_config - or { - "backend": "full", - "watch_paths": [working_dir, working_dir + "/memory"], - "suffix_filters": [".md"], - "recursive": False, - "scan_on_start": True, - }, + default_file_watcher_config=default_file_watcher_config, **kwargs, ) self.working_dir: str = working_dir diff --git a/tests/test_fs_file_watch_integration.py b/tests/test_fs_file_watch_integration.py new file mode 100644 index 00000000..e7a1658d --- /dev/null +++ b/tests/test_fs_file_watch_integration.py @@ -0,0 +1,523 @@ +"""Integration test for ReMeFs file watching with memory_search and memory_get. + +This test demonstrates the complete workflow: +1. Create markdown files with personal information in test_reme folder +2. Initialize ReMeFs with file watching enabled +3. Start file watching to automatically index files into the database +4. Use memory_search and memory_get to retrieve the indexed content +5. Modify the markdown files +6. Verify that modified content is properly indexed and retrievable + +This validates the full pipeline: + - File creation → File watcher → Database indexing + - Search and retrieval functionality + - File modification → Re-indexing → Updated search results +""" + +import asyncio +import json +import shutil +from pathlib import Path + +from reme import ReMeFs + + +# ==================== Test Configuration ==================== + + +class TestConfig: + """Test configuration settings.""" + + WORKING_DIR = "test_reme" + MEMORY_SUBDIR = "memory" + + +# ==================== Helper Functions ==================== + + +def create_test_markdown_files(base_dir: str): + """Create test markdown files with personal information. + + Args: + base_dir: Base directory to create test files in + """ + base_path = Path(base_dir) + base_path.mkdir(parents=True, exist_ok=True) + + memory_path = base_path / TestConfig.MEMORY_SUBDIR + memory_path.mkdir(parents=True, exist_ok=True) + + # Create personal profile markdown + profile_file = memory_path / "profile.md" + profile_content = """# Personal Profile + +## Basic Information +My name is Zhang Wei (张伟). I am a 32-year-old software engineer living in Beijing, China. +I work at ByteDance as a senior backend engineer. + +## Professional Skills +- Programming Languages: Python, Go, Java +- Specialization: Distributed systems and microservices architecture +- Experience: 8 years in software development + +## Education +- Master's degree in Computer Science from Tsinghua University (2014) +- Focus on machine learning and data mining +""" + profile_file.write_text(profile_content, encoding="utf-8") + print(f"✓ Created: {profile_file}") + + # Create hobbies and interests markdown + hobbies_file = memory_path / "hobbies.md" + hobbies_content = """# Hobbies and Interests + +## Technical Interests +I am passionate about cloud computing and containerization technologies. +Recently, I've been exploring Kubernetes and service mesh architectures. + +## Personal Hobbies +- Reading: Love science fiction novels, especially works by Liu Cixin +- Sports: Play basketball every weekend with friends +- Travel: Visited 15 provinces in China, planning to visit Japan next year + +## Learning Goals +- Deep dive into distributed tracing systems +- Learn more about database internals +- Improve English communication skills +""" + hobbies_file.write_text(hobbies_content, encoding="utf-8") + print(f"✓ Created: {hobbies_file}") + + # Create work projects markdown + projects_file = memory_path / "projects.md" + projects_content = """# Work Projects + +## Current Projects + +### Project Alpha (2024-present) +Building a high-performance message queue system to handle 1M+ QPS. +Using Go and Redis for the core infrastructure. + +### Project Beta (2023-2024) +Developed a distributed configuration management system. +Integrated with Kubernetes for dynamic config updates. + +## Past Experience +- Led the migration of monolithic services to microservices (2021-2023) +- Built automated deployment pipelines using Jenkins and GitLab CI (2020-2021) + +## Technical Challenges Solved +- Resolved race conditions in concurrent data processing +- Optimized database queries reducing response time by 60% +""" + projects_file.write_text(projects_content, encoding="utf-8") + print(f"✓ Created: {projects_file}") + + return [profile_file, hobbies_file, projects_file] + + +def modify_test_markdown_files(base_dir: str): + """Modify the test markdown files with updated information. + + Args: + base_dir: Base directory containing test files + """ + base_path = Path(base_dir) + memory_path = base_path / TestConfig.MEMORY_SUBDIR + + # Modify profile - update job title and add new skill + profile_file = memory_path / "profile.md" + profile_content = """# Personal Profile + +## Basic Information +My name is Zhang Wei (张伟). I am a 32-year-old software engineer living in Beijing, China. +I work at ByteDance as a **principal engineer** and tech lead. + +## Professional Skills +- Programming Languages: Python, Go, Java, Rust +- Specialization: Distributed systems, microservices, and cloud-native architectures +- Experience: 8 years in software development +- **New**: Expert in observability and monitoring systems + +## Education +- Master's degree in Computer Science from Tsinghua University (2014) +- Focus on machine learning and data mining +""" + profile_file.write_text(profile_content, encoding="utf-8") + print(f"✓ Modified: {profile_file}") + + # Modify hobbies - add new hobby + hobbies_file = memory_path / "hobbies.md" + hobbies_content = """# Hobbies and Interests + +## Technical Interests +I am passionate about cloud computing and containerization technologies. +Recently, I've been exploring Kubernetes, service mesh, and eBPF technologies. + +## Personal Hobbies +- Reading: Love science fiction novels, especially works by Liu Cixin +- Sports: Play basketball every weekend with friends +- Travel: Visited 15 provinces in China, planning to visit Japan next year +- **New**: Photography - Recently bought a Sony A7 III camera + +## Learning Goals +- Deep dive into distributed tracing and eBPF +- Learn more about database internals and query optimization +- Improve English communication skills +- **New**: Master advanced photography techniques +""" + hobbies_file.write_text(hobbies_content, encoding="utf-8") + print(f"✓ Modified: {hobbies_file}") + + # Modify projects - add new project + projects_file = memory_path / "projects.md" + projects_content = """# Work Projects + +## Current Projects + +### Project Gamma (2024-present) **NEW** +Leading the development of an observability platform using OpenTelemetry. +Integrating metrics, traces, and logs into a unified dashboard. + +### Project Alpha (2024-present) +Building a high-performance message queue system to handle 1M+ QPS. +Using Go and Redis for the core infrastructure. +**Update**: Successfully deployed to production, handling 2M+ QPS now. + +### Project Beta (2023-2024) +Developed a distributed configuration management system. +Integrated with Kubernetes for dynamic config updates. + +## Past Experience +- Led the migration of monolithic services to microservices (2021-2023) +- Built automated deployment pipelines using Jenkins and GitLab CI (2020-2021) + +## Technical Challenges Solved +- Resolved race conditions in concurrent data processing +- Optimized database queries reducing response time by 60% +- **New**: Implemented distributed tracing reducing MTTR by 40% +""" + projects_file.write_text(projects_content, encoding="utf-8") + print(f"✓ Modified: {projects_file}") + + +def print_separator(title: str): + """Print a formatted separator line.""" + print(f"\n{'=' * 80}") + print(f" {title}") + print(f"{'=' * 80}\n") + + +def print_search_results(results: list[dict], query: str, context: str): + """Pretty print search results. + + Args: + results: List of search results + query: The search query + context: Context description (e.g., "BEFORE MODIFICATION") + """ + print(f"\n{'-' * 80}") + print(f"Search Results - {context}") + print(f"Query: '{query}'") + print(f"Found: {len(results)} results") + print(f"{'-' * 80}") + + for i, result in enumerate(results, 1): + print(f"\n[{i}] Path: {result.get('path', 'N/A')}") + print(f" Lines: {result.get('start_line', '?')}-{result.get('end_line', '?')}") + print(f" Score: {result.get('score', 0):.4f}") + snippet = result.get("snippet", result.get("text", "")) + if len(snippet) > 200: + snippet = snippet[:200] + "..." + print(f" Snippet: {snippet}") + + print(f"{'-' * 80}\n") + + +def print_get_result(content: str, path: str, context: str): + """Pretty print memory_get result. + + Args: + content: Content retrieved from memory_get + path: File path + context: Context description + """ + print(f"\n{'-' * 80}") + print(f"Memory Get Result - {context}") + print(f"Path: {path}") + print(f"Content length: {len(content)} chars, {len(content.split(chr(10)))} lines") + print(f"{'-' * 80}") + print(content[:500] + ("..." if len(content) > 500 else "")) + print(f"{'-' * 80}\n") + + +# ==================== Test Functions ==================== + + +async def test_file_watch_integration(): + """Complete integration test for file watching with search and get. + + This test validates: + 1. File creation and automatic indexing via file watcher + 2. Search functionality returns correct results + 3. Get functionality retrieves correct content + 4. File modification triggers re-indexing + 5. Updated content is properly searchable and retrievable + """ + print_separator("FILE WATCH INTEGRATION TEST - START") + + # Clean up any existing test directory + test_dir = Path(TestConfig.WORKING_DIR) + if test_dir.exists(): + shutil.rmtree(test_dir) + print(f"✓ Cleaned up existing test directory: {test_dir}") + + # ==================== STEP 1: Create Test Files ==================== + print_separator("STEP 1: Creating Test Files") + + test_files = create_test_markdown_files(TestConfig.WORKING_DIR) + print(f"\n✓ Created {len(test_files)} markdown files in {TestConfig.WORKING_DIR}") + + # ==================== STEP 2: Initialize ReMeFs ==================== + print_separator("STEP 2: Initializing ReMeFs with File Watching") + + reme_fs = ReMeFs( + enable_logo=False, + working_dir=TestConfig.WORKING_DIR, + default_memory_store_config={ + "backend": "sqlite", + "store_name": "test_integration", + "embedding_model": "default", + "fts_enabled": True, + "snippet_max_chars": 700, + }, + default_file_watcher_config={ + "backend": "full", + "watch_paths": [TestConfig.WORKING_DIR, f"{TestConfig.WORKING_DIR}/memory"], + "suffix_filters": [".md"], + "recursive": False, + "scan_on_start": True, + }, + ) + + print("✓ ReMeFs instance created") + print(f" Working directory: {TestConfig.WORKING_DIR}") + print(f" Watch paths: {TestConfig.WORKING_DIR}, {TestConfig.WORKING_DIR}/memory") + print(" File filters: .md files") + + # ==================== STEP 3: Start File Watching ==================== + print_separator("STEP 3: Starting File Watcher") + + await reme_fs.start() + print("✓ File watcher started") + print(" Files will be automatically indexed into the database") + + # Give file watcher time to process files + print("\nWaiting 3 seconds for file watcher to index files...") + await asyncio.sleep(3) + print("✓ File watcher should have processed all files") + + # ==================== STEP 4: Search Initial Content ==================== + print_separator("STEP 4: Searching Initial Content") + + queries_initial = [ + "What programming languages does Zhang Wei know?", + "What are Zhang Wei's hobbies?", + "What projects is Zhang Wei working on?", + ] + + results_before = {} + + for query in queries_initial: + print(f"\n📍 Searching: '{query}'") + result_json = await reme_fs.memory_search( + query=query, + max_results=3, + min_score=0.0, + ) + results = json.loads(result_json) + results_before[query] = results + print_search_results(results, query, "BEFORE MODIFICATION") + + assert len(results) > 0, f"Should find results for query: {query}" + print(f"✓ Found {len(results)} results") + + # ==================== STEP 5: Get Specific Content ==================== + print_separator("STEP 5: Getting Specific Content with memory_get") + + # Try to get content from profile.md + profile_path = f"{TestConfig.MEMORY_SUBDIR}/profile.md" + print(f"\n📍 Getting content from: {profile_path}") + + profile_content_before = await reme_fs.memory_get( + path=profile_path, + offset=1, + limit=10, + ) + print_get_result(profile_content_before, profile_path, "BEFORE MODIFICATION") + + assert "Zhang Wei" in profile_content_before, "Should contain Zhang Wei" + assert "software engineer" in profile_content_before, "Should contain job title" + print("✓ Content retrieved successfully") + + # Get full hobbies.md content + hobbies_path = f"{TestConfig.MEMORY_SUBDIR}/hobbies.md" + print(f"\n📍 Getting full content from: {hobbies_path}") + + hobbies_content_before = await reme_fs.memory_get(path=hobbies_path) + print_get_result(hobbies_content_before, hobbies_path, "BEFORE MODIFICATION") + + assert "basketball" in hobbies_content_before, "Should contain hobbies" + print("✓ Full content retrieved successfully") + + # ==================== STEP 6: Modify Files ==================== + print_separator("STEP 6: Modifying Test Files") + + print("Modifying markdown files with updated information...") + modify_test_markdown_files(TestConfig.WORKING_DIR) + + # Give file watcher time to detect and re-index changes + print("\nWaiting 3 seconds for file watcher to detect and re-index changes...") + await asyncio.sleep(3) + print("✓ File watcher should have re-indexed modified files") + + # ==================== STEP 7: Search Modified Content ==================== + print_separator("STEP 7: Searching Modified Content") + + queries_modified = [ + "What is Zhang Wei's current job title?", + "Does Zhang Wei have any new hobbies?", + "What new projects is Zhang Wei working on?", + "What expertise does Zhang Wei have in observability?", + ] + + results_after = {} + + for query in queries_modified: + print(f"\n📍 Searching: '{query}'") + result_json = await reme_fs.memory_search( + query=query, + max_results=3, + min_score=0.0, + ) + results = json.loads(result_json) + results_after[query] = results + print_search_results(results, query, "AFTER MODIFICATION") + + assert len(results) > 0, f"Should find results for query: {query}" + print(f"✓ Found {len(results)} results") + + # ==================== STEP 8: Get Modified Content ==================== + print_separator("STEP 8: Getting Modified Content") + + # Get updated profile content + print(f"\n📍 Getting updated content from: {profile_path}") + profile_content_after = await reme_fs.memory_get( + path=profile_path, + offset=1, + limit=10, + ) + print_get_result(profile_content_after, profile_path, "AFTER MODIFICATION") + + assert "principal engineer" in profile_content_after, "Should contain updated job title" + assert "Rust" in profile_content_after, "Should contain new programming language" + print("✓ Updated profile content retrieved successfully") + + # Get updated hobbies content + print(f"\n📍 Getting updated content from: {hobbies_path}") + hobbies_content_after = await reme_fs.memory_get(path=hobbies_path) + print_get_result(hobbies_content_after, hobbies_path, "AFTER MODIFICATION") + + assert "Photography" in hobbies_content_after, "Should contain new hobby" + assert "Sony A7 III" in hobbies_content_after, "Should contain camera info" + print("✓ Updated hobbies content retrieved successfully") + + # Get updated projects content + projects_path = f"{TestConfig.MEMORY_SUBDIR}/projects.md" + print(f"\n📍 Getting updated content from: {projects_path}") + projects_content_after = await reme_fs.memory_get(path=projects_path) + print_get_result(projects_content_after, projects_path, "AFTER MODIFICATION") + + assert "Project Gamma" in projects_content_after, "Should contain new project" + assert "OpenTelemetry" in projects_content_after, "Should contain new technology" + print("✓ Updated projects content retrieved successfully") + + # ==================== STEP 9: Verify Changes ==================== + print_separator("STEP 9: Verifying Content Changes") + + print("\n📍 Comparing BEFORE vs AFTER content:") + + # Verify profile changes + print("\n1. Profile.md changes:") + print(f" Before: Contains 'software engineer' = {('software engineer' in profile_content_before.lower())}") + print(f" After: Contains 'principal engineer' = {('principal engineer' in profile_content_after.lower())}") + print(f" After: Contains 'Rust' = {('rust' in profile_content_after.lower())}") + + # Verify hobbies changes + print("\n2. Hobbies.md changes:") + print(f" Before: Contains 'Photography' = {('photography' in hobbies_content_before.lower())}") + print(f" After: Contains 'Photography' = {('photography' in hobbies_content_after.lower())}") + print(f" After: Contains 'Sony A7 III' = {('sony' in hobbies_content_after.lower())}") + + # Verify projects changes + print("\n3. Projects.md changes:") + print(f" After: Contains 'Project Gamma' = {('Project Gamma' in projects_content_after)}") + print(f" After: Contains 'OpenTelemetry' = {('OpenTelemetry' in projects_content_after)}") + + print("\n✓ All content changes verified successfully") + + # ==================== STEP 10: Cleanup ==================== + print_separator("STEP 10: Cleanup") + + await reme_fs.close() + print("✓ ReMeFs closed") + + # Clean up test directory + if test_dir.exists(): + shutil.rmtree(test_dir) + print(f"✓ Removed test directory: {test_dir}") + else: + print(f"⚠️ Test directory does not exist: {test_dir}") + + print("\n✓ All test data cleaned up") + + print_separator("FILE WATCH INTEGRATION TEST - COMPLETED SUCCESSFULLY") + + +# ==================== Main Entry Point ==================== + + +async def main(): + """Run the file watch integration test.""" + print("\n" + "=" * 80) + print(" ReMeFs File Watch Integration Test") + print("=" * 80) + print("\nThis test validates the complete file watching workflow:") + print(" 1. Create markdown files with personal information") + print(" 2. Initialize ReMeFs and start file watching") + print(" 3. Verify automatic indexing into database") + print(" 4. Search and retrieve initial content") + print(" 5. Modify files and verify re-indexing") + print(" 6. Search and retrieve modified content") + print(" 7. Compare before/after results") + print("=" * 80) + + try: + await test_file_watch_integration() + + print("\n" + "=" * 80) + print(" ✓ All tests passed successfully!") + print("=" * 80) + + except Exception as e: + print("\n" + "=" * 80) + print(f" ✗ Test failed with error: {e}") + print("=" * 80) + import traceback + + traceback.print_exc() + raise + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/test_memory_store.py b/tests/test_memory_store.py index 0052049c..59dc7a99 100644 --- a/tests/test_memory_store.py +++ b/tests/test_memory_store.py @@ -39,6 +39,7 @@ class TestConfig: """Configuration for test execution.""" # SqliteMemoryStore settings + NAME = "test" SQLITE_DB_PATH = "./test_memory_store_sqlite/memory.db" SQLITE_VEC_EXT_PATH = "" # Empty string to use default vec0/sqlite_vec/vector0 SQLITE_FTS_ENABLED = True @@ -205,6 +206,7 @@ def create_memory_store(store_type: str) -> BaseMemoryStore: if store_type == "sqlite": return SqliteMemoryStore( + store_name=config.NAME, db_path=config.SQLITE_DB_PATH, embedding_model=embedding_model, vec_ext_path=config.SQLITE_VEC_EXT_PATH, @@ -235,8 +237,8 @@ async def test_start_store(store: BaseMemoryStore, _store_name: str): cursor.close() logger.info(f"Created tables: {tables}") - assert "files" in tables, "files table should exist" - assert "chunks" in tables, "chunks table should exist" + assert store.files_table_name in tables, f"{store.files_table_name} table should exist" + assert store.chunks_table_name in tables, f"{store.chunks_table_name} table should exist" logger.info("✓ Required tables created") @@ -577,6 +579,42 @@ async def test_keyword_search_with_source_filter(store: BaseMemoryStore, _store_ logger.info("\n✓ Keyword search with source filter test passed") +async def test_keyword_search_special_chars(store: BaseMemoryStore, _store_name: str): + """Test keyword search with special characters like ?, *, etc.""" + logger.info("=" * 20 + " KEYWORD SEARCH SPECIAL CHARS TEST " + "=" * 20) + + # Check if FTS is available + if isinstance(store, SqliteMemoryStore) and not store.fts_available: + logger.info("⊘ Skipped: FTS not available") + return + + # Test various queries with special characters + test_queries = [ + "What is the status?", + "How does it work?", + "Why is this important?", + "data?", + "test*", + "query with ? marks", + ] + + for query in test_queries: + logger.info(f"\nTesting query: '{query}'") + try: + results = await store.keyword_search(query, limit=3) + logger.info(f"✓ Query succeeded, found {len(results)} results") + if results: + for i, result in enumerate(results[:2], 1): # Show first 2 results + logger.info( + f" {i}. {result.path}:{result.start_line}-{result.end_line} (score: {result.score:.4f})", + ) + except Exception as e: + logger.error(f"✗ Query failed: {e}") + raise + + logger.info("\n✓ Keyword search with special characters test passed") + + async def test_delete_file(store: BaseMemoryStore, _store_name: str): """Test file deletion.""" logger.info("=" * 20 + " DELETE FILE TEST " + "=" * 20) @@ -852,6 +890,7 @@ async def run_all_tests_for_store(store_type: str, store_name: str): await test_vector_search_with_source_filter(store, store_name) await test_keyword_search(store, store_name) await test_keyword_search_with_source_filter(store, store_name) + await test_keyword_search_special_chars(store, store_name) # ========== Advanced Tests ========== logger.info(f"\n{'#' * 60}")