From 27561aa94f302336c34557d9c55086ccff96c115 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Fri, 7 Nov 2025 14:53:14 +0800 Subject: [PATCH] docs(vector_store): update vector store configuration guide --- docs/contribution.md | 30 + docs/cookbook/experiment_overview.md | 8 +- docs/tool_memory/tool_bench.md | 10 +- docs/vector_store_api_guide.md | 851 +++++---------------------- reme_ai/__init__.py | 1 + test/config.json | 6 - test/test2.py | 18 +- test/test3.py | 11 +- test/test_update_insight_op.py | 1 - 9 files changed, 214 insertions(+), 722 deletions(-) delete mode 100644 test/config.json diff --git a/docs/contribution.md b/docs/contribution.md index fd1328b2..079dca68 100644 --- a/docs/contribution.md +++ b/docs/contribution.md @@ -22,6 +22,36 @@ git checkout -b your-feature-branch-name ### Making Changes With your new branch checked out, you can now make your changes to the code. Remember to keep your changes as focused as possible. If you're addressing multiple issues or features, it's better to create separate branches and pull requests for each. +### Set Up Pre-commit Hooks +Before committing your changes, you should set up pre-commit hooks to ensure code quality and consistency. Pre-commit hooks will automatically check your code for common issues and format it according to the project's standards. + +**Install pre-commit:** +```bash +pip install pre-commit +``` + +**Install the git hooks:** +```bash +pre-commit install +``` + +**Run pre-commit manually (optional):** +If you want to run pre-commit checks on all files before committing, you can run: +```bash +pre-commit run --all-files +``` + +After installation, pre-commit will automatically run on `git commit` to check your code. The hooks will check for: +- Code syntax and AST validation +- YAML, XML, TOML, and JSON format validation +- Trailing whitespace +- Code formatting (Black) +- Code style (Flake8) +- Code quality (Pylint) +- Package metadata (Pyroma) + +If any checks fail, please fix the issues before committing. + ### Commit Your Changes Once you've made your changes, it's time to commit them. Write clear and concise commit messages that explain your changes. ```bash diff --git a/docs/cookbook/experiment_overview.md b/docs/cookbook/experiment_overview.md index c90a2b66..a95b06e4 100644 --- a/docs/cookbook/experiment_overview.md +++ b/docs/cookbook/experiment_overview.md @@ -43,10 +43,10 @@ We tested ReMe on BFCL-V3 multi-turn-base (randomly split 50train/150val) using We evaluated Tool Memory effectiveness using a controlled benchmark with three mock search tools using Qwen3-30B-Instruct: -| Scenario | Avg Score | Improvement | -|-----------------------|-----------|--------------------| -| Train (No Memory) | 0.650 | - | -| Test (No Memory) | 0.672 | Baseline | +| Scenario | Avg Score | Improvement | +|------------------------|-----------|-------------| +| Train (No Memory) | 0.650 | - | +| Test (No Memory) | 0.672 | Baseline | | **Test (With Memory)** | **0.772** | **+14.88%** | **Key Findings:** diff --git a/docs/tool_memory/tool_bench.md b/docs/tool_memory/tool_bench.md index a1b35939..7f0fec94 100644 --- a/docs/tool_memory/tool_bench.md +++ b/docs/tool_memory/tool_bench.md @@ -24,11 +24,11 @@ This benchmark evaluates Tool Memory effectiveness by comparing agent performanc Three LLM-based mock search tools with different performance profiles: -| Tool | Simple Queries | Medium Queries | Complex Queries | -|------|---------------|----------------|-----------------| -| **SearchToolA** | ⭐⭐⭐ Fast, high success (90%) | ❌ Poor (20% success) | ⚠️ Weak (50% success) | -| **SearchToolB** | ⚠️ Over-engineered (30%) | ⭐⭐⭐ Optimal (90% success) | ⚠️ Limited (50% success) | -| **SearchToolC** | ⚠️ Overkill (30%) | ⚠️ Excessive (40%) | ⭐⭐⭐ Best (90% success) | +| Tool | Simple Queries | Medium Queries | Complex Queries | +|-----------------|------------------------------|---------------------------|--------------------------| +| **SearchToolA** | ⭐⭐⭐ Fast, high success (90%) | ❌ Poor (20% success) | ⚠️ Weak (50% success) | +| **SearchToolB** | ⚠️ Over-engineered (30%) | ⭐⭐⭐ Optimal (90% success) | ⚠️ Limited (50% success) | +| **SearchToolC** | ⚠️ Overkill (30%) | ⚠️ Excessive (40%) | ⭐⭐⭐ Best (90% success) | **Performance Characteristics:** - `success_rate`: Probability of successful execution (vs "Service busy" error) diff --git a/docs/vector_store_api_guide.md b/docs/vector_store_api_guide.md index 9917a359..e522aaf6 100644 --- a/docs/vector_store_api_guide.md +++ b/docs/vector_store_api_guide.md @@ -12,9 +12,9 @@ kernelspec: name: python3 --- -# Vector Store API Guide +# Vector Store Configuration Guide -This guide covers the vector store implementations available in ReMe, their APIs, and how to use them effectively. +This guide covers how to configure vector store backends in ReMe using the `default.yaml` configuration file. ## 📋 Overview @@ -43,71 +43,28 @@ All vector stores implement the `BaseVectorStore` interface, providing a consist | **Async Support** | ❌ No | ❌ No | ❌ No | ✅ Native | ❌ No | | **Best For** | Development | Local Apps | Production | Production/Cloud | Testing | -## 🔄 Common API Methods +## ⚙️ Configuration in default.yaml -All vector store implementations share these core methods: +All vector stores are configured in the `vector_store` section of `reme_ai/config/default.yaml`. The configuration structure is: -### 🔄 Async Support - -All vector stores provide both synchronous and asynchronous versions of every method: - -```python -# Synchronous methods -store.search(query="example", workspace_id="workspace", top_k=5) -store.insert(nodes, workspace_id="workspace") - -# Asynchronous methods (with async_ prefix) -await store.async_search(query="example", workspace_id="workspace", top_k=5) -await store.async_insert(nodes, workspace_id="workspace") +```yaml +vector_store: + default: + backend: # Required: local, chroma, elasticsearch, qdrant, or memory + embedding_model: default # Required: Name of the embedding model configuration + params: # Optional: Backend-specific parameters + # Backend-specific parameters go here ``` -### Workspace Management +### Configuration Fields -```python -# Check if workspace exists -store.exist_workspace(workspace_id: str) -> bool +- **`backend`** (required): The vector store backend to use. Valid values: `local`, `chroma`, `elasticsearch`, `qdrant`, `memory` +- **`embedding_model`** (required): The name of the embedding model configuration from the `embedding_model` section +- **`params`** (optional): A dictionary of backend-specific parameters that will be passed to the vector store constructor -# Create a new workspace -store.create_workspace(workspace_id: str, **kwargs) +## 📁 Vector Store Backend Configurations -# Delete a workspace -store.delete_workspace(workspace_id: str, **kwargs) - -# Copy a workspace -store.copy_workspace(src_workspace_id: str, dest_workspace_id: str, **kwargs) -``` - -### Data Operations - -```python -# Insert nodes (single or list) -store.insert(nodes: VectorNode | List[VectorNode], workspace_id: str, **kwargs) - -# Delete nodes by ID -store.delete(node_ids: str | List[str], workspace_id: str, **kwargs) - -# Search for similar nodes -store.search(query: str, workspace_id: str, top_k: int = 1, **kwargs) -> List[VectorNode] - -# Iterate through workspace nodes -for node in store.iter_workspace_nodes(workspace_id: str, **kwargs): - # Process each node -``` - -### Import/Export - -```python -# Export workspace to file -store.dump_workspace(workspace_id: str, path: str | Path = "", callback_fn=None, **kwargs) - -# Import workspace from file -store.load_workspace(workspace_id: str, path: str | Path = "", nodes: List[VectorNode] = None, - callback_fn=None, **kwargs) -``` - -## ⚡ Vector Store Implementations - -### 1. 📁 LocalVectorStore (`backend=local`) +### 1. LocalVectorStore (`backend=local`) A simple file-based vector store that saves data to local JSONL files. @@ -118,62 +75,22 @@ A simple file-based vector store that saves data to local JSONL files. #### ⚙️ Configuration -```python -from flowllm.storage.vector_store import LocalVectorStore -from flowllm.embedding_model import OpenAICompatibleEmbeddingModel -from flowllm.utils.common_utils import load_env - -# Load environment variables (for API keys) -load_env() - -# Initialize embedding model -embedding_model = OpenAICompatibleEmbeddingModel(dimensions=64, model_name="text-embedding-v4") - -# Initialize vector store -vector_store = LocalVectorStore( - embedding_model=embedding_model, - store_dir="./file_vector_store", # Directory to store JSONL files - batch_size=1024 # Batch size for operations -) +```yaml +vector_store: + default: + backend: local + embedding_model: default + params: + store_dir: "./local_vector_store" # Directory to store JSONL files (default: "./local_vector_store") + batch_size: 1024 # Batch size for operations (default: 1024) ``` -#### 💻 Example Usage +#### Configuration Parameters -```python -from flowllm.schema.vector_node import VectorNode +- **`store_dir`** (optional): Directory path where workspace files are stored. Default: `"./local_vector_store"` +- **`batch_size`** (optional): Batch size for bulk operations. Default: `1024` -# Create workspace -workspace_id = "my_workspace" -vector_store.create_workspace(workspace_id) - -# Create nodes -nodes = [ - VectorNode( - unique_id="node1", - workspace_id=workspace_id, - content="Artificial intelligence is revolutionizing technology", - metadata={"category": "tech", "source": "article1"} - ), - VectorNode( - unique_id="node2", - workspace_id=workspace_id, - content="Machine learning enables data-driven insights", - metadata={"category": "tech", "source": "article2"} - ) -] - -# Insert nodes -vector_store.insert(nodes, workspace_id) - -# Search -results = vector_store.search("What is AI?", workspace_id, top_k=2) -for result in results: - print(f"Content: {result.content}") - print(f"Metadata: {result.metadata}") - print(f"Score: {result.metadata.get('score', 'N/A')}") -``` - -### 2. 🔮 ChromaVectorStore (`backend=chroma`) +### 2. ChromaVectorStore (`backend=chroma`) An embedded vector database that provides persistent storage with advanced features. @@ -184,71 +101,22 @@ An embedded vector database that provides persistent storage with advanced featu #### ⚙️ Configuration -```python -from flowllm.storage.vector_store import ChromaVectorStore -from flowllm.embedding_model import OpenAICompatibleEmbeddingModel -from flowllm.utils.common_utils import load_env - -# Load environment variables -load_env() - -# Initialize embedding model -embedding_model = OpenAICompatibleEmbeddingModel(dimensions=64, model_name="text-embedding-v4") - -# Initialize vector store -vector_store = ChromaVectorStore( - embedding_model=embedding_model, - store_dir="./chroma_vector_store", # Directory for Chroma database - batch_size=1024 # Batch size for operations -) +```yaml +vector_store: + default: + backend: chroma + embedding_model: default + params: + store_dir: "./chroma_vector_store" # Directory for Chroma database (default: "./chroma_vector_store") + batch_size: 1024 # Batch size for operations (default: 1024) ``` -#### 💻 Example Usage +#### Configuration Parameters -```python -from flowllm.schema.vector_node import VectorNode +- **`store_dir`** (optional): Directory path where ChromaDB data is persisted. Default: `"./chroma_vector_store"` +- **`batch_size`** (optional): Batch size for bulk operations. Default: `1024` -workspace_id = "chroma_workspace" - -# Check if workspace exists and create if needed -if not vector_store.exist_workspace(workspace_id): - vector_store.create_workspace(workspace_id) - -# Create nodes with metadata -nodes = [ - VectorNode( - unique_id="node1", - workspace_id=workspace_id, - content="Deep learning models require large datasets", - metadata={ - "category": "AI", - "difficulty": "advanced", - "topic": "deep_learning" - } - ), - VectorNode( - unique_id="node2", - workspace_id=workspace_id, - content="Transformer architecture revolutionized NLP", - metadata={ - "category": "AI", - "difficulty": "intermediate", - "topic": "transformers" - } - ) -] - -# Insert nodes -vector_store.insert(nodes, workspace_id) - -# Search -results = vector_store.search("deep learning", workspace_id, top_k=5) -for result in results: - print(f"Content: {result.content}") - print(f"Metadata: {result.metadata}") -``` - -### 3. 🔍 EsVectorStore (`backend=elasticsearch`) +### 3. EsVectorStore (`backend=elasticsearch`) Production-grade vector search using Elasticsearch with advanced filtering and scaling capabilities. @@ -282,114 +150,24 @@ export FLOW_ES_HOSTS=http://localhost:9200 #### ⚙️ Configuration -```python -from flowllm.storage.vector_store import EsVectorStore -from flowllm.embedding_model import OpenAICompatibleEmbeddingModel -from flowllm.utils.common_utils import load_env -import os - -# Load environment variables -load_env() - -# Initialize embedding model -embedding_model = OpenAICompatibleEmbeddingModel(dimensions=64, model_name="text-embedding-v4") - -# Initialize vector store -vector_store = EsVectorStore( - embedding_model=embedding_model, - hosts=os.getenv("FLOW_ES_HOSTS", "http://localhost:9200"), # Elasticsearch hosts - basic_auth=None, # ("username", "password") for auth - batch_size=1024 # Batch size for bulk operations -) +```yaml +vector_store: + default: + backend: elasticsearch + embedding_model: default + params: + hosts: "http://localhost:9200" # Elasticsearch host(s) - can be string or list (default: from FLOW_ES_HOSTS env var or "http://localhost:9200") + basic_auth: null # Optional: ("username", "password") tuple for authentication + batch_size: 1024 # Batch size for bulk operations (default: 1024) ``` -#### 🎯 Advanced Filtering +#### Configuration Parameters -EsVectorStore supports advanced filtering capabilities through the `filter_dict` parameter: +- **`hosts`** (optional): Elasticsearch host(s) as a string or list of strings. Defaults to the `FLOW_ES_HOSTS` environment variable or `"http://localhost:9200"` if not set +- **`basic_auth`** (optional): Tuple of `("username", "password")` for basic authentication. Default: `null` (no authentication) +- **`batch_size`** (optional): Batch size for bulk operations. Default: `1024` -```python -# Term filters (exact match) -term_filter = { - "category": "technology", - "author": "research_team" -} - -# Range filters (numeric and date ranges) -range_filter = { - "score": {"gte": 0.8}, # Score >= 0.8 - "confidence": {"gte": 0.5, "lte": 0.9}, # Between 0.5 and 0.9 - "timestamp": {"gte": "2024-01-01", "lte": "2024-12-31"} -} - -# Combined filters (filters are combined with AND logic) -combined_filter = { - "category": "AI", - "confidence": {"gte": 0.9} -} - -# Search with filters applied -results = vector_store.search("machine learning", workspace_id, top_k=10, filter_dict=combined_filter) -``` - -#### ⚡ Performance Optimization - -```python -# Refresh index for immediate availability (useful after bulk inserts) -vector_store.insert(nodes, workspace_id, refresh=True) # Auto-refresh -vector_store.refresh(workspace_id) # Manual refresh - -# Bulk operations with custom batch size -vector_store.insert(large_node_list, workspace_id, refresh=False) # Skip refresh for speed -vector_store.refresh(workspace_id) # Refresh once after all inserts -``` - -#### 💻 Example Usage - -```python -from flowllm.schema.vector_node import VectorNode - -# Define workspace -workspace_id = "production_workspace" - -# Create workspace if needed -if not vector_store.exist_workspace(workspace_id): - vector_store.create_workspace(workspace_id) - -# Create nodes with rich metadata -nodes = [ - VectorNode( - unique_id="doc1", - workspace_id=workspace_id, - content="Transformer architecture revolutionized NLP", - metadata={ - "category": "AI", - "subcategory": "NLP", - "author": "research_team", - "timestamp": "2024-01-15", - "confidence": 0.95, - "tags": ["transformer", "nlp", "attention"] - } - ) -] - -# Insert with refresh for immediate availability -vector_store.insert(nodes, workspace_id, refresh=True) - -# Advanced search with filters -filter_dict = { - "category": "AI", - "confidence": {"gte": 0.9} -} - -results = vector_store.search("transformer models", workspace_id, top_k=5, filter_dict=filter_dict) - -for result in results: - print(f"Score: {result.metadata.get('score', 'N/A')}") - print(f"Content: {result.content}") - print(f"Metadata: {result.metadata}") -``` - -### 4. 🎯 QdrantVectorStore (`backend=qdrant`) +### 4. QdrantVectorStore (`backend=qdrant`) A high-performance vector database designed for production workloads with native async support and advanced filtering. @@ -430,284 +208,40 @@ export FLOW_QDRANT_API_KEY=your-api-key #### ⚙️ Configuration -```python -from flowllm.storage.vector_store import QdrantVectorStore -from flowllm.embedding_model import OpenAICompatibleEmbeddingModel -from flowllm.utils.common_utils import load_env -import os - -# Load environment variables -load_env() - -# Initialize embedding model -embedding_model = OpenAICompatibleEmbeddingModel(dimensions=64, model_name="text-embedding-v4") - -# Option 1: Use localhost with environment variables -vector_store = QdrantVectorStore( - embedding_model=embedding_model, - host=os.getenv("FLOW_QDRANT_HOST", "localhost"), - port=int(os.getenv("FLOW_QDRANT_PORT", "6333")), - batch_size=1024 -) - -# Option 2: Use URL (for Qdrant Cloud or remote servers) -vector_store = QdrantVectorStore( - embedding_model=embedding_model, - url="http://your-qdrant-server:6333", - api_key="your-api-key", # Optional, for cloud - batch_size=1024 -) - -# Option 3: Specify custom distance metric -from qdrant_client.http.models import Distance - -vector_store = QdrantVectorStore( - embedding_model=embedding_model, - host="localhost", - port=6333, - distance=Distance.COSINE, # or Distance.EUCLIDEAN, Distance.DOT - batch_size=1024 -) +##### Local Qdrant Instance +```yaml +vector_store: + default: + backend: qdrant + embedding_model: default + params: + host: "localhost" # Qdrant host (default: from FLOW_QDRANT_HOST env var or "localhost") + port: 6333 # Qdrant port (default: from FLOW_QDRANT_PORT env var or 6333) + batch_size: 1024 # Batch size for operations (default: 1024) + distance: "COSINE" # Distance metric: "COSINE", "EUCLIDEAN", or "DOT" (default: "COSINE") ``` -#### 🎯 Advanced Filtering - -QdrantVectorStore supports advanced filtering capabilities similar to Elasticsearch: - -```python -# Term filters (exact match) -term_filter = { - "category": "AI", - "node_type": "research" -} - -# Range filters (numeric) -range_filter = { - "confidence": {"gte": 0.8, "lte": 1.0}, # Between 0.8 and 1.0 - "score": {"gt": 0.5} # Greater than 0.5 -} - -# Combined filters (all conditions must match - AND logic) -combined_filter = { - "category": "AI", - "confidence": {"gte": 0.9}, - "node_type": "research" -} - -# Search with filters -results = vector_store.search( - query="machine learning", - workspace_id=workspace_id, - top_k=10, - filter_dict=combined_filter -) +##### Qdrant Cloud or Remote Server +```yaml +vector_store: + default: + backend: qdrant + embedding_model: default + params: + url: "https://your-cluster.qdrant.io:6333" # Qdrant server URL (if provided, host and port are ignored) + api_key: "your-api-key" # API key for Qdrant Cloud authentication + batch_size: 1024 # Batch size for operations (default: 1024) + distance: "COSINE" # Distance metric (default: "COSINE") ``` -##### Filter Operations Supported: -- **Exact match**: `{"field": "value"}` -- **Range queries**: - - `gte`: Greater than or equal - - `lte`: Less than or equal - - `gt`: Greater than - - `lt`: Less than +#### Configuration Parameters -#### ⚡ Async Operations - -QdrantVectorStore provides **native async support** for all operations: - -```python -import asyncio - -async def main(): - # All operations have async equivalents - - # Check if workspace exists - exists = await vector_store.async_exist_workspace(workspace_id) - - # Create workspace - if not exists: - await vector_store.async_create_workspace(workspace_id) - - # Insert nodes with async embedding - await vector_store.async_insert(nodes, workspace_id) - - # Search with async embedding - results = await vector_store.async_search( - query="AI research", - workspace_id=workspace_id, - top_k=5, - filter_dict={"category": "AI"} - ) - - # Delete nodes - await vector_store.async_delete(node_ids, workspace_id) - - # Delete workspace - await vector_store.async_delete_workspace(workspace_id) - - # Close client - await vector_store.async_close() - -# Run async operations -asyncio.run(main()) -``` - -#### 💻 Example Usage - -```python -from flowllm.schema.vector_node import VectorNode - -workspace_id = "qdrant_workspace" - -# Check and create workspace -if not vector_store.exist_workspace(workspace_id): - vector_store.create_workspace(workspace_id) - -# Create nodes with rich metadata -nodes = [ - VectorNode( - unique_id="node1", - workspace_id=workspace_id, - content="Artificial intelligence is revolutionizing technology", - metadata={ - "category": "AI", - "node_type": "research", - "confidence": 0.95, - "author": "research_team" - } - ), - VectorNode( - unique_id="node2", - workspace_id=workspace_id, - content="Machine learning models require large datasets", - metadata={ - "category": "AI", - "node_type": "tutorial", - "confidence": 0.85, - "author": "education_team" - } - ), - VectorNode( - unique_id="node3", - workspace_id=workspace_id, - content="Deep learning excels at image recognition", - metadata={ - "category": "AI", - "node_type": "research", - "confidence": 0.92, - "author": "research_team" - } - ) -] - -# Insert nodes (upsert - creates or updates) -vector_store.insert(nodes, workspace_id) - -# Simple search -results = vector_store.search("What is AI?", workspace_id, top_k=3) -for result in results: - print(f"Content: {result.content}") - print(f"Score: {result.metadata.get('score', 'N/A')}") - print(f"Metadata: {result.metadata}") - print("-" * 50) - -# Advanced search with filters -filter_dict = { - "node_type": "research", - "confidence": {"gte": 0.9} -} - -filtered_results = vector_store.search( - query="AI technology", - workspace_id=workspace_id, - top_k=5, - filter_dict=filter_dict -) - -print(f"Found {len(filtered_results)} filtered results") -for result in filtered_results: - print(f"Content: {result.content}") - print(f"Metadata: {result.metadata}") - -# Iterate through all nodes -print("\nAll nodes in workspace:") -for node in vector_store.iter_workspace_nodes(workspace_id, limit=100): - print(f"ID: {node.unique_id}, Content: {node.content[:50]}...") - -# Update a node (delete + insert) -updated_node = VectorNode( - unique_id="node1", - workspace_id=workspace_id, - content="Artificial intelligence is transforming industries worldwide", - metadata={ - "category": "AI", - "node_type": "research", - "confidence": 0.98, - "author": "research_team", - "updated": True - } -) -vector_store.delete("node1", workspace_id) -vector_store.insert(updated_node, workspace_id) - -# Export workspace for backup -vector_store.dump_workspace(workspace_id, path="./qdrant_backup") - -# Clean up -vector_store.close() -``` - -#### 🔄 Async Example - -```python -import asyncio -from flowllm.schema.vector_node import VectorNode - -async def async_example(): - workspace_id = "async_qdrant_workspace" - - # Create workspace - if not await vector_store.async_exist_workspace(workspace_id): - await vector_store.async_create_workspace(workspace_id) - - # Create nodes - nodes = [ - VectorNode( - unique_id="async_node1", - workspace_id=workspace_id, - content="Async operations enable better performance", - metadata={"type": "performance", "async": True} - ), - VectorNode( - unique_id="async_node2", - workspace_id=workspace_id, - content="Concurrent requests improve throughput", - metadata={"type": "performance", "async": True} - ) - ] - - # Insert with async embedding - await vector_store.async_insert(nodes, workspace_id) - - # Search with async embedding - results = await vector_store.async_search( - query="performance optimization", - workspace_id=workspace_id, - top_k=2, - filter_dict={"async": True} - ) - - for result in results: - print(f"Score: {result.metadata['score']:.4f}") - print(f"Content: {result.content}") - - # Cleanup - await vector_store.async_delete_workspace(workspace_id) - await vector_store.async_close() - -# Run async example -asyncio.run(async_example()) -``` +- **`url`** (optional): Complete URL for connecting to Qdrant. If provided, `host` and `port` are ignored. Useful for Qdrant Cloud or custom deployments +- **`host`** (optional): Host address of the Qdrant server. Defaults to the `FLOW_QDRANT_HOST` environment variable or `"localhost"` if not set +- **`port`** (optional): Port number of the Qdrant server. Defaults to the `FLOW_QDRANT_PORT` environment variable or `6333` if not set +- **`api_key`** (optional): API key for authentication (required for Qdrant Cloud). Can also be set via `FLOW_QDRANT_API_KEY` environment variable +- **`distance`** (optional): Distance metric for vector similarity. Valid values: `"COSINE"`, `"EUCLIDEAN"`, `"DOT"`. Default: `"COSINE"` +- **`batch_size`** (optional): Batch size for bulk operations. Default: `1024` #### 🌟 Key Features @@ -720,23 +254,7 @@ asyncio.run(async_example()) - **Persistent Storage** - Data is automatically persisted to disk - **Efficient Iteration** - Scroll through large collections with pagination -#### 🚨 Important Notes - -- **Collection = Workspace** - Qdrant uses "collections" which map to workspace_id -- **Automatic Embedding** - Nodes without vectors are automatically embedded -- **ID-based Upsert** - Using the same unique_id will update existing nodes -- **Metadata Indexing** - All metadata fields are automatically indexed for filtering -- **Connection Management** - Call `close()` or `async_close()` to cleanup connections - -#### 📊 Performance Tips - -1. **Batch Operations** - Insert multiple nodes at once for better performance -2. **Use Async** - For high-concurrency scenarios, use async methods -3. **Optimize Filters** - Use indexed metadata fields for faster filtering -4. **Pagination** - Use `iter_workspace_nodes()` with appropriate `limit` for large collections -5. **Distance Metric** - Choose appropriate distance metric for your use case (COSINE for normalized vectors) - -### 5. ⚡ MemoryVectorStore (`backend=memory`) +### 5. MemoryVectorStore (`backend=memory`) An ultra-fast in-memory vector store that keeps all data in RAM for maximum performance. @@ -748,74 +266,20 @@ An ultra-fast in-memory vector store that keeps all data in RAM for maximum perf #### ⚙️ Configuration -```python -from flowllm.storage.vector_store import MemoryVectorStore -from flowllm.embedding_model import OpenAICompatibleEmbeddingModel -from flowllm.utils.common_utils import load_env - -# Load environment variables -load_env() - -# Initialize embedding model -embedding_model = OpenAICompatibleEmbeddingModel(dimensions=64, model_name="text-embedding-v4") - -# Initialize vector store -vector_store = MemoryVectorStore( - embedding_model=embedding_model, - store_dir="./memory_vector_store", # Directory for backup/restore operations - batch_size=1024 # Batch size for operations -) +```yaml +vector_store: + default: + backend: memory + embedding_model: default + params: + store_dir: "./memory_vector_store" # Directory for backup/restore operations (default: "./memory_vector_store") + batch_size: 1024 # Batch size for operations (default: 1024) ``` -#### 💻 Example Usage +#### Configuration Parameters -```python -from flowllm.schema.vector_node import VectorNode - -workspace_id = "memory_workspace" - -# Create workspace in memory -vector_store.create_workspace(workspace_id) - -# Create nodes -nodes = [ - VectorNode( - unique_id="mem_node1", - workspace_id=workspace_id, - content="Memory stores provide ultra-fast access to data", - metadata={ - "category": "performance", - "type": "memory", - "speed": "ultra_fast" - } - ), - VectorNode( - unique_id="mem_node2", - workspace_id=workspace_id, - content="In-memory databases excel at low-latency operations", - metadata={ - "category": "performance", - "type": "database", - "latency": "low" - } - ) -] - -# Insert nodes (stored in memory) -vector_store.insert(nodes, workspace_id) - -# Ultra-fast search -results = vector_store.search("fast memory access", workspace_id, top_k=2) -for result in results: - print(f"Content: {result.content}") - print(f"Score: {result.metadata.get('score', 'N/A')}") - -# Optional: Save to disk for backup -vector_store.dump_workspace(workspace_id, path="./backup") - -# Optional: Load from disk to memory -vector_store.load_workspace(workspace_id, path="./backup") -``` +- **`store_dir`** (optional): Directory path for backup/restore operations. Default: `"./memory_vector_store"` +- **`batch_size`** (optional): Batch size for bulk operations. Default: `1024` #### ⚡ Performance Benefits @@ -831,69 +295,80 @@ vector_store.load_workspace(workspace_id, path="./backup") - **No persistence** - Use `dump_workspace()` to save to disk - **Single process** - Not suitable for distributed applications -## 📝 Working with VectorNode +## 📝 Example Configurations -The `VectorNode` class is the fundamental data unit for all vector stores: - -```python -from flowllm.schema.vector_node import VectorNode - -# Create a node -node = VectorNode( - unique_id="unique_identifier", # Unique ID for the node (required) - workspace_id="my_workspace", # Workspace ID (required) - content="Text content to embed", # Content to be embedded (required) - metadata={ # Optional metadata - "source": "document1", - "category": "technology", - "timestamp": "2024-08-29" - }, - vector=None # Vector will be generated automatically if None -) +### Minimal Configuration (Memory Store) +```yaml +vector_store: + default: + backend: memory + embedding_model: default ``` -## 🔄 Import/Export Example - -Export and import workspaces for backup or transfer: - -```python -# Export workspace to file -vector_store.dump_workspace( - workspace_id="my_workspace", - path="./backup_data" # Directory to store the exported data -) - -# Import workspace from file -vector_store.load_workspace( - workspace_id="new_workspace", - path="./backup_data" # Directory containing the exported data -) - -# Copy workspace within the same store -vector_store.copy_workspace( - src_workspace_id="original_workspace", - dest_workspace_id="copied_workspace" -) +### Local File Storage +```yaml +vector_store: + default: + backend: local + embedding_model: default + params: + store_dir: "./my_vector_store" + batch_size: 2048 ``` +### Elasticsearch Production Setup +```yaml +vector_store: + default: + backend: elasticsearch + embedding_model: default + params: + hosts: "http://elasticsearch.example.com:9200" + basic_auth: ["username", "password"] + batch_size: 2048 +``` + +### Qdrant Cloud Setup +```yaml +vector_store: + default: + backend: qdrant + embedding_model: default + params: + url: "https://your-cluster.qdrant.io:6333" + api_key: "your-api-key-here" + distance: "COSINE" + batch_size: 1024 +``` + +## 🔄 Environment Variables + +Some vector store backends support environment variables for configuration: + +- **Elasticsearch**: `FLOW_ES_HOSTS` - Elasticsearch host(s) +- **Qdrant**: + - `FLOW_QDRANT_HOST` - Qdrant host (default: "localhost") + - `FLOW_QDRANT_PORT` - Qdrant port (default: 6333) + - `FLOW_QDRANT_API_KEY` - Qdrant API key for authentication + +Environment variables are used as fallbacks when parameters are not explicitly set in the YAML configuration. + ## 🧩 Integration with Embedding Models -All vector stores require an embedding model to function: +All vector stores require an embedding model configuration. The `embedding_model` field in the vector store configuration references a model defined in the `embedding_model` section of `default.yaml`: -```python -from flowllm.embedding_model import OpenAICompatibleEmbeddingModel +```yaml +embedding_model: + default: + backend: openai_compatible + model_name: text-embedding-v4 + params: + dimensions: 1024 -# Initialize embedding model -embedding_model = OpenAICompatibleEmbeddingModel( - dimensions=64, # Embedding dimensions - model_name="text-embedding-v4", # Model name - batch_size=32 # Batch size for embedding generation -) - -# Pass to vector store (example with LocalVectorStore) -# You can also use: ChromaVectorStore, EsVectorStore, QdrantVectorStore, or MemoryVectorStore -vector_store = LocalVectorStore( - embedding_model=embedding_model, - store_dir="./vector_store" -) +vector_store: + default: + backend: memory + embedding_model: default # References the embedding_model.default configuration ``` + +The embedding model configuration provides the model name, backend, and parameters needed for generating vector embeddings. diff --git a/reme_ai/__init__.py b/reme_ai/__init__.py index 500d7a3c..62450870 100644 --- a/reme_ai/__init__.py +++ b/reme_ai/__init__.py @@ -15,6 +15,7 @@ from . import service # noqa: E402 from . import summary # noqa: E402 from . import utils # noqa: E402 from . import vector_store # noqa: E402 +from .main import ReMeApp # noqa: E402 F401 __all__ = [ "agent", diff --git a/test/config.json b/test/config.json deleted file mode 100644 index d1fb68f8..00000000 --- a/test/config.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "stock_code": "002027", - "min_pages": 1, - "download_dir": "reports_pdf", - "years_ago": 5 -} \ No newline at end of file diff --git a/test/test2.py b/test/test2.py index c0a2969d..acd329c3 100644 --- a/test/test2.py +++ b/test/test2.py @@ -1,15 +1,15 @@ -import random - -import requests import json -import re import os -from urllib.parse import urljoin -from time import sleep -import pycurl -from io import BytesIO -from PyPDF2 import PdfReader +import random +import re from datetime import datetime, timedelta +from io import BytesIO +from time import sleep +from urllib.parse import urljoin + +import pycurl +import requests +from PyPDF2 import PdfReader # 全局配置 BASE_URL = "https://reportapi.eastmoney.com/report/list" diff --git a/test/test3.py b/test/test3.py index a143a79b..2b0021a8 100644 --- a/test/test3.py +++ b/test/test3.py @@ -1,15 +1,8 @@ -import random - -import requests -import json -import re import os -from urllib.parse import urljoin -from time import sleep -import pycurl from io import BytesIO + +import pycurl from PyPDF2 import PdfReader -from datetime import datetime, timedelta DOWNLOAD_DIR = "./" diff --git a/test/test_update_insight_op.py b/test/test_update_insight_op.py index e8583441..6505b6a4 100644 --- a/test/test_update_insight_op.py +++ b/test/test_update_insight_op.py @@ -5,7 +5,6 @@ This is a basic validation test to ensure the class structure is correct. """ import sys -import os sys.path.append("/Users/yuli/workspace/MemoryScope")