diff --git a/README.md b/README.md index 7c93e489..781f5412 100644 --- a/README.md +++ b/README.md @@ -506,7 +506,7 @@ async def main(): "dimensions": 1024, }, default_vector_store_config={ - "backend": "local", # Supports local/chroma/qdrant/elasticsearch + "backend": "local", # Supports local/chroma/qdrant/elasticsearch/obvec }, ) await reme.start() diff --git a/README_ZH.md b/README_ZH.md index 5fa2e4d2..7e6ccd58 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -486,7 +486,7 @@ async def main(): "dimensions": 1024, }, default_vector_store_config={ - "backend": "local", # 支持 local/chroma/qdrant/elasticsearch + "backend": "local", # 支持 local/chroma/qdrant/elasticsearch/obvec }, ) await reme.start() diff --git a/docs/README_0_2_x.md b/docs/README_0_2_x.md index e1bc85a1..e4f80b69 100644 --- a/docs/README_0_2_x.md +++ b/docs/README_0_2_x.md @@ -66,6 +66,7 @@ Agent Memory = Long-Term Memory + Short-Term Memory - **[2025-09]** 🧪 Validated effectiveness in appworld, bfcl(v3), and frozenlake ([Experiments](docs/cookbook)) - **[2025-08]** 🚀 MCP protocol support ([Quick Start](docs/mcp_quick_start.md)) - **[2025-06]** 🚀 Multiple backend vector storage (Elasticsearch & ChromaDB) ([Guide](docs/vector_store_api_guide.md)) +- **[2026-04]** 🚀 ObVec vector storage (OceanBase / seekdb via pyobvector) ([Guide](docs/vector_store_api_guide.md)) - **[2024-09]** 🧠 Personalized and time-aware memory storage --- @@ -814,7 +815,7 @@ You can find more details in [tool_bench.md](docs/tool_memory/tool_bench.md) and ### Advanced Topics - **[Operator Pipelines](reme_ai/config/default.yaml)**: Customize memory processing workflows by modifying operator chains -- **[Vector Store Backends](docs/vector_store_api_guide.md)**: Configure local, Elasticsearch, Qdrant, or ChromaDB storage +- **[Vector Store Backends](docs/vector_store_api_guide.md)**: Configure local, Elasticsearch, Qdrant, ChromaDB, or ObVec (OceanBase / seekdb) storage - **[Example Collection](./cookbook)**: Real-world use cases and best practices --- diff --git a/docs/README_0_2_x_ZH.md b/docs/README_0_2_x_ZH.md index e2b7d5c4..a3b62005 100644 --- a/docs/README_0_2_x_ZH.md +++ b/docs/README_0_2_x_ZH.md @@ -67,6 +67,7 @@ Agent Memory = Long-Term Memory + Short-Term Memory - **[2025-09]** 🧪 在 appworld、bfcl(v3)、frozenlake 等环境中验证有效性([实验文档](docs/cookbook)) - **[2025-08]** 🚀 支持 MCP 协议([快速开始](docs/mcp_quick_start.md)) - **[2025-06]** 🚀 支持多种向量存储后端(Elasticsearch & ChromaDB)([向量库指南](docs/vector_store_api_guide.md)) +- **[2026-04]** 🚀 支持 ObVec 向量存储(OceanBase / seekdb,基于 pyobvector)([向量库指南](docs/vector_store_api_guide.md)) - **[2024-09]** 🧠 支持个性化与时间敏感的记忆存储 --- @@ -820,7 +821,7 @@ Pass@K 衡量在生成 K 个候选中,至少一个成功完成任务(score=1 ### 高级主题 - **[算子管道](reme_ai/config/default.yaml)**:通过修改算子链来自定义记忆处理工作流 -- **[向量存储后端](docs/vector_store_api_guide.md)**:配置本地、Elasticsearch、Qdrant 或 ChromaDB 存储 +- **[向量存储后端](docs/vector_store_api_guide.md)**:配置本地、Elasticsearch、Qdrant、ChromaDB 或 ObVec(OceanBase / seekdb)存储 - **[案例集](./cookbook)**:真实场景的用例和最佳实践 --- diff --git a/docs/index.md b/docs/index.md index 5cbbcb56..d7e2e716 100644 --- a/docs/index.md +++ b/docs/index.md @@ -139,7 +139,7 @@ response = requests.post("http://localhost:8002/retrieve_task_memory", json={ ## 📚 Resources - **[Installation Guide](installation.md)**, **[Quick Start](quick_start.md)**: Get started quickly with practical examples -- **[Vector Storage Setup](vector_store_api_guide.md)**: Configure local/vector databases and usage +- **[Vector Storage Setup](vector_store_api_guide.md)**: Configure local, Elasticsearch, Qdrant, ChromaDB, or ObVec (OceanBase / seekdb via pyobvector) storage and usage - **[MCP Guide](mcp_quick_start.md)**: Create MCP services - **[Personal Memory](personal_memory/personal_memory.md)**, **[Task Memory](task_memory/task_memory.md)** & **[Tool Memory](tool_memory/tool_memory.md)**: Operators used in personal memory, task memory and tool memory. You can modify the config to customize the pipelines. - **[Example Collection](./cookbook/appworld/quickstart.md)**: Real use cases and best practices diff --git a/docs/vector_store_api_guide.md b/docs/vector_store_api_guide.md index df0800ef..661808e2 100644 --- a/docs/vector_store_api_guide.md +++ b/docs/vector_store_api_guide.md @@ -33,8 +33,9 @@ FlowLLM provides multiple Vector Store implementations tailored to different use - **QdrantVectorStore** ([source code](https://github.com/flowllm-ai/flowllm/blob/main/flowllm/core/vector_store/qdrant_vector_store.py)): Built on the Qdrant vector database, supporting high-performance vector search. Recommended for large-scale production environments. - **ChromaVectorStore** ([source code](https://github.com/flowllm-ai/flowllm/blob/main/flowllm/core/vector_store/chroma_vector_store.py)): Based on ChromaDB, providing persistent storage and metadata filtering capabilities. - **EsVectorStore** ([source code](https://github.com/flowllm-ai/flowllm/blob/main/flowllm/core/vector_store/es_vector_store.py)): Built on Elasticsearch, enabling powerful combined full-text and vector search functionalities. +- **ObVecVectorStore** ([source code](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/obvec_vector_store.py)): Uses [pyobvector](https://pypi.org/project/pyobvector/) against **OceanBase** or **seekdb** (MySQL-compatible wire protocol). Suitable when you already run OceanBase/seekdb or need a SQL-native vector table with HNSW-style ANN search and JSON metadata filters. -All Vector Store implementations inherit from **BaseVectorStore** ([source code](https://github.com/flowllm-ai/flowllm/blob/main/flowllm/core/vector_store/base_vector_store.py)), ensuring a consistent interface specification. +All Vector Store implementations inherit from **BaseVectorStore** ([source code](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/base_vector_store.py)) in ReMe, ensuring a consistent interface specification. ## Core Features @@ -108,6 +109,31 @@ The asynchronous interface is particularly useful in the following scenarios: - **hosts**: Elasticsearch host address(es), either a string or a list (default: `http://localhost:9200`). - **basic_auth**: Basic authentication credentials (username and password). +### ObVecVectorStore Configuration + +- **uri**: Server address as `host:port` (default: `127.0.0.1:2881`). +- **user**: MySQL-compatible user. seekdb single-tenant images often use `root`; OceanBase multi-tenant setups typically use `root@` (e.g. `root@test`). +- **password**: Database password (seekdb Docker images commonly set this via `ROOT_PASSWORD`). +- **database**: Logical database name (default: `test`). +- **index_type**: Vector index family (default: `HNSW`). +- **index_metric**: Distance metric for the vector index: `cosine`, `l2`, or `ip` (inner product); default `cosine`. +- **index_ef_search**: HNSW `ef_search` parameter passed to pyobvector (default: `100`). +- **collection_name**: Table name for the collection (from `VectorStoreConfig`, default `reme`). Use lowercase names if your deployment restricts identifiers. + +**Local seekdb via Docker** (repository root): + +```text +docker compose -f docker-compose.obvec.yml up -d +``` + +**Integration tests** (requires a running server, embedding API credentials in `.env`, and matching DB password): + +```shell +OBVEC_PASSWORD= python tests/test_vector_store.py --obvec +``` + +**Dependencies**: `pyobvector` is declared in ReMe’s `pyproject.toml`. A compatible `sqlglot` range is pinned so the pyobvector client imports cleanly. + ## Configuration File Examples Configure Vector Store in `flowllm/config/default.yaml` under the `vector_store` section. The basic structure is as follows: @@ -128,7 +154,7 @@ vector_store.default.params.= ### Configuration Field Descriptions -- **`backend`** (required): Vector store backend type. Options: `local`, `memory`, `chroma`, `qdrant`, `elasticsearch`. +- **`backend`** (required): Vector store backend type. Options: `local`, `memory`, `chroma`, `qdrant`, `elasticsearch`, `obvec`. - **`embedding_model`** (required): Name of the embedding model configuration, referencing the `embedding_model` section. - **`params`** (optional): Dictionary of backend-specific parameters passed to the vector store constructor. @@ -295,6 +321,35 @@ vector_store.default.backend=elasticsearch vector_store.default.params.hosts='["http://es-node1:9200", "http://es-node2:9200", "http://es-node3:9200"]' ``` +#### 6. ObVecVectorStore Configuration (OceanBase / seekdb) + +**Implementation**: [`reme/core/vector_store/obvec_vector_store.py`](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/obvec_vector_store.py) + +**Example (seekdb on localhost)**: + +```yaml +vector_stores: + default: + backend: obvec + embedding_model: default + collection_name: reme + uri: "127.0.0.1:2881" + user: "root" + password: "your-root-password" + database: "test" + index_metric: "cosine" + index_ef_search: 100 +``` + +```shell +vector_stores.default.backend=obvec +vector_stores.default.uri=127.0.0.1:2881 +vector_stores.default.user=root +vector_stores.default.password=your-root-password +``` + +ReMe service YAML uses the key `vector_stores` (plural); CLI overrides use the same nested paths. + ### Complete Configuration Example Below is a complete `default.yaml` example including both embedding model and vector store configurations: @@ -352,8 +407,9 @@ Two types of metadata filtering are supported: - **Development & Testing**: Use MemoryVectorStore or LocalVectorStore—no additional services required. - **Small-Scale Applications**: Use LocalVectorStore or ChromaVectorStore for simplicity and ease of use. -- **Production Environments**: Use QdrantVectorStore or EsVectorStore for high performance and scalability. +- **Production Environments**: Use QdrantVectorStore, EsVectorStore, or ObVecVectorStore (OceanBase/seekdb) for high performance and scalability, depending on your existing infrastructure. - **Hybrid Search**: Use EsVectorStore to combine vector search with full-text search capabilities. +- **OceanBase / seekdb**: Use ObVecVectorStore when you standardize on pyobvector and SQL-accessible vector tables. ## Important Notes diff --git a/reme/core/vector_store/obvec_vector_store.py b/reme/core/vector_store/obvec_vector_store.py index ac9ee6b8..69cef443 100644 --- a/reme/core/vector_store/obvec_vector_store.py +++ b/reme/core/vector_store/obvec_vector_store.py @@ -1,4 +1,4 @@ -"""OceanBase / SeekDB vector store for ReMe (pyobvector). +"""OceanBase / seekdb vector store for ReMe (pyobvector). Dense vectors, kNN via ``ObVecClient.ann_search``, JSON metadata filters, and helpers for coercion / metrics live in this module (pyobvector-aligned). @@ -139,7 +139,7 @@ def _vector_node_from_db_row(row: tuple[Any, ...]) -> VectorNode: class ObVecVectorStore(BaseVectorStore): - """OceanBase or SeekDB vector store for dense vectors and kNN search.""" + """OceanBase or seekdb vector store for dense vectors and kNN search.""" def __init__( self, diff --git a/tests/test_vector_store.py b/tests/test_vector_store.py index ff644f66..1dbad55a 100644 --- a/tests/test_vector_store.py +++ b/tests/test_vector_store.py @@ -76,7 +76,7 @@ class TestConfig: CHROMA_TENANT = None # Set for ChromaDB Cloud tenant CHROMA_DATABASE = None # Set for ChromaDB Cloud database - # ObVecVectorStore: SeekDB uses user `root`; OceanBase multi-tenant often uses `root@test`. + # ObVecVectorStore: seekdb uses user `root`; OceanBase multi-tenant often uses `root@test`. # Defaults match docker-compose.obvec.yml (ROOT_PASSWORD=root). OBVEC_URI = os.environ.get("OBVEC_URI", "127.0.0.1:2881") OBVEC_USER = os.environ.get("OBVEC_USER", "root")