mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-10-23 05:35:30 +00:00
refactor: simplify core, Studio, and unit test scaffolding (#658)
* refactor: share backend search, file responses, and vector digests * refactor(studio): remove unused scaffolding and share workspace updates * test: simplify async tests and share deterministic store fixtures
This commit is contained in:
parent
925ef0dad1
commit
ff2956a41f
78 changed files with 4171 additions and 7348 deletions
2
.github/workflows/_build-docs.yml
vendored
2
.github/workflows/_build-docs.yml
vendored
|
|
@ -67,7 +67,7 @@ jobs:
|
|||
- name: Verify browser demo bundle
|
||||
if: inputs.run_tests
|
||||
working-directory: reme_studio
|
||||
run: node --test tests/demo-build.test.mjs
|
||||
run: node --test tests/distribution/demo-build.test.mjs
|
||||
|
||||
- name: Configure Pages
|
||||
if: inputs.upload_pages_artifact
|
||||
|
|
|
|||
2
.github/workflows/ci-reme-studio.yml
vendored
2
.github/workflows/ci-reme-studio.yml
vendored
|
|
@ -66,7 +66,7 @@ jobs:
|
|||
|
||||
- name: Verify npm package
|
||||
run: |
|
||||
npm pack --pack-destination "${RUNNER_TEMP}"
|
||||
npm pack --ignore-scripts --pack-destination "${RUNNER_TEMP}"
|
||||
tar -tzf "${RUNNER_TEMP}"/agentscope-ai-reme_studio-*.tgz | grep '^package/dist-static/index.html$'
|
||||
|
||||
- name: Set up Python
|
||||
|
|
|
|||
|
|
@ -14,4 +14,3 @@ LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
|
|||
# Optional data-source mirrors. Unset variables use the official services.
|
||||
# HF_MIRROR_URL=https://hf-mirror.com
|
||||
# ARXIV_MIRROR_URL=https://export.arxiv.org
|
||||
# TUSHARE_MIRROR_URL=http://112.124.63.173:4000/tushare
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
"""FAISS-backed file store: chunk JSONL stays authoritative; FAISS HNSW replaces the linear vector scan."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from contextlib import suppress
|
||||
|
|
@ -389,28 +388,6 @@ class FaissLocalFileStore(LocalFileStore):
|
|||
|
||||
# -- persistence ------------------------------------------------------
|
||||
|
||||
def _chunks_embedding_digest(self) -> str:
|
||||
"""Order-independent digest of the live (chunk_id, embedding) set.
|
||||
|
||||
Hashes float16 canonical bytes (the chunk JSONL serialization dtype) so
|
||||
the value is identical whether an embedding sits fresh in memory
|
||||
(possibly float32 from the provider; assignment bypasses the pydantic
|
||||
validator) or has round-tripped through the JSONL. Written into the
|
||||
idmap sidecar at dump time and recomputed from ``self.file_chunks`` at
|
||||
load time: a mismatch means the sidecar vectors belong to a different
|
||||
chunk generation than the authoritative JSONL, which the live-id set
|
||||
check alone cannot detect for same-ID in-place updates.
|
||||
"""
|
||||
digest = hashlib.sha256()
|
||||
eligible = sorted(
|
||||
cid for cid, chunk in self.file_chunks.items() if self._embedding_dim_matches(chunk.embedding)
|
||||
)
|
||||
for cid in eligible:
|
||||
digest.update(cid.encode("utf-8"))
|
||||
digest.update(b"\x00")
|
||||
digest.update(np.asarray(self.file_chunks[cid].embedding, dtype=np.float16).tobytes())
|
||||
return digest.hexdigest()
|
||||
|
||||
async def load(self) -> None:
|
||||
"""Load chunks via the parent, then attach FAISS state (sidecar or rebuild)."""
|
||||
await super().load()
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
import asyncio
|
||||
import base64
|
||||
import datetime
|
||||
import hashlib
|
||||
import heapq
|
||||
import json
|
||||
import time
|
||||
|
|
@ -107,6 +108,23 @@ class LocalFileStore(BaseFileStore):
|
|||
return 0
|
||||
return max(0, int(getattr(self.embedding_store, "dimensions", 0)))
|
||||
|
||||
def _chunks_embedding_digest(self) -> str:
|
||||
"""Order-independent digest of the live (chunk_id, embedding) set.
|
||||
|
||||
Float16 bytes match the authoritative chunk JSONL serialization even
|
||||
before a fresh provider vector has round-tripped through persistence.
|
||||
Vector backends use this digest to reject stale index sidecars.
|
||||
"""
|
||||
digest = hashlib.sha256()
|
||||
eligible = sorted(
|
||||
cid for cid, chunk in self.file_chunks.items() if self._embedding_dim_matches(chunk.embedding)
|
||||
)
|
||||
for cid in eligible:
|
||||
digest.update(cid.encode("utf-8"))
|
||||
digest.update(b"\x00")
|
||||
digest.update(np.asarray(self.file_chunks[cid].embedding, dtype=np.float16).tobytes())
|
||||
return digest.hexdigest()
|
||||
|
||||
# -- lifecycle ------------------------------------------------------------
|
||||
|
||||
async def _start(self) -> None:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
"""Zvec-backed file store: chunk JSONL stays authoritative; a zvec collection replaces the linear vector scan."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import time
|
||||
|
|
@ -227,26 +226,6 @@ class ZvecLocalFileStore(LocalFileStore):
|
|||
|
||||
# -- persistence ------------------------------------------------------
|
||||
|
||||
def _chunks_embedding_digest(self) -> str:
|
||||
"""Order-independent digest of the live (chunk_id, embedding) set.
|
||||
|
||||
Hashes float16 canonical bytes (the chunk JSONL serialization dtype) so
|
||||
the value is identical whether an embedding sits fresh in memory or has
|
||||
round-tripped through the JSONL. Written into the sidecar at dump time
|
||||
and recomputed from ``self.file_chunks`` at load time: a mismatch means
|
||||
the collection belongs to a different chunk generation than the
|
||||
authoritative JSONL.
|
||||
"""
|
||||
digest = hashlib.sha256()
|
||||
eligible = sorted(
|
||||
cid for cid, chunk in self.file_chunks.items() if self._embedding_dim_matches(chunk.embedding)
|
||||
)
|
||||
for cid in eligible:
|
||||
digest.update(cid.encode("utf-8"))
|
||||
digest.update(b"\x00")
|
||||
digest.update(np.asarray(self.file_chunks[cid].embedding, dtype=np.float16).tobytes())
|
||||
return digest.hexdigest()
|
||||
|
||||
async def load(self) -> None:
|
||||
"""Load chunks via the parent, then attach the zvec collection (open or rebuild)."""
|
||||
await super().load()
|
||||
|
|
|
|||
13
reme/steps/file_io/_base.py
Normal file
13
reme/steps/file_io/_base.py
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
"""Common response handling for file operations."""
|
||||
|
||||
from ..base_step import BaseStep
|
||||
|
||||
|
||||
class _FileIOStep(BaseStep):
|
||||
"""Preserve the file-operation failure envelope without changing its payload."""
|
||||
|
||||
def _fail(self, message: str, **meta) -> None:
|
||||
assert self.context is not None
|
||||
self.context.response.success = False
|
||||
self.context.response.answer = f"Error: {message}"
|
||||
self.context.response.metadata.update(meta)
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
"""Write a daily markdown note by dispatching the generic ``write`` step."""
|
||||
|
||||
from ._base import _FileIOStep
|
||||
from ._daily_index import parse_daily_date, refresh_day_index, validate_session_id
|
||||
from ._path import validate_filename_component
|
||||
from ..base_step import BaseStep
|
||||
from ..index import normalize_posix_path
|
||||
from ...components import R
|
||||
from ...steps.evolve import now
|
||||
|
|
@ -11,16 +11,9 @@ _RESERVED_METADATA_KEYS = {"name", "description", "session_id", "source_conversa
|
|||
|
||||
|
||||
@R.register("daily_write_step")
|
||||
class DailyWriteStep(BaseStep):
|
||||
class DailyWriteStep(_FileIOStep):
|
||||
"""Write ``daily/<date>/<name>.md`` with conversation frontmatter."""
|
||||
|
||||
def _fail(self, message: str, **meta) -> None:
|
||||
assert self.context is not None
|
||||
self.context.response.success = False
|
||||
self.context.response.answer = f"Error: {message}"
|
||||
if meta:
|
||||
self.context.response.metadata.update(meta)
|
||||
|
||||
def _session_dir(self) -> str:
|
||||
return normalize_posix_path(str(self.config_value("session_dir")))
|
||||
|
||||
|
|
|
|||
|
|
@ -3,14 +3,14 @@
|
|||
import frontmatter
|
||||
import yaml
|
||||
|
||||
from ._base import _FileIOStep
|
||||
from ._file_io import get_path_lock, read_file_safe, write_file_safe
|
||||
from ._path import _check_path_permission, NON_MD_WARNING, gate_md, resolve_path
|
||||
from ..base_step import BaseStep
|
||||
from ...components import R
|
||||
|
||||
|
||||
@R.register("edit_step")
|
||||
class EditStep(BaseStep):
|
||||
class EditStep(_FileIOStep):
|
||||
"""Replace every occurrence of ``old`` with ``new`` inside the file body.
|
||||
|
||||
The YAML front matter block (if any) is parsed out, kept verbatim and
|
||||
|
|
@ -24,13 +24,6 @@ class EditStep(BaseStep):
|
|||
read-modify-write cycle against the same file (multi-worker / multi-
|
||||
process safety is out of scope)."""
|
||||
|
||||
def _fail(self, message: str, **meta) -> None:
|
||||
assert self.context is not None
|
||||
self.context.response.success = False
|
||||
self.context.response.answer = f"Error: {message}"
|
||||
if meta:
|
||||
self.context.response.metadata.update(meta)
|
||||
|
||||
async def execute(self): # pylint: disable=too-many-return-statements
|
||||
assert self.context is not None
|
||||
raw = str(self.context.get("path") or "")
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ from pathlib import Path
|
|||
from stat import S_ISREG
|
||||
from typing import Iterable
|
||||
|
||||
from ._base import _FileIOStep
|
||||
from ._path import resolve_path
|
||||
from ..base_step import BaseStep
|
||||
from ...components import R
|
||||
|
||||
# Default cap on returned items so huge workspaces don't blow up the response.
|
||||
|
|
@ -28,17 +28,9 @@ DEFAULT_LIMIT = 100
|
|||
|
||||
|
||||
@R.register("list_step")
|
||||
class ListStep(BaseStep):
|
||||
class ListStep(_FileIOStep):
|
||||
"""Enumerate files under a directory in the workspace."""
|
||||
|
||||
def _fail(self, message: str, **meta) -> None:
|
||||
"""Set a failed response (matches the read/edit/... fail envelope)."""
|
||||
assert self.context is not None
|
||||
self.context.response.success = False
|
||||
self.context.response.answer = f"Error: {message}"
|
||||
if meta:
|
||||
self.context.response.metadata.update(meta)
|
||||
|
||||
def _collect_params(self) -> tuple[str, bool, int, str, frozenset[str]]:
|
||||
"""Read ``path`` / ``recursive`` / ``limit`` from context; coerce permissively."""
|
||||
assert self.context is not None
|
||||
|
|
|
|||
|
|
@ -2,24 +2,18 @@
|
|||
|
||||
from datetime import datetime
|
||||
|
||||
from ._base import _FileIOStep
|
||||
from ._file_io import read_file_safe
|
||||
from ._path import _check_path_permission, resolve_path
|
||||
from ..base_step import BaseStep
|
||||
from ...components import R
|
||||
|
||||
DEFAULT_EDITOR_MAX_BYTES = 5 * 1024 * 1024
|
||||
|
||||
|
||||
@R.register("load_step")
|
||||
class LoadStep(BaseStep):
|
||||
class LoadStep(_FileIOStep):
|
||||
"""Return complete text without the truncation used by agent-facing reads."""
|
||||
|
||||
def _fail(self, message: str, **metadata) -> None:
|
||||
assert self.context is not None
|
||||
self.context.response.success = False
|
||||
self.context.response.answer = f"Error: {message}"
|
||||
self.context.response.metadata.update(metadata)
|
||||
|
||||
async def execute(self):
|
||||
assert self.context is not None
|
||||
raw_path = str(self.context.get("path") or "")
|
||||
|
|
|
|||
|
|
@ -5,14 +5,14 @@ import base64
|
|||
import aiofiles
|
||||
import aiofiles.os
|
||||
|
||||
from ._base import _FileIOStep
|
||||
from ._path import NON_IMAGE_WARNING, gate_image, resolve_path
|
||||
from ..base_step import BaseStep
|
||||
from ...components import R
|
||||
from ...constants import DEFAULT_MAX_IMAGE_BYTES
|
||||
|
||||
|
||||
@R.register("read_image_step")
|
||||
class ReadImageStep(BaseStep):
|
||||
class ReadImageStep(_FileIOStep):
|
||||
"""Read an image file under ``workspace_dir`` and return base64 in ``answer``.
|
||||
|
||||
Step-level attributes (``kwargs``, configured in yaml under ``steps:`` —
|
||||
|
|
@ -32,13 +32,6 @@ class ReadImageStep(BaseStep):
|
|||
base64; ``metadata.non_image_warning=True`` flags it.
|
||||
"""
|
||||
|
||||
def _fail(self, message: str, **meta) -> None:
|
||||
assert self.context is not None
|
||||
self.context.response.success = False
|
||||
self.context.response.answer = f"Error: {message}"
|
||||
if meta:
|
||||
self.context.response.metadata.update(meta)
|
||||
|
||||
async def execute(self): # pylint: disable=too-many-return-statements
|
||||
assert self.context is not None
|
||||
raw = str(self.context.get("path") or "")
|
||||
|
|
|
|||
|
|
@ -2,14 +2,14 @@
|
|||
|
||||
from datetime import datetime
|
||||
|
||||
from ._base import _FileIOStep
|
||||
from ._file_io import detect_file_encoding, get_path_lock, write_file_safe
|
||||
from ._path import _check_path_permission, resolve_path
|
||||
from ..base_step import BaseStep
|
||||
from ...components import R
|
||||
|
||||
|
||||
@R.register("save_step")
|
||||
class SaveStep(BaseStep):
|
||||
class SaveStep(_FileIOStep):
|
||||
"""Persist editor content without interpreting or rebuilding frontmatter.
|
||||
|
||||
``expected_mtime`` is the ISO timestamp returned by ``stat``. When supplied,
|
||||
|
|
@ -17,12 +17,6 @@ class SaveStep(BaseStep):
|
|||
filesystem authoritative when ReMe and another editor are used together.
|
||||
"""
|
||||
|
||||
def _fail(self, message: str, **metadata) -> None:
|
||||
assert self.context is not None
|
||||
self.context.response.success = False
|
||||
self.context.response.answer = f"Error: {message}"
|
||||
self.context.response.metadata.update(metadata)
|
||||
|
||||
async def execute(self):
|
||||
assert self.context is not None
|
||||
raw_path = str(self.context.get("path") or "")
|
||||
|
|
|
|||
|
|
@ -2,14 +2,14 @@
|
|||
|
||||
import frontmatter
|
||||
|
||||
from ._base import _FileIOStep
|
||||
from ._file_io import detect_file_encoding, get_path_lock, write_file_safe
|
||||
from ._path import _check_path_permission, NON_MD_WARNING, gate_md, resolve_path
|
||||
from ..base_step import BaseStep
|
||||
from ...components import R
|
||||
|
||||
|
||||
@R.register("write_step")
|
||||
class WriteStep(BaseStep):
|
||||
class WriteStep(_FileIOStep):
|
||||
"""Write (create or overwrite) a markdown file.
|
||||
|
||||
Frontmatter accepts two reserved string fields (``name`` / ``description``)
|
||||
|
|
@ -26,13 +26,6 @@ class WriteStep(BaseStep):
|
|||
scope).
|
||||
"""
|
||||
|
||||
def _fail(self, message: str, **meta) -> None:
|
||||
assert self.context is not None
|
||||
self.context.response.success = False
|
||||
self.context.response.answer = f"Error: {message}"
|
||||
if meta:
|
||||
self.context.response.metadata.update(meta)
|
||||
|
||||
async def execute(self): # pylint: disable=too-many-return-statements
|
||||
assert self.context is not None
|
||||
raw = str(self.context.get("path") or "")
|
||||
|
|
|
|||
|
|
@ -12,15 +12,13 @@ When ``app_context`` is absent the same structure is mirrored under
|
|||
"""
|
||||
|
||||
import datetime
|
||||
from typing import TYPE_CHECKING, Any, Callable, Final
|
||||
from typing import Callable, Final
|
||||
|
||||
from ._tool_context import _ToolContextMixin
|
||||
from ...schema import FileChunk
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ...components import ApplicationContext
|
||||
|
||||
|
||||
class _ToolContextDedupMixin:
|
||||
class _ToolContextDedupMixin(_ToolContextMixin):
|
||||
"""Mixin providing tool_context-scoped chunk dedup with TTL.
|
||||
|
||||
Must be mixed into a ``BaseStep`` subclass (e.g. ``SearchStep``); it
|
||||
|
|
@ -29,7 +27,6 @@ class _ToolContextDedupMixin:
|
|||
set by the host step's ``__init__`` (default 24h).
|
||||
"""
|
||||
|
||||
TOOL_CONTEXTS_KEY: Final[str] = "tool_contexts"
|
||||
SEARCH_SEEN_RANGES_KEY: Final[str] = "search_seen_chunk_ranges"
|
||||
|
||||
def __init_subclass__(cls, **kwargs):
|
||||
|
|
@ -53,25 +50,7 @@ class _ToolContextDedupMixin:
|
|||
)
|
||||
return super().__new__(cls, *args, **kwargs)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Declared by BaseStep; repeated here so static analysis resolves
|
||||
# attribute access on the mixin without inheriting BaseStep.
|
||||
app_context: "ApplicationContext | None"
|
||||
kwargs: dict[str, Any]
|
||||
seen_ttl_hours: float
|
||||
|
||||
def _tool_context_store(self, tool_context_id: str) -> dict:
|
||||
"""Return the mutable state bucket for a tool context.
|
||||
|
||||
The bucket is created lazily on first access and lives at
|
||||
``metadata["tool_contexts"][tool_context_id]`` (or the same path under
|
||||
``kwargs`` when no ``app_context`` is available, e.g. in unit tests).
|
||||
"""
|
||||
if self.app_context is not None:
|
||||
contexts = self.app_context.metadata.setdefault(self.TOOL_CONTEXTS_KEY, {})
|
||||
else:
|
||||
contexts = self.kwargs.setdefault(self.TOOL_CONTEXTS_KEY, {})
|
||||
return contexts.setdefault(tool_context_id, {})
|
||||
seen_ttl_hours: float
|
||||
|
||||
@staticmethod
|
||||
def _now_ts() -> float:
|
||||
|
|
|
|||
68
reme/steps/index/_single_search.py
Normal file
68
reme/steps/index/_single_search.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
"""Shared execution for single-backend search steps."""
|
||||
|
||||
from abc import abstractmethod
|
||||
from typing import Final
|
||||
|
||||
from ._dedup import _ToolContextDedupMixin
|
||||
from ._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE, join_chunk_entries
|
||||
from ._source_format import merge_session_chunk_intervals, render_chunk_entries
|
||||
from ..base_step import BaseStep
|
||||
from ...schema import FileChunk
|
||||
|
||||
_MAX_CANDIDATES: Final = 200
|
||||
_CANDIDATE_MULTIPLIER: Final = 10
|
||||
|
||||
|
||||
class _SingleSearchStep(_ToolContextDedupMixin, BaseStep):
|
||||
"""Retrieve candidates, then apply the common single-backend search contract."""
|
||||
|
||||
def __init__(self, *args, seen_ttl_hours: float = 24, include_source: bool = True, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.seen_ttl_hours = seen_ttl_hours
|
||||
self.include_source = include_source
|
||||
|
||||
async def execute(self):
|
||||
assert self.context is not None
|
||||
raw_query = self.context.get("query")
|
||||
query: str = "" if raw_query is None else str(raw_query).strip()
|
||||
limit: int = int(self.context.get("limit") or 5)
|
||||
min_score: float = float(self.context.get("min_score") or 0.0)
|
||||
tool_context_id: str = (self.context.get("tool_context_id", "") or "").strip()
|
||||
|
||||
if not query:
|
||||
self.context.response.success = False
|
||||
self.context.response.answer = "Error: query cannot be empty"
|
||||
return self.context.response
|
||||
assert limit > 0, f"limit must be positive, got {limit}"
|
||||
|
||||
candidates = min(_MAX_CANDIDATES, max(1, limit * _CANDIDATE_MULTIPLIER))
|
||||
results = await self._retrieve(query, candidates)
|
||||
self.logger.info(f"[{self.name}] query={query!r} candidates={candidates} hits={len(results)}")
|
||||
|
||||
if min_score > 0.0:
|
||||
results = [chunk for chunk in results if chunk.score >= min_score]
|
||||
|
||||
pre_dedup_count = 0
|
||||
if tool_context_id:
|
||||
pre_dedup_count = len(results)
|
||||
results, _ = self._dedupe_tool_context(results, tool_context_id, limit)
|
||||
else:
|
||||
results = results[:limit]
|
||||
|
||||
session_dir = self.config_value("session_dir")
|
||||
entries = render_chunk_entries(
|
||||
merge_session_chunk_intervals(results, session_dir),
|
||||
session_dir,
|
||||
include_source=self.include_source,
|
||||
)
|
||||
self.context.response.answer = join_chunk_entries(entries)
|
||||
if not results:
|
||||
self.context.response.answer = ALL_RETURNED_MESSAGE if pre_dedup_count > 0 else NO_RESULTS_MESSAGE
|
||||
self.context.response.metadata["results"] = [
|
||||
c.model_dump(exclude_none=True, exclude={"embedding"}) for c in results
|
||||
]
|
||||
return self.context.response
|
||||
|
||||
@abstractmethod
|
||||
async def _retrieve(self, query: str, candidates: int) -> list[FileChunk]:
|
||||
"""Fetch ranked candidates from the concrete backend."""
|
||||
23
reme/steps/index/_tool_context.py
Normal file
23
reme/steps/index/_tool_context.py
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
"""Shared access to application-lifetime tool context state."""
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ...components import ApplicationContext
|
||||
|
||||
|
||||
class _ToolContextMixin:
|
||||
"""Share storage access while leaving each step's state format independent."""
|
||||
|
||||
TOOL_CONTEXTS_KEY: Final[str] = "tool_contexts"
|
||||
|
||||
if TYPE_CHECKING:
|
||||
app_context: "ApplicationContext | None"
|
||||
kwargs: dict[str, Any]
|
||||
|
||||
def _tool_context_store(self, tool_context_id: str) -> dict:
|
||||
if self.app_context is not None:
|
||||
contexts = self.app_context.metadata.setdefault(self.TOOL_CONTEXTS_KEY, {})
|
||||
else:
|
||||
contexts = self.kwargs.setdefault(self.TOOL_CONTEXTS_KEY, {})
|
||||
return contexts.setdefault(tool_context_id, {})
|
||||
|
|
@ -1,64 +1,13 @@
|
|||
"""``bm25_search_step`` — plain BM25 keyword search with tool_context dedup."""
|
||||
"""Plain BM25 keyword search with tool-context deduplication."""
|
||||
|
||||
from typing import Final
|
||||
|
||||
from ._dedup import _ToolContextDedupMixin
|
||||
from ._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE, join_chunk_entries
|
||||
from ._source_format import merge_session_chunk_intervals, render_chunk_entries
|
||||
from ..base_step import BaseStep
|
||||
from ._single_search import _SingleSearchStep
|
||||
from ...components import R
|
||||
|
||||
_MAX_CANDIDATES: Final = 200
|
||||
_CANDIDATE_MULTIPLIER: Final = 10
|
||||
from ...schema import FileChunk
|
||||
|
||||
|
||||
@R.register("bm25_search_step")
|
||||
class Bm25SearchStep(_ToolContextDedupMixin, BaseStep):
|
||||
"""BM25-only search: retrieve, filter by min_score, dedup by tool_context, truncate."""
|
||||
class Bm25SearchStep(_SingleSearchStep):
|
||||
"""Search the BM25 keyword backend using the shared result contract."""
|
||||
|
||||
def __init__(self, *args, seen_ttl_hours: float = 24, include_source: bool = True, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.seen_ttl_hours = seen_ttl_hours
|
||||
self.include_source = include_source
|
||||
|
||||
async def execute(self):
|
||||
assert self.context is not None
|
||||
raw_query = self.context.get("query")
|
||||
query: str = "" if raw_query is None else str(raw_query).strip()
|
||||
limit: int = int(self.context.get("limit") or 5)
|
||||
min_score: float = float(self.context.get("min_score") or 0.0)
|
||||
tool_context_id: str = (self.context.get("tool_context_id", "") or "").strip()
|
||||
|
||||
if not query:
|
||||
self.context.response.success = False
|
||||
self.context.response.answer = "Error: query cannot be empty"
|
||||
return self.context.response
|
||||
assert limit > 0, f"limit must be positive, got {limit}"
|
||||
|
||||
candidates = min(_MAX_CANDIDATES, max(1, limit * _CANDIDATE_MULTIPLIER))
|
||||
results = await self.file_store.keyword_search(query, candidates, {})
|
||||
self.logger.info(f"[{self.name}] query={query!r} candidates={candidates} hits={len(results)}")
|
||||
|
||||
if min_score > 0.0:
|
||||
results = [chunk for chunk in results if chunk.score >= min_score]
|
||||
|
||||
pre_dedup_count = 0
|
||||
if tool_context_id:
|
||||
pre_dedup_count = len(results)
|
||||
results, _ = self._dedupe_tool_context(results, tool_context_id, limit)
|
||||
else:
|
||||
results = results[:limit]
|
||||
|
||||
session_dir = self.config_value("session_dir")
|
||||
entries = render_chunk_entries(
|
||||
merge_session_chunk_intervals(results, session_dir),
|
||||
session_dir,
|
||||
include_source=self.include_source,
|
||||
)
|
||||
self.context.response.answer = join_chunk_entries(entries)
|
||||
if not results:
|
||||
self.context.response.answer = ALL_RETURNED_MESSAGE if pre_dedup_count > 0 else NO_RESULTS_MESSAGE
|
||||
self.context.response.metadata["results"] = [
|
||||
c.model_dump(exclude_none=True, exclude={"embedding"}) for c in results
|
||||
]
|
||||
return self.context.response
|
||||
async def _retrieve(self, query: str, candidates: int) -> list[FileChunk]:
|
||||
return await self.file_store.keyword_search(query, candidates, {})
|
||||
|
|
|
|||
|
|
@ -2,24 +2,17 @@
|
|||
|
||||
from typing import Final
|
||||
|
||||
from ._tool_context import _ToolContextMixin
|
||||
from ..base_step import BaseStep
|
||||
from ...components import R
|
||||
|
||||
|
||||
@R.register("add_draft_step")
|
||||
class AddDraftStep(BaseStep):
|
||||
class AddDraftStep(_ToolContextMixin, BaseStep):
|
||||
"""Append one draft text item to the current tool context."""
|
||||
|
||||
TOOL_CONTEXTS_KEY: Final[str] = "tool_contexts"
|
||||
DRAFTS_KEY: Final[str] = "drafts"
|
||||
|
||||
def _tool_context_store(self, tool_context_id: str) -> dict:
|
||||
if self.app_context is not None:
|
||||
contexts = self.app_context.metadata.setdefault(self.TOOL_CONTEXTS_KEY, {})
|
||||
else:
|
||||
contexts = self.kwargs.setdefault(self.TOOL_CONTEXTS_KEY, {})
|
||||
return contexts.setdefault(tool_context_id, {})
|
||||
|
||||
async def execute(self):
|
||||
assert self.context is not None
|
||||
text = self.context.get("text", "")
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import datetime
|
|||
import os
|
||||
from typing import Final
|
||||
|
||||
from ._tool_context import _ToolContextMixin
|
||||
from ..base_step import BaseStep
|
||||
from ..file_io import extract_daily_date
|
||||
from ...components import R
|
||||
|
|
@ -35,10 +36,9 @@ def _filter_values(value: object) -> set:
|
|||
|
||||
|
||||
@R.register("search_step")
|
||||
class SearchStep(BaseStep):
|
||||
class SearchStep(_ToolContextMixin, BaseStep):
|
||||
"""Hybrid search: run vector + keyword in parallel, fuse via RRF, filter, truncate."""
|
||||
|
||||
TOOL_CONTEXTS_KEY: Final[str] = "tool_contexts"
|
||||
SEARCH_SEEN_KEY: Final[str] = "search_seen_chunk_ids"
|
||||
|
||||
def __init__(
|
||||
|
|
@ -98,14 +98,6 @@ class SearchStep(BaseStep):
|
|||
def _now_ts() -> float:
|
||||
return datetime.datetime.now().timestamp()
|
||||
|
||||
def _tool_context_store(self, tool_context_id: str) -> dict:
|
||||
"""Return the mutable state bucket for a tool context."""
|
||||
if self.app_context is not None:
|
||||
contexts = self.app_context.metadata.setdefault(self.TOOL_CONTEXTS_KEY, {})
|
||||
else:
|
||||
contexts = self.kwargs.setdefault(self.TOOL_CONTEXTS_KEY, {})
|
||||
return contexts.setdefault(tool_context_id, {})
|
||||
|
||||
def _dedupe_tool_context(
|
||||
self,
|
||||
chunks: list[FileChunk],
|
||||
|
|
|
|||
|
|
@ -1,64 +1,13 @@
|
|||
"""``vector_search_step`` — plain vector search with tool_context dedup."""
|
||||
"""Plain vector search with tool-context deduplication."""
|
||||
|
||||
from typing import Final
|
||||
|
||||
from ._dedup import _ToolContextDedupMixin
|
||||
from ._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE, join_chunk_entries
|
||||
from ._source_format import merge_session_chunk_intervals, render_chunk_entries
|
||||
from ..base_step import BaseStep
|
||||
from ._single_search import _SingleSearchStep
|
||||
from ...components import R
|
||||
|
||||
_MAX_CANDIDATES: Final = 200
|
||||
_CANDIDATE_MULTIPLIER: Final = 10
|
||||
from ...schema import FileChunk
|
||||
|
||||
|
||||
@R.register("vector_search_step")
|
||||
class VectorSearchStep(_ToolContextDedupMixin, BaseStep):
|
||||
"""Vector-only search: retrieve, filter by min_score, dedup by tool_context, truncate."""
|
||||
class VectorSearchStep(_SingleSearchStep):
|
||||
"""Search the vector backend using the shared result contract."""
|
||||
|
||||
def __init__(self, *args, seen_ttl_hours: float = 24, include_source: bool = True, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.seen_ttl_hours = seen_ttl_hours
|
||||
self.include_source = include_source
|
||||
|
||||
async def execute(self):
|
||||
assert self.context is not None
|
||||
raw_query = self.context.get("query")
|
||||
query: str = "" if raw_query is None else str(raw_query).strip()
|
||||
limit: int = int(self.context.get("limit") or 5)
|
||||
min_score: float = float(self.context.get("min_score") or 0.0)
|
||||
tool_context_id: str = (self.context.get("tool_context_id", "") or "").strip()
|
||||
|
||||
if not query:
|
||||
self.context.response.success = False
|
||||
self.context.response.answer = "Error: query cannot be empty"
|
||||
return self.context.response
|
||||
assert limit > 0, f"limit must be positive, got {limit}"
|
||||
|
||||
candidates = min(_MAX_CANDIDATES, max(1, limit * _CANDIDATE_MULTIPLIER))
|
||||
results = await self.file_store.vector_search(query, candidates, {})
|
||||
self.logger.info(f"[{self.name}] query={query!r} candidates={candidates} hits={len(results)}")
|
||||
|
||||
if min_score > 0.0:
|
||||
results = [chunk for chunk in results if chunk.score >= min_score]
|
||||
|
||||
pre_dedup_count = 0
|
||||
if tool_context_id:
|
||||
pre_dedup_count = len(results)
|
||||
results, _ = self._dedupe_tool_context(results, tool_context_id, limit)
|
||||
else:
|
||||
results = results[:limit]
|
||||
|
||||
session_dir = self.config_value("session_dir")
|
||||
entries = render_chunk_entries(
|
||||
merge_session_chunk_intervals(results, session_dir),
|
||||
session_dir,
|
||||
include_source=self.include_source,
|
||||
)
|
||||
self.context.response.answer = join_chunk_entries(entries)
|
||||
if not results:
|
||||
self.context.response.answer = ALL_RETURNED_MESSAGE if pre_dedup_count > 0 else NO_RESULTS_MESSAGE
|
||||
self.context.response.metadata["results"] = [
|
||||
c.model_dump(exclude_none=True, exclude={"embedding"}) for c in results
|
||||
]
|
||||
return self.context.response
|
||||
async def _retrieve(self, query: str, candidates: int) -> list[FileChunk]:
|
||||
return await self.file_store.vector_search(query, candidates, {})
|
||||
|
|
|
|||
|
|
@ -1,17 +0,0 @@
|
|||
"""TuShare client construction with an optional mirror endpoint."""
|
||||
|
||||
import os
|
||||
|
||||
|
||||
def create_tushare_api(token: str):
|
||||
"""Create a TuShare DataApi, using TUSHARE_MIRROR_URL when configured."""
|
||||
try:
|
||||
import tushare as ts
|
||||
except ImportError as exc: # pragma: no cover - optional core dependency.
|
||||
raise RuntimeError("tushare is required for market-data research") from exc
|
||||
|
||||
api = ts.pro_api(token)
|
||||
api._DataApi__timeout = 600 # pylint: disable=protected-access
|
||||
if mirror_url := os.getenv("TUSHARE_MIRROR_URL", "").strip():
|
||||
api._DataApi__http_url = mirror_url.rstrip("/") # pylint: disable=protected-access
|
||||
return api
|
||||
|
|
@ -402,8 +402,10 @@ Run proportionate checks from `reme_studio/`:
|
|||
```bash
|
||||
npm run format:check
|
||||
npm run lint
|
||||
npm run build
|
||||
npm run build:static
|
||||
npm run test:unit
|
||||
# Build and verify the Sites, static, and demo distributions:
|
||||
npm run test:build
|
||||
# Run both suites:
|
||||
npm test
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -357,8 +357,10 @@ npm run build:demo
|
|||
```bash
|
||||
npm run format:check
|
||||
npm run lint
|
||||
npm run build
|
||||
npm run build:static
|
||||
npm run test:unit
|
||||
# 构建并验证 Sites、静态和演示分发:
|
||||
npm run test:build
|
||||
# 运行以上全部检查:
|
||||
npm test
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -1,90 +0,0 @@
|
|||
import { headers } from "next/headers";
|
||||
import { redirect } from "next/navigation";
|
||||
|
||||
export type ChatGPTUser = {
|
||||
userId: string;
|
||||
displayName: string;
|
||||
email: string;
|
||||
fullName: string | null;
|
||||
};
|
||||
|
||||
const USER_ID_HEADER = "oai-authenticated-user-id";
|
||||
const USER_EMAIL_HEADER = "oai-authenticated-user-email";
|
||||
const USER_FULL_NAME_HEADER = "oai-authenticated-user-full-name";
|
||||
const USER_FULL_NAME_ENCODING_HEADER =
|
||||
"oai-authenticated-user-full-name-encoding";
|
||||
const PERCENT_ENCODED_UTF8 = "percent-encoded-utf-8";
|
||||
const SIGN_IN_PATH = "/signin-with-chatgpt";
|
||||
const SIGN_OUT_PATH = "/signout-with-chatgpt";
|
||||
const CALLBACK_PATH = "/callback";
|
||||
|
||||
export async function getChatGPTUser(): Promise<ChatGPTUser | null> {
|
||||
const requestHeaders = await headers();
|
||||
const userId = requestHeaders.get(USER_ID_HEADER);
|
||||
const email = requestHeaders.get(USER_EMAIL_HEADER);
|
||||
if (!userId || !email) return null;
|
||||
|
||||
const encodedFullName = requestHeaders.get(USER_FULL_NAME_HEADER);
|
||||
const fullName =
|
||||
encodedFullName &&
|
||||
requestHeaders.get(USER_FULL_NAME_ENCODING_HEADER) === PERCENT_ENCODED_UTF8
|
||||
? safeDecodeURIComponent(encodedFullName)
|
||||
: null;
|
||||
|
||||
return {
|
||||
userId,
|
||||
displayName: fullName ?? email,
|
||||
email,
|
||||
fullName,
|
||||
};
|
||||
}
|
||||
|
||||
export async function requireChatGPTUser(
|
||||
returnTo: string,
|
||||
): Promise<ChatGPTUser> {
|
||||
const user = await getChatGPTUser();
|
||||
if (user) return user;
|
||||
|
||||
redirect(chatGPTSignInPath(returnTo));
|
||||
}
|
||||
|
||||
export function chatGPTSignInPath(returnTo: string): string {
|
||||
const safeReturnTo = safeRelativeReturnPath(returnTo);
|
||||
return `${SIGN_IN_PATH}?return_to=${encodeURIComponent(safeReturnTo)}`;
|
||||
}
|
||||
|
||||
export function chatGPTSignOutPath(returnTo = "/"): string {
|
||||
const safeReturnTo = safeRelativeReturnPath(returnTo);
|
||||
return `${SIGN_OUT_PATH}?return_to=${encodeURIComponent(safeReturnTo)}`;
|
||||
}
|
||||
|
||||
function safeRelativeReturnPath(value: string): string {
|
||||
if (!value.startsWith("/") || value.startsWith("//")) return "/";
|
||||
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(value, "https://app.local");
|
||||
} catch {
|
||||
return "/";
|
||||
}
|
||||
if (url.origin !== "https://app.local") return "/";
|
||||
if (isReservedAuthPath(url.pathname)) return "/";
|
||||
|
||||
return `${url.pathname}${url.search}${url.hash}`;
|
||||
}
|
||||
|
||||
function isReservedAuthPath(pathname: string): boolean {
|
||||
return (
|
||||
pathname === SIGN_IN_PATH ||
|
||||
pathname === SIGN_OUT_PATH ||
|
||||
pathname === CALLBACK_PATH
|
||||
);
|
||||
}
|
||||
|
||||
function safeDecodeURIComponent(value: string): string | null {
|
||||
try {
|
||||
return decodeURIComponent(value);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -1610,27 +1610,6 @@ textarea:focus-visible {
|
|||
margin: 8px 0 10px;
|
||||
font-size: 12px;
|
||||
}
|
||||
.component-row {
|
||||
min-height: 48px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
border-top: 1px solid var(--line);
|
||||
}
|
||||
.component-row > span {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
font-size: 12px;
|
||||
}
|
||||
.component-row small {
|
||||
margin-top: 2px;
|
||||
color: var(--muted);
|
||||
font-size: 10px;
|
||||
}
|
||||
.component-row strong {
|
||||
font-size: 12px;
|
||||
}
|
||||
.index-card {
|
||||
padding: 22px;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,13 +9,24 @@ export async function openWorkspaceFile(path: string) {
|
|||
);
|
||||
const id = store.openMarkdown(path);
|
||||
if (existing) return;
|
||||
await loadWorkspaceFile(id, path);
|
||||
}
|
||||
|
||||
/** Load both newly opened and restored tabs, respecting the caller's lifetime. */
|
||||
export async function loadWorkspaceFile(
|
||||
id: string,
|
||||
path: string,
|
||||
shouldApply: () => boolean = () => true,
|
||||
) {
|
||||
const store = useWorkspaceStore.getState();
|
||||
try {
|
||||
const file = await readWorkspaceFile(path);
|
||||
store.hydrateMarkdown(id, file.content, file.stat.mtime);
|
||||
if (shouldApply()) store.hydrateMarkdown(id, file.content, file.stat.mtime);
|
||||
} catch (error) {
|
||||
store.failMarkdown(
|
||||
id,
|
||||
error instanceof Error ? error.message : "Failed to read file",
|
||||
);
|
||||
if (shouldApply())
|
||||
store.failMarkdown(
|
||||
id,
|
||||
error instanceof Error ? error.message : "Failed to read file",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -54,6 +54,19 @@ interface WorkspaceState {
|
|||
finishChat: (tabId: string, messageId: string, error?: string) => void;
|
||||
}
|
||||
|
||||
function updateChatMessage(
|
||||
tab: Extract<WorkspaceTab, { type: "agent" }>,
|
||||
messageId: string,
|
||||
update: (message: ChatMessage) => ChatMessage,
|
||||
) {
|
||||
return {
|
||||
...tab,
|
||||
messages: tab.messages.map((message) =>
|
||||
message.id === messageId ? update(message) : message,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
const fileTitle = (path: string) => path.split("/").pop() || path;
|
||||
// Preserve the original key so existing tabs, chats, and unsaved drafts survive the Studio rename.
|
||||
const WORKSPACE_STORAGE_KEY = IS_DEMO
|
||||
|
|
@ -62,189 +75,153 @@ const WORKSPACE_STORAGE_KEY = IS_DEMO
|
|||
|
||||
export const useWorkspaceStore = create<WorkspaceState>()(
|
||||
persist(
|
||||
(set, get) => ({
|
||||
tabs: [],
|
||||
activeTabId: undefined,
|
||||
openMarkdown: (path) => {
|
||||
const existing = get().tabs.find(
|
||||
(tab) => tab.type === "markdown" && tab.path === path,
|
||||
);
|
||||
if (existing) {
|
||||
set({ activeTabId: existing.id });
|
||||
return existing.id;
|
||||
}
|
||||
const id = `file:${path}`;
|
||||
(set, get) => {
|
||||
const updateTab = <Type extends WorkspaceTab["type"]>(
|
||||
id: string,
|
||||
type: Type,
|
||||
update: (tab: Extract<WorkspaceTab, { type: Type }>) => WorkspaceTab,
|
||||
) =>
|
||||
set((state) => ({
|
||||
tabs: [
|
||||
...state.tabs,
|
||||
{
|
||||
id,
|
||||
type: "markdown",
|
||||
title: fileTitle(path),
|
||||
path,
|
||||
content: "",
|
||||
savedContent: "",
|
||||
loading: true,
|
||||
},
|
||||
],
|
||||
activeTabId: id,
|
||||
tabs: state.tabs.map((tab) =>
|
||||
tab.id === id && tab.type === type
|
||||
? update(tab as Extract<WorkspaceTab, { type: Type }>)
|
||||
: tab,
|
||||
),
|
||||
}));
|
||||
return id;
|
||||
},
|
||||
openAgent: () => {
|
||||
const id = `agent:${crypto.randomUUID()}`;
|
||||
set((state) => ({
|
||||
tabs: [...state.tabs, { id, type: "agent", title: "", messages: [] }],
|
||||
activeTabId: id,
|
||||
}));
|
||||
return id;
|
||||
},
|
||||
openGraph: (root) => {
|
||||
const id = `graph:${root}`;
|
||||
const existing = get().tabs.find(
|
||||
(tab) => tab.id === id && tab.type === "graph",
|
||||
);
|
||||
if (existing) {
|
||||
set({ activeTabId: id });
|
||||
return {
|
||||
tabs: [],
|
||||
activeTabId: undefined,
|
||||
openMarkdown: (path) => {
|
||||
const existing = get().tabs.find(
|
||||
(tab) => tab.type === "markdown" && tab.path === path,
|
||||
);
|
||||
if (existing) {
|
||||
set({ activeTabId: existing.id });
|
||||
return existing.id;
|
||||
}
|
||||
const id = `file:${path}`;
|
||||
set((state) => ({
|
||||
tabs: [
|
||||
...state.tabs,
|
||||
{
|
||||
id,
|
||||
type: "markdown",
|
||||
title: fileTitle(path),
|
||||
path,
|
||||
content: "",
|
||||
savedContent: "",
|
||||
loading: true,
|
||||
},
|
||||
],
|
||||
activeTabId: id,
|
||||
}));
|
||||
return id;
|
||||
}
|
||||
set((state) => ({
|
||||
tabs: [
|
||||
...state.tabs,
|
||||
{ id, type: "graph", title: `${root} graph`, root },
|
||||
],
|
||||
activeTabId: id,
|
||||
}));
|
||||
return id;
|
||||
},
|
||||
closeTab: (id, discardUnsaved = false) =>
|
||||
set((state) => {
|
||||
if (
|
||||
!discardUnsaved &&
|
||||
unsavedTabsClosedBy(state.tabs, id, false).length
|
||||
)
|
||||
return state;
|
||||
const index = state.tabs.findIndex((tab) => tab.id === id);
|
||||
const tabs = state.tabs.filter((tab) => tab.id !== id);
|
||||
const activeTabId =
|
||||
state.activeTabId === id
|
||||
? tabs[Math.max(0, index - 1)]?.id
|
||||
: state.activeTabId;
|
||||
return { tabs, activeTabId };
|
||||
}),
|
||||
closeOtherTabs: (id, discardUnsaved = false) =>
|
||||
set((state) => {
|
||||
if (
|
||||
!discardUnsaved &&
|
||||
unsavedTabsClosedBy(state.tabs, id, true).length
|
||||
)
|
||||
return state;
|
||||
const tab = state.tabs.find((item) => item.id === id);
|
||||
return tab ? { tabs: [tab], activeTabId: id } : state;
|
||||
}),
|
||||
setActiveTab: (activeTabId) => set({ activeTabId }),
|
||||
hydrateMarkdown: (id, content, mtime) =>
|
||||
set((state) => ({
|
||||
tabs: state.tabs.map((tab) =>
|
||||
tab.id === id && tab.type === "markdown"
|
||||
? {
|
||||
...tab,
|
||||
content,
|
||||
savedContent: content,
|
||||
mtime,
|
||||
loading: false,
|
||||
error: undefined,
|
||||
}
|
||||
: tab,
|
||||
),
|
||||
})),
|
||||
failMarkdown: (id, error) =>
|
||||
set((state) => ({
|
||||
tabs: state.tabs.map((tab) =>
|
||||
tab.id === id && tab.type === "markdown"
|
||||
? { ...tab, loading: false, error }
|
||||
: tab,
|
||||
),
|
||||
})),
|
||||
updateMarkdown: (id, content) =>
|
||||
set((state) => ({
|
||||
tabs: state.tabs.map((tab) =>
|
||||
tab.id === id && tab.type === "markdown"
|
||||
? { ...tab, content }
|
||||
: tab,
|
||||
),
|
||||
})),
|
||||
markSaved: (id, content, mtime) =>
|
||||
set((state) => ({
|
||||
tabs: state.tabs.map((tab) =>
|
||||
tab.id === id && tab.type === "markdown"
|
||||
? markMarkdownContentSaved(tab, content, mtime)
|
||||
: tab,
|
||||
),
|
||||
})),
|
||||
addChatTurn: (tabId, user, assistant) =>
|
||||
set((state) => ({
|
||||
tabs: state.tabs.map((tab) =>
|
||||
tab.id === tabId && tab.type === "agent"
|
||||
? {
|
||||
...tab,
|
||||
title: tab.messages.length
|
||||
? tab.title
|
||||
: user.content.slice(0, 18),
|
||||
messages: [...tab.messages, user, assistant],
|
||||
streaming: true,
|
||||
}
|
||||
: tab,
|
||||
),
|
||||
})),
|
||||
applyChatChunk: (tabId, messageId, chunk) =>
|
||||
set((state) => ({
|
||||
tabs: state.tabs.map((tab) => {
|
||||
if (tab.id !== tabId || tab.type !== "agent") return tab;
|
||||
const messages = tab.messages.map((message) =>
|
||||
message.id === messageId
|
||||
? applyStreamChunk(message, chunk)
|
||||
: message,
|
||||
);
|
||||
return {
|
||||
...tab,
|
||||
sessionId: chunk.session_id || tab.sessionId,
|
||||
messages,
|
||||
};
|
||||
},
|
||||
openAgent: () => {
|
||||
const id = `agent:${crypto.randomUUID()}`;
|
||||
set((state) => ({
|
||||
tabs: [
|
||||
...state.tabs,
|
||||
{ id, type: "agent", title: "", messages: [] },
|
||||
],
|
||||
activeTabId: id,
|
||||
}));
|
||||
return id;
|
||||
},
|
||||
openGraph: (root) => {
|
||||
const id = `graph:${root}`;
|
||||
const existing = get().tabs.find(
|
||||
(tab) => tab.id === id && tab.type === "graph",
|
||||
);
|
||||
if (existing) {
|
||||
set({ activeTabId: id });
|
||||
return id;
|
||||
}
|
||||
set((state) => ({
|
||||
tabs: [
|
||||
...state.tabs,
|
||||
{ id, type: "graph", title: `${root} graph`, root },
|
||||
],
|
||||
activeTabId: id,
|
||||
}));
|
||||
return id;
|
||||
},
|
||||
closeTab: (id, discardUnsaved = false) =>
|
||||
set((state) => {
|
||||
if (
|
||||
!discardUnsaved &&
|
||||
unsavedTabsClosedBy(state.tabs, id, false).length
|
||||
)
|
||||
return state;
|
||||
const index = state.tabs.findIndex((tab) => tab.id === id);
|
||||
const tabs = state.tabs.filter((tab) => tab.id !== id);
|
||||
const activeTabId =
|
||||
state.activeTabId === id
|
||||
? tabs[Math.max(0, index - 1)]?.id
|
||||
: state.activeTabId;
|
||||
return { tabs, activeTabId };
|
||||
}),
|
||||
})),
|
||||
toggleChatBlock: (tabId, messageId, blockId, expanded) =>
|
||||
set((state) => ({
|
||||
tabs: state.tabs.map((tab) =>
|
||||
tab.id === tabId && tab.type === "agent"
|
||||
? {
|
||||
...tab,
|
||||
messages: tab.messages.map((message) =>
|
||||
message.id === messageId
|
||||
? toggleChatBlock(message, blockId, expanded)
|
||||
: message,
|
||||
),
|
||||
}
|
||||
: tab,
|
||||
closeOtherTabs: (id, discardUnsaved = false) =>
|
||||
set((state) => {
|
||||
if (
|
||||
!discardUnsaved &&
|
||||
unsavedTabsClosedBy(state.tabs, id, true).length
|
||||
)
|
||||
return state;
|
||||
const tab = state.tabs.find((item) => item.id === id);
|
||||
return tab ? { tabs: [tab], activeTabId: id } : state;
|
||||
}),
|
||||
setActiveTab: (activeTabId) => set({ activeTabId }),
|
||||
hydrateMarkdown: (id, content, mtime) =>
|
||||
updateTab(id, "markdown", (tab) => ({
|
||||
...tab,
|
||||
content,
|
||||
savedContent: content,
|
||||
mtime,
|
||||
loading: false,
|
||||
error: undefined,
|
||||
})),
|
||||
failMarkdown: (id, error) =>
|
||||
updateTab(id, "markdown", (tab) => ({
|
||||
...tab,
|
||||
loading: false,
|
||||
error,
|
||||
})),
|
||||
updateMarkdown: (id, content) =>
|
||||
updateTab(id, "markdown", (tab) => ({ ...tab, content })),
|
||||
markSaved: (id, content, mtime) =>
|
||||
updateTab(id, "markdown", (tab) =>
|
||||
markMarkdownContentSaved(tab, content, mtime),
|
||||
),
|
||||
})),
|
||||
finishChat: (tabId, messageId, error) =>
|
||||
set((state) => ({
|
||||
tabs: state.tabs.map((tab) =>
|
||||
tab.id === tabId && tab.type === "agent"
|
||||
? {
|
||||
...tab,
|
||||
streaming: false,
|
||||
messages: tab.messages.map((message) =>
|
||||
message.id === messageId
|
||||
? finishChatMessage(message, error)
|
||||
: message,
|
||||
),
|
||||
}
|
||||
: tab,
|
||||
addChatTurn: (tabId, user, assistant) =>
|
||||
updateTab(tabId, "agent", (tab) => ({
|
||||
...tab,
|
||||
title: tab.messages.length ? tab.title : user.content.slice(0, 18),
|
||||
messages: [...tab.messages, user, assistant],
|
||||
streaming: true,
|
||||
})),
|
||||
applyChatChunk: (tabId, messageId, chunk) =>
|
||||
updateTab(tabId, "agent", (tab) => ({
|
||||
...updateChatMessage(tab, messageId, (message) =>
|
||||
applyStreamChunk(message, chunk),
|
||||
),
|
||||
sessionId: chunk.session_id || tab.sessionId,
|
||||
})),
|
||||
toggleChatBlock: (tabId, messageId, blockId, expanded) =>
|
||||
updateTab(tabId, "agent", (tab) =>
|
||||
updateChatMessage(tab, messageId, (message) =>
|
||||
toggleChatBlock(message, blockId, expanded),
|
||||
),
|
||||
),
|
||||
})),
|
||||
}),
|
||||
finishChat: (tabId, messageId, error) =>
|
||||
updateTab(tabId, "agent", (tab) => ({
|
||||
...updateChatMessage(tab, messageId, (message) =>
|
||||
finishChatMessage(message, error),
|
||||
),
|
||||
streaming: false,
|
||||
})),
|
||||
};
|
||||
},
|
||||
{
|
||||
name: WORKSPACE_STORAGE_KEY,
|
||||
version: 1,
|
||||
|
|
|
|||
296
reme_studio/app/workspace-chat.tsx
Normal file
296
reme_studio/app/workspace-chat.tsx
Normal file
|
|
@ -0,0 +1,296 @@
|
|||
"use client";
|
||||
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { Check, CircleAlert, LoaderCircle, Send } from "lucide-react";
|
||||
import WorkspaceMarkdown from "./files-workspace/WorkspaceMarkdown";
|
||||
import { streamChat } from "./api";
|
||||
import { chatStreamError, formatStreamPayloads } from "./chat-stream";
|
||||
import { useI18n } from "./i18n";
|
||||
import { useWorkspaceStore } from "./store";
|
||||
import { IS_DEMO } from "./studio-mode";
|
||||
import type {
|
||||
ChatBlock,
|
||||
ChatMessage,
|
||||
DetailBlock,
|
||||
WorkspaceTab,
|
||||
} from "./types";
|
||||
import {
|
||||
appendWorkspaceFileReference,
|
||||
WORKSPACE_FILE_DRAG_TYPE,
|
||||
} from "./workspace-drag";
|
||||
import { scenarios } from "../demo/scenarios";
|
||||
|
||||
function MarkdownView({ content }: { content: string }) {
|
||||
return (
|
||||
<article className="markdown">
|
||||
<WorkspaceMarkdown content={content} />
|
||||
</article>
|
||||
);
|
||||
}
|
||||
|
||||
function ChunkStatus({
|
||||
active,
|
||||
error = false,
|
||||
}: {
|
||||
active: boolean;
|
||||
error?: boolean;
|
||||
}) {
|
||||
if (error) return <CircleAlert size={13} />;
|
||||
return active ? (
|
||||
<LoaderCircle className="spin" size={13} />
|
||||
) : (
|
||||
<Check size={13} />
|
||||
);
|
||||
}
|
||||
|
||||
function detailLabel(
|
||||
block: DetailBlock,
|
||||
t: ReturnType<typeof useI18n>["t"],
|
||||
): string {
|
||||
if (block.type === "think") return t("thinking");
|
||||
if (block.type === "data")
|
||||
return block.mediaType
|
||||
? `${t("dataChunk")} · ${block.mediaType}`
|
||||
: t("dataChunk");
|
||||
if (block.type === "approval") return t("approvalChunk");
|
||||
if (block.type === "usage") return t("usageChunk");
|
||||
return t("unknownChunk", { type: block.sourceType });
|
||||
}
|
||||
|
||||
function ChatBlockView({
|
||||
block,
|
||||
onToggle,
|
||||
}: {
|
||||
block: ChatBlock;
|
||||
onToggle: (blockId: string, expanded: boolean) => void;
|
||||
}) {
|
||||
const { t } = useI18n();
|
||||
if (block.type === "content") return <MarkdownView content={block.text} />;
|
||||
if (block.type === "error")
|
||||
return (
|
||||
<div className="chunk-error">
|
||||
<CircleAlert size={14} />
|
||||
{block.text}
|
||||
</div>
|
||||
);
|
||||
if (block.type === "usage") return null;
|
||||
|
||||
const active =
|
||||
block.status === "streaming" ||
|
||||
block.status === "calling" ||
|
||||
block.status === "running";
|
||||
const failed = block.status === "error";
|
||||
if (block.type === "tool") {
|
||||
const call = formatStreamPayloads(block.callPayloads);
|
||||
const result = formatStreamPayloads(block.resultPayloads);
|
||||
return (
|
||||
<details
|
||||
className={`stream-block tool ${failed ? "failed" : ""}`}
|
||||
open={block.expanded}
|
||||
onToggle={(event) => {
|
||||
if (event.currentTarget.open !== block.expanded)
|
||||
onToggle(block.id, event.currentTarget.open);
|
||||
}}
|
||||
>
|
||||
<summary>
|
||||
<ChunkStatus active={active} error={failed} />
|
||||
<span>{block.name}</span>
|
||||
<small>{active ? t("streaming") : t("completed")}</small>
|
||||
</summary>
|
||||
<div className="stream-block-body">
|
||||
{call && (
|
||||
<section>
|
||||
<strong>{t("toolCall")}</strong>
|
||||
<pre>{call}</pre>
|
||||
</section>
|
||||
)}
|
||||
{result && (
|
||||
<section>
|
||||
<strong>{t("toolResult")}</strong>
|
||||
<pre className="tool-result-scroll">{result}</pre>
|
||||
</section>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
);
|
||||
}
|
||||
|
||||
const detail = formatStreamPayloads(block.payloads);
|
||||
return (
|
||||
<details
|
||||
className={`stream-block ${block.type} ${failed ? "failed" : ""}`}
|
||||
open={block.expanded}
|
||||
onToggle={(event) => {
|
||||
if (event.currentTarget.open !== block.expanded)
|
||||
onToggle(block.id, event.currentTarget.open);
|
||||
}}
|
||||
>
|
||||
<summary>
|
||||
<ChunkStatus active={active} error={failed} />
|
||||
<span>{detailLabel(block, t)}</span>
|
||||
<small>{active ? t("streaming") : t("completed")}</small>
|
||||
</summary>
|
||||
{detail && (
|
||||
<div className="stream-block-body">
|
||||
<pre>{detail}</pre>
|
||||
</div>
|
||||
)}
|
||||
</details>
|
||||
);
|
||||
}
|
||||
|
||||
export default function Chat({
|
||||
tab,
|
||||
}: {
|
||||
tab: Extract<WorkspaceTab, { type: "agent" }>;
|
||||
}) {
|
||||
const { language, t } = useI18n();
|
||||
const [input, setInput] = useState("");
|
||||
const [fileDragOver, setFileDragOver] = useState(false);
|
||||
const endRef = useRef<HTMLDivElement>(null);
|
||||
const controller = useRef<AbortController | null>(null);
|
||||
const { addChatTurn, applyChatChunk, toggleChatBlock, finishChat } =
|
||||
useWorkspaceStore();
|
||||
|
||||
useEffect(() => {
|
||||
endRef.current?.scrollIntoView({ behavior: "smooth" });
|
||||
}, [tab.messages]);
|
||||
useEffect(() => () => controller.current?.abort(), []);
|
||||
|
||||
const suggestions = (
|
||||
<div className="suggestions">
|
||||
{(IS_DEMO
|
||||
? scenarios[language].map((scenario) => scenario.question)
|
||||
: [t("promptRecent"), t("promptTasks"), t("promptIdeas")]
|
||||
).map((question) => (
|
||||
<button key={question} onClick={() => setInput(question)}>
|
||||
{question}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
|
||||
const send = async () => {
|
||||
const query = input.trim();
|
||||
if (!query || tab.streaming) return;
|
||||
setInput("");
|
||||
const user: ChatMessage = {
|
||||
id: crypto.randomUUID(),
|
||||
role: "user",
|
||||
content: query,
|
||||
};
|
||||
const assistant: ChatMessage = {
|
||||
id: crypto.randomUUID(),
|
||||
role: "assistant",
|
||||
content: "",
|
||||
blocks: [],
|
||||
};
|
||||
addChatTurn(tab.id, user, assistant);
|
||||
const requestController = new AbortController();
|
||||
controller.current = requestController;
|
||||
const error = await chatStreamError(
|
||||
() =>
|
||||
streamChat(query, tab.sessionId, requestController.signal, (chunk) =>
|
||||
applyChatChunk(tab.id, assistant.id, chunk),
|
||||
),
|
||||
requestController.signal,
|
||||
t("chatFailed"),
|
||||
);
|
||||
finishChat(tab.id, assistant.id, error);
|
||||
if (controller.current === requestController) controller.current = null;
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="chat">
|
||||
<div className="messages">
|
||||
{!tab.messages.length && (
|
||||
<div className="chat-empty">
|
||||
<div className="agent-logo" aria-hidden="true" />
|
||||
<h1>{t("chatTitle")}</h1>
|
||||
<p>{t(IS_DEMO ? "demoChatDescription" : "chatDescription")}</p>
|
||||
{suggestions}
|
||||
</div>
|
||||
)}
|
||||
{tab.messages.map((message) => (
|
||||
<div className={`message ${message.role}`} key={message.id}>
|
||||
<div className="avatar">
|
||||
{message.role === "user" ? (
|
||||
t("you")
|
||||
) : (
|
||||
<span
|
||||
className="assistant-logo"
|
||||
aria-label="ReMe"
|
||||
title="ReMe"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
<div className="bubble">
|
||||
{message.role === "user" && <p>{message.content}</p>}
|
||||
{message.role === "assistant" &&
|
||||
message.blocks?.map((block) => (
|
||||
<ChatBlockView
|
||||
key={block.id}
|
||||
block={block}
|
||||
onToggle={(blockId, expanded) =>
|
||||
toggleChatBlock(tab.id, message.id, blockId, expanded)
|
||||
}
|
||||
/>
|
||||
))}
|
||||
{message.role === "assistant" &&
|
||||
tab.streaming &&
|
||||
!message.blocks?.some((block) => block.type !== "usage") && (
|
||||
<LoaderCircle className="spin" size={16} />
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
{IS_DEMO && tab.messages.length > 0 && suggestions}
|
||||
<div ref={endRef} />
|
||||
</div>
|
||||
<div
|
||||
className={`composer ${fileDragOver ? "file-drag-over" : ""}`}
|
||||
onDragOver={(event) => {
|
||||
if (!event.dataTransfer.types.includes(WORKSPACE_FILE_DRAG_TYPE))
|
||||
return;
|
||||
event.preventDefault();
|
||||
event.dataTransfer.dropEffect = "copy";
|
||||
setFileDragOver(true);
|
||||
}}
|
||||
onDragLeave={(event) => {
|
||||
if (!event.currentTarget.contains(event.relatedTarget as Node | null))
|
||||
setFileDragOver(false);
|
||||
}}
|
||||
onDrop={(event) => {
|
||||
const path = event.dataTransfer.getData(WORKSPACE_FILE_DRAG_TYPE);
|
||||
if (!path) return;
|
||||
event.preventDefault();
|
||||
setFileDragOver(false);
|
||||
setInput((current) => appendWorkspaceFileReference(current, path));
|
||||
}}
|
||||
>
|
||||
<div>
|
||||
<textarea
|
||||
value={input}
|
||||
onChange={(event) => setInput(event.target.value)}
|
||||
placeholder={t("askWorkspace")}
|
||||
rows={1}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter" && !event.shiftKey) {
|
||||
event.preventDefault();
|
||||
void send();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<button
|
||||
onClick={send}
|
||||
disabled={!input.trim() || tab.streaming}
|
||||
aria-label={t("send")}
|
||||
>
|
||||
<Send size={17} />
|
||||
</button>
|
||||
</div>
|
||||
<span>{t(IS_DEMO ? "demoChatHint" : "composerHint")}</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
136
reme_studio/app/workspace-tabs.tsx
Normal file
136
reme_studio/app/workspace-tabs.tsx
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
"use client";
|
||||
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { Bot, FileText, Network, X } from "lucide-react";
|
||||
import { useI18n } from "./i18n";
|
||||
import { useWorkspaceStore } from "./store";
|
||||
import { hasUnsavedChanges, unsavedTabsClosedBy } from "./tab-close";
|
||||
|
||||
export default function Tabs() {
|
||||
const { t } = useI18n();
|
||||
const { tabs, activeTabId, setActiveTab, closeTab, closeOtherTabs } =
|
||||
useWorkspaceStore();
|
||||
const [contextMenu, setContextMenu] = useState<{
|
||||
tabId: string;
|
||||
left: number;
|
||||
top: number;
|
||||
}>();
|
||||
const menuRef = useRef<HTMLDivElement>(null);
|
||||
useEffect(() => {
|
||||
if (!contextMenu) return;
|
||||
const close = (event: PointerEvent) => {
|
||||
if (!menuRef.current?.contains(event.target as Node))
|
||||
setContextMenu(undefined);
|
||||
};
|
||||
const closeOnEscape = (event: KeyboardEvent) => {
|
||||
if (event.key === "Escape") setContextMenu(undefined);
|
||||
};
|
||||
const closeOnViewportChange = () => setContextMenu(undefined);
|
||||
document.addEventListener("pointerdown", close);
|
||||
document.addEventListener("keydown", closeOnEscape);
|
||||
window.addEventListener("resize", closeOnViewportChange);
|
||||
window.addEventListener("scroll", closeOnViewportChange, true);
|
||||
return () => {
|
||||
document.removeEventListener("pointerdown", close);
|
||||
document.removeEventListener("keydown", closeOnEscape);
|
||||
window.removeEventListener("resize", closeOnViewportChange);
|
||||
window.removeEventListener("scroll", closeOnViewportChange, true);
|
||||
};
|
||||
}, [contextMenu]);
|
||||
const confirmDiscard = (tabId: string, closeOthers: boolean) => {
|
||||
const unsaved = unsavedTabsClosedBy(tabs, tabId, closeOthers);
|
||||
return (
|
||||
!unsaved.length ||
|
||||
window.confirm(
|
||||
t("discardUnsavedConfirm", { count: String(unsaved.length) }),
|
||||
)
|
||||
);
|
||||
};
|
||||
return (
|
||||
<>
|
||||
<div className="tabs" role="tablist">
|
||||
{tabs.map((tab) => {
|
||||
const dirty = hasUnsavedChanges(tab);
|
||||
return (
|
||||
<button
|
||||
key={tab.id}
|
||||
className={`tab ${tab.id === activeTabId ? "active" : ""}`}
|
||||
onClick={() => setActiveTab(tab.id)}
|
||||
onContextMenu={(event) => {
|
||||
event.preventDefault();
|
||||
setContextMenu({
|
||||
tabId: tab.id,
|
||||
left: Math.max(
|
||||
8,
|
||||
Math.min(event.clientX, window.innerWidth - 176),
|
||||
),
|
||||
top: Math.max(
|
||||
8,
|
||||
Math.min(event.clientY, window.innerHeight - 82),
|
||||
),
|
||||
});
|
||||
}}
|
||||
role="tab"
|
||||
>
|
||||
{tab.type === "agent" ? (
|
||||
<Bot size={14} />
|
||||
) : tab.type === "graph" ? (
|
||||
<Network size={14} />
|
||||
) : (
|
||||
<FileText size={14} />
|
||||
)}
|
||||
<span>
|
||||
{tab.type === "markdown"
|
||||
? tab.path.split("/").slice(-2).join("/")
|
||||
: tab.type === "graph"
|
||||
? `${tab.root} · ${t("memoryGraphShort")}`
|
||||
: tab.title || t("newConversation")}
|
||||
</span>
|
||||
{dirty ? (
|
||||
<i className="dirty" />
|
||||
) : (
|
||||
<X
|
||||
size={13}
|
||||
onClick={(event) => {
|
||||
event.stopPropagation();
|
||||
closeTab(tab.id);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
{contextMenu && (
|
||||
<div
|
||||
className="tab-context-menu"
|
||||
ref={menuRef}
|
||||
role="menu"
|
||||
style={{ left: contextMenu.left, top: contextMenu.top }}
|
||||
>
|
||||
<button
|
||||
role="menuitem"
|
||||
onClick={() => {
|
||||
if (!confirmDiscard(contextMenu.tabId, false)) return;
|
||||
closeTab(contextMenu.tabId, true);
|
||||
setContextMenu(undefined);
|
||||
}}
|
||||
>
|
||||
{t("closeCurrentTab")}
|
||||
</button>
|
||||
<button
|
||||
role="menuitem"
|
||||
disabled={tabs.length <= 1}
|
||||
onClick={() => {
|
||||
if (!confirmDiscard(contextMenu.tabId, true)) return;
|
||||
closeOtherTabs(contextMenu.tabId, true);
|
||||
setContextMenu(undefined);
|
||||
}}
|
||||
>
|
||||
{t("closeOtherTabs")}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
51
reme_studio/app/workspace-theme.tsx
Normal file
51
reme_studio/app/workspace-theme.tsx
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
"use client";
|
||||
|
||||
import { useEffect, useRef } from "react";
|
||||
import { Check, Monitor, Moon, Sun, SunMoon } from "lucide-react";
|
||||
import { useI18n } from "./i18n";
|
||||
import { type ThemePreference, useThemeStore } from "./theme";
|
||||
|
||||
export default function ThemeMenu() {
|
||||
const { t } = useI18n();
|
||||
const menuRef = useRef<HTMLDetailsElement>(null);
|
||||
const preference = useThemeStore((state) => state.preference);
|
||||
const setPreference = useThemeStore((state) => state.setPreference);
|
||||
useEffect(() => {
|
||||
const close = (event: PointerEvent) => {
|
||||
if (!menuRef.current?.contains(event.target as Node))
|
||||
menuRef.current?.removeAttribute("open");
|
||||
};
|
||||
document.addEventListener("pointerdown", close);
|
||||
return () => document.removeEventListener("pointerdown", close);
|
||||
}, []);
|
||||
const options: Array<[ThemePreference, string, React.ReactNode]> = [
|
||||
["light", t("lightTheme"), <Sun key="light" size={15} />],
|
||||
["dark", t("darkTheme"), <Moon key="dark" size={15} />],
|
||||
["system", t("systemTheme"), <Monitor key="system" size={15} />],
|
||||
];
|
||||
return (
|
||||
<details className="theme-menu" ref={menuRef}>
|
||||
<summary aria-label={t("appearance")} title={t("appearance")}>
|
||||
<SunMoon size={18} />
|
||||
</summary>
|
||||
<div role="menu">
|
||||
{options.map(([value, label, icon]) => (
|
||||
<button
|
||||
key={value}
|
||||
role="menuitemradio"
|
||||
aria-checked={preference === value}
|
||||
className={preference === value ? "active" : ""}
|
||||
onClick={() => {
|
||||
setPreference(value);
|
||||
menuRef.current?.removeAttribute("open");
|
||||
}}
|
||||
>
|
||||
{icon}
|
||||
<span>{label}</span>
|
||||
{preference === value && <Check size={14} />}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</details>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,55 +1,29 @@
|
|||
"use client";
|
||||
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { useEffect, useState } from "react";
|
||||
import dynamic from "next/dynamic";
|
||||
import WorkspaceMarkdown from "./files-workspace/WorkspaceMarkdown";
|
||||
import { IS_DEMO } from "./studio-mode";
|
||||
import { openWorkspaceFile } from "./open-file";
|
||||
import { loadWorkspaceFile, openWorkspaceFile } from "./open-file";
|
||||
import { SparkMenuExpandLine, SparkMenuFoldLine } from "@agentscope-ai/icons";
|
||||
import {
|
||||
BookOpenText,
|
||||
Bot,
|
||||
Check,
|
||||
CircleAlert,
|
||||
FileText,
|
||||
LoaderCircle,
|
||||
Monitor,
|
||||
Moon,
|
||||
Network,
|
||||
Send,
|
||||
Settings,
|
||||
Sparkles,
|
||||
Sun,
|
||||
SunMoon,
|
||||
X,
|
||||
} from "lucide-react";
|
||||
import { readWorkspaceFile, streamChat } from "./api";
|
||||
import { chatStreamError, formatStreamPayloads } from "./chat-stream";
|
||||
import { BookOpenText, Settings, Sparkles } from "lucide-react";
|
||||
import FilesNavigator from "./files-workspace/FilesNavigator";
|
||||
import MemoryGraphView from "./files-workspace/MemoryGraphView";
|
||||
import { clampNavigatorWidth } from "./files-workspace/panel-resize";
|
||||
import { useI18n, useLanguageStore } from "./i18n";
|
||||
import { useWorkspaceStore } from "./store";
|
||||
import { type ThemePreference, useThemeStore } from "./theme";
|
||||
import type {
|
||||
ChatBlock,
|
||||
ChatMessage,
|
||||
DetailBlock,
|
||||
WorkspaceTab,
|
||||
} from "./types";
|
||||
import {
|
||||
appendWorkspaceFileReference,
|
||||
WORKSPACE_FILE_DRAG_TYPE,
|
||||
} from "./workspace-drag";
|
||||
import { useThemeStore } from "./theme";
|
||||
import type { WorkspaceTab } from "./types";
|
||||
import ThemeMenu from "./workspace-theme";
|
||||
import Tabs from "./workspace-tabs";
|
||||
import Chat from "./workspace-chat";
|
||||
import packageJson from "../package.json";
|
||||
|
||||
const SettingsCenter = IS_DEMO
|
||||
? null
|
||||
: dynamic(() => import("./settings-center"), { ssr: false });
|
||||
const DemoBar = IS_DEMO
|
||||
? dynamic(() => import("../demo/DemoBar"), { ssr: false })
|
||||
: null;
|
||||
import { hasUnsavedChanges, unsavedTabsClosedBy } from "./tab-close";
|
||||
import packageJson from "../package.json";
|
||||
import { scenarios } from "../demo/scenarios";
|
||||
|
||||
const TabbedEditor = dynamic(() => import("./files-workspace/TabbedEditor"), {
|
||||
ssr: false,
|
||||
|
|
@ -69,451 +43,6 @@ function GitHubIcon({ size = 18 }: { size?: number }) {
|
|||
);
|
||||
}
|
||||
|
||||
function ThemeMenu() {
|
||||
const { t } = useI18n();
|
||||
const menuRef = useRef<HTMLDetailsElement>(null);
|
||||
const preference = useThemeStore((state) => state.preference);
|
||||
const setPreference = useThemeStore((state) => state.setPreference);
|
||||
useEffect(() => {
|
||||
const close = (event: PointerEvent) => {
|
||||
if (!menuRef.current?.contains(event.target as Node))
|
||||
menuRef.current?.removeAttribute("open");
|
||||
};
|
||||
document.addEventListener("pointerdown", close);
|
||||
return () => document.removeEventListener("pointerdown", close);
|
||||
}, []);
|
||||
const options: Array<[ThemePreference, string, React.ReactNode]> = [
|
||||
["light", t("lightTheme"), <Sun key="light" size={15} />],
|
||||
["dark", t("darkTheme"), <Moon key="dark" size={15} />],
|
||||
["system", t("systemTheme"), <Monitor key="system" size={15} />],
|
||||
];
|
||||
return (
|
||||
<details className="theme-menu" ref={menuRef}>
|
||||
<summary aria-label={t("appearance")} title={t("appearance")}>
|
||||
<SunMoon size={18} />
|
||||
</summary>
|
||||
<div role="menu">
|
||||
{options.map(([value, label, icon]) => (
|
||||
<button
|
||||
key={value}
|
||||
role="menuitemradio"
|
||||
aria-checked={preference === value}
|
||||
className={preference === value ? "active" : ""}
|
||||
onClick={() => {
|
||||
setPreference(value);
|
||||
menuRef.current?.removeAttribute("open");
|
||||
}}
|
||||
>
|
||||
{icon}
|
||||
<span>{label}</span>
|
||||
{preference === value && <Check size={14} />}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</details>
|
||||
);
|
||||
}
|
||||
|
||||
function Tabs() {
|
||||
const { t } = useI18n();
|
||||
const { tabs, activeTabId, setActiveTab, closeTab, closeOtherTabs } =
|
||||
useWorkspaceStore();
|
||||
const [contextMenu, setContextMenu] = useState<{
|
||||
tabId: string;
|
||||
left: number;
|
||||
top: number;
|
||||
}>();
|
||||
const menuRef = useRef<HTMLDivElement>(null);
|
||||
useEffect(() => {
|
||||
if (!contextMenu) return;
|
||||
const close = (event: PointerEvent) => {
|
||||
if (!menuRef.current?.contains(event.target as Node))
|
||||
setContextMenu(undefined);
|
||||
};
|
||||
const closeOnEscape = (event: KeyboardEvent) => {
|
||||
if (event.key === "Escape") setContextMenu(undefined);
|
||||
};
|
||||
const closeOnViewportChange = () => setContextMenu(undefined);
|
||||
document.addEventListener("pointerdown", close);
|
||||
document.addEventListener("keydown", closeOnEscape);
|
||||
window.addEventListener("resize", closeOnViewportChange);
|
||||
window.addEventListener("scroll", closeOnViewportChange, true);
|
||||
return () => {
|
||||
document.removeEventListener("pointerdown", close);
|
||||
document.removeEventListener("keydown", closeOnEscape);
|
||||
window.removeEventListener("resize", closeOnViewportChange);
|
||||
window.removeEventListener("scroll", closeOnViewportChange, true);
|
||||
};
|
||||
}, [contextMenu]);
|
||||
const confirmDiscard = (tabId: string, closeOthers: boolean) => {
|
||||
const unsaved = unsavedTabsClosedBy(tabs, tabId, closeOthers);
|
||||
return (
|
||||
!unsaved.length ||
|
||||
window.confirm(
|
||||
t("discardUnsavedConfirm", { count: String(unsaved.length) }),
|
||||
)
|
||||
);
|
||||
};
|
||||
return (
|
||||
<>
|
||||
<div className="tabs" role="tablist">
|
||||
{tabs.map((tab) => {
|
||||
const dirty = hasUnsavedChanges(tab);
|
||||
return (
|
||||
<button
|
||||
key={tab.id}
|
||||
className={`tab ${tab.id === activeTabId ? "active" : ""}`}
|
||||
onClick={() => setActiveTab(tab.id)}
|
||||
onContextMenu={(event) => {
|
||||
event.preventDefault();
|
||||
setContextMenu({
|
||||
tabId: tab.id,
|
||||
left: Math.max(
|
||||
8,
|
||||
Math.min(event.clientX, window.innerWidth - 176),
|
||||
),
|
||||
top: Math.max(
|
||||
8,
|
||||
Math.min(event.clientY, window.innerHeight - 82),
|
||||
),
|
||||
});
|
||||
}}
|
||||
role="tab"
|
||||
>
|
||||
{tab.type === "agent" ? (
|
||||
<Bot size={14} />
|
||||
) : tab.type === "graph" ? (
|
||||
<Network size={14} />
|
||||
) : (
|
||||
<FileText size={14} />
|
||||
)}
|
||||
<span>
|
||||
{tab.type === "markdown"
|
||||
? tab.path.split("/").slice(-2).join("/")
|
||||
: tab.type === "graph"
|
||||
? `${tab.root} · ${t("memoryGraphShort")}`
|
||||
: tab.title || t("newConversation")}
|
||||
</span>
|
||||
{dirty ? (
|
||||
<i className="dirty" />
|
||||
) : (
|
||||
<X
|
||||
size={13}
|
||||
onClick={(event) => {
|
||||
event.stopPropagation();
|
||||
closeTab(tab.id);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
{contextMenu && (
|
||||
<div
|
||||
className="tab-context-menu"
|
||||
ref={menuRef}
|
||||
role="menu"
|
||||
style={{ left: contextMenu.left, top: contextMenu.top }}
|
||||
>
|
||||
<button
|
||||
role="menuitem"
|
||||
onClick={() => {
|
||||
if (!confirmDiscard(contextMenu.tabId, false)) return;
|
||||
closeTab(contextMenu.tabId, true);
|
||||
setContextMenu(undefined);
|
||||
}}
|
||||
>
|
||||
{t("closeCurrentTab")}
|
||||
</button>
|
||||
<button
|
||||
role="menuitem"
|
||||
disabled={tabs.length <= 1}
|
||||
onClick={() => {
|
||||
if (!confirmDiscard(contextMenu.tabId, true)) return;
|
||||
closeOtherTabs(contextMenu.tabId, true);
|
||||
setContextMenu(undefined);
|
||||
}}
|
||||
>
|
||||
{t("closeOtherTabs")}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function MarkdownView({ content }: { content: string }) {
|
||||
return (
|
||||
<article className="markdown">
|
||||
<WorkspaceMarkdown content={content} />
|
||||
</article>
|
||||
);
|
||||
}
|
||||
|
||||
function ChunkStatus({
|
||||
active,
|
||||
error = false,
|
||||
}: {
|
||||
active: boolean;
|
||||
error?: boolean;
|
||||
}) {
|
||||
if (error) return <CircleAlert size={13} />;
|
||||
return active ? (
|
||||
<LoaderCircle className="spin" size={13} />
|
||||
) : (
|
||||
<Check size={13} />
|
||||
);
|
||||
}
|
||||
|
||||
function detailLabel(
|
||||
block: DetailBlock,
|
||||
t: ReturnType<typeof useI18n>["t"],
|
||||
): string {
|
||||
if (block.type === "think") return t("thinking");
|
||||
if (block.type === "data")
|
||||
return block.mediaType
|
||||
? `${t("dataChunk")} · ${block.mediaType}`
|
||||
: t("dataChunk");
|
||||
if (block.type === "approval") return t("approvalChunk");
|
||||
if (block.type === "usage") return t("usageChunk");
|
||||
return t("unknownChunk", { type: block.sourceType });
|
||||
}
|
||||
|
||||
function ChatBlockView({
|
||||
block,
|
||||
onToggle,
|
||||
}: {
|
||||
block: ChatBlock;
|
||||
onToggle: (blockId: string, expanded: boolean) => void;
|
||||
}) {
|
||||
const { t } = useI18n();
|
||||
if (block.type === "content") return <MarkdownView content={block.text} />;
|
||||
if (block.type === "error")
|
||||
return (
|
||||
<div className="chunk-error">
|
||||
<CircleAlert size={14} />
|
||||
{block.text}
|
||||
</div>
|
||||
);
|
||||
if (block.type === "usage") return null;
|
||||
|
||||
const active =
|
||||
block.status === "streaming" ||
|
||||
block.status === "calling" ||
|
||||
block.status === "running";
|
||||
const failed = block.status === "error";
|
||||
if (block.type === "tool") {
|
||||
const call = formatStreamPayloads(block.callPayloads);
|
||||
const result = formatStreamPayloads(block.resultPayloads);
|
||||
return (
|
||||
<details
|
||||
className={`stream-block tool ${failed ? "failed" : ""}`}
|
||||
open={block.expanded}
|
||||
onToggle={(event) => {
|
||||
if (event.currentTarget.open !== block.expanded)
|
||||
onToggle(block.id, event.currentTarget.open);
|
||||
}}
|
||||
>
|
||||
<summary>
|
||||
<ChunkStatus active={active} error={failed} />
|
||||
<span>{block.name}</span>
|
||||
<small>{active ? t("streaming") : t("completed")}</small>
|
||||
</summary>
|
||||
<div className="stream-block-body">
|
||||
{call && (
|
||||
<section>
|
||||
<strong>{t("toolCall")}</strong>
|
||||
<pre>{call}</pre>
|
||||
</section>
|
||||
)}
|
||||
{result && (
|
||||
<section>
|
||||
<strong>{t("toolResult")}</strong>
|
||||
<pre className="tool-result-scroll">{result}</pre>
|
||||
</section>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
);
|
||||
}
|
||||
|
||||
const detail = formatStreamPayloads(block.payloads);
|
||||
return (
|
||||
<details
|
||||
className={`stream-block ${block.type} ${failed ? "failed" : ""}`}
|
||||
open={block.expanded}
|
||||
onToggle={(event) => {
|
||||
if (event.currentTarget.open !== block.expanded)
|
||||
onToggle(block.id, event.currentTarget.open);
|
||||
}}
|
||||
>
|
||||
<summary>
|
||||
<ChunkStatus active={active} error={failed} />
|
||||
<span>{detailLabel(block, t)}</span>
|
||||
<small>{active ? t("streaming") : t("completed")}</small>
|
||||
</summary>
|
||||
{detail && (
|
||||
<div className="stream-block-body">
|
||||
<pre>{detail}</pre>
|
||||
</div>
|
||||
)}
|
||||
</details>
|
||||
);
|
||||
}
|
||||
|
||||
function Chat({ tab }: { tab: Extract<WorkspaceTab, { type: "agent" }> }) {
|
||||
const { language, t } = useI18n();
|
||||
const [input, setInput] = useState("");
|
||||
const [fileDragOver, setFileDragOver] = useState(false);
|
||||
const endRef = useRef<HTMLDivElement>(null);
|
||||
const controller = useRef<AbortController | null>(null);
|
||||
const { addChatTurn, applyChatChunk, toggleChatBlock, finishChat } =
|
||||
useWorkspaceStore();
|
||||
|
||||
useEffect(() => {
|
||||
endRef.current?.scrollIntoView({ behavior: "smooth" });
|
||||
}, [tab.messages]);
|
||||
useEffect(() => () => controller.current?.abort(), []);
|
||||
|
||||
const suggestions = (
|
||||
<div className="suggestions">
|
||||
{(IS_DEMO
|
||||
? scenarios[language].map((scenario) => scenario.question)
|
||||
: [t("promptRecent"), t("promptTasks"), t("promptIdeas")]
|
||||
).map((question) => (
|
||||
<button key={question} onClick={() => setInput(question)}>
|
||||
{question}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
|
||||
const send = async () => {
|
||||
const query = input.trim();
|
||||
if (!query || tab.streaming) return;
|
||||
setInput("");
|
||||
const user: ChatMessage = {
|
||||
id: crypto.randomUUID(),
|
||||
role: "user",
|
||||
content: query,
|
||||
};
|
||||
const assistant: ChatMessage = {
|
||||
id: crypto.randomUUID(),
|
||||
role: "assistant",
|
||||
content: "",
|
||||
blocks: [],
|
||||
};
|
||||
addChatTurn(tab.id, user, assistant);
|
||||
const requestController = new AbortController();
|
||||
controller.current = requestController;
|
||||
const error = await chatStreamError(
|
||||
() =>
|
||||
streamChat(query, tab.sessionId, requestController.signal, (chunk) =>
|
||||
applyChatChunk(tab.id, assistant.id, chunk),
|
||||
),
|
||||
requestController.signal,
|
||||
t("chatFailed"),
|
||||
);
|
||||
finishChat(tab.id, assistant.id, error);
|
||||
if (controller.current === requestController) controller.current = null;
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="chat">
|
||||
<div className="messages">
|
||||
{!tab.messages.length && (
|
||||
<div className="chat-empty">
|
||||
<div className="agent-logo" aria-hidden="true" />
|
||||
<h1>{t("chatTitle")}</h1>
|
||||
<p>{t(IS_DEMO ? "demoChatDescription" : "chatDescription")}</p>
|
||||
{suggestions}
|
||||
</div>
|
||||
)}
|
||||
{tab.messages.map((message) => (
|
||||
<div className={`message ${message.role}`} key={message.id}>
|
||||
<div className="avatar">
|
||||
{message.role === "user" ? (
|
||||
t("you")
|
||||
) : (
|
||||
<span
|
||||
className="assistant-logo"
|
||||
aria-label="ReMe"
|
||||
title="ReMe"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
<div className="bubble">
|
||||
{message.role === "user" && <p>{message.content}</p>}
|
||||
{message.role === "assistant" &&
|
||||
message.blocks?.map((block) => (
|
||||
<ChatBlockView
|
||||
key={block.id}
|
||||
block={block}
|
||||
onToggle={(blockId, expanded) =>
|
||||
toggleChatBlock(tab.id, message.id, blockId, expanded)
|
||||
}
|
||||
/>
|
||||
))}
|
||||
{message.role === "assistant" &&
|
||||
tab.streaming &&
|
||||
!message.blocks?.some((block) => block.type !== "usage") && (
|
||||
<LoaderCircle className="spin" size={16} />
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
{IS_DEMO && tab.messages.length > 0 && suggestions}
|
||||
<div ref={endRef} />
|
||||
</div>
|
||||
<div
|
||||
className={`composer ${fileDragOver ? "file-drag-over" : ""}`}
|
||||
onDragOver={(event) => {
|
||||
if (!event.dataTransfer.types.includes(WORKSPACE_FILE_DRAG_TYPE))
|
||||
return;
|
||||
event.preventDefault();
|
||||
event.dataTransfer.dropEffect = "copy";
|
||||
setFileDragOver(true);
|
||||
}}
|
||||
onDragLeave={(event) => {
|
||||
if (!event.currentTarget.contains(event.relatedTarget as Node | null))
|
||||
setFileDragOver(false);
|
||||
}}
|
||||
onDrop={(event) => {
|
||||
const path = event.dataTransfer.getData(WORKSPACE_FILE_DRAG_TYPE);
|
||||
if (!path) return;
|
||||
event.preventDefault();
|
||||
setFileDragOver(false);
|
||||
setInput((current) => appendWorkspaceFileReference(current, path));
|
||||
}}
|
||||
>
|
||||
<div>
|
||||
<textarea
|
||||
value={input}
|
||||
onChange={(event) => setInput(event.target.value)}
|
||||
placeholder={t("askWorkspace")}
|
||||
rows={1}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter" && !event.shiftKey) {
|
||||
event.preventDefault();
|
||||
void send();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<button
|
||||
onClick={send}
|
||||
disabled={!input.trim() || tab.streaming}
|
||||
aria-label={t("send")}
|
||||
>
|
||||
<Send size={17} />
|
||||
</button>
|
||||
</div>
|
||||
<span>{t(IS_DEMO ? "demoChatHint" : "composerHint")}</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function Workspace() {
|
||||
const { language, setLanguage, t } = useI18n();
|
||||
const hydrateLanguage = useLanguageStore((state) => state.hydrate);
|
||||
|
|
@ -522,8 +51,7 @@ function Workspace() {
|
|||
const [navigatorWidth, setNavigatorWidth] = useState(260);
|
||||
const [resizingNavigator, setResizingNavigator] = useState(false);
|
||||
const [settingsOpen, setSettingsOpen] = useState(false);
|
||||
const { tabs, activeTabId, openAgent, hydrateMarkdown, failMarkdown } =
|
||||
useWorkspaceStore();
|
||||
const { tabs, activeTabId, openAgent } = useWorkspaceStore();
|
||||
const active = tabs.find((tab) => tab.id === activeTabId);
|
||||
useEffect(() => {
|
||||
let mounted = true;
|
||||
|
|
@ -539,28 +67,16 @@ function Workspace() {
|
|||
tab.type === "markdown" && Boolean(tab.loading),
|
||||
);
|
||||
await Promise.all(
|
||||
restoredFiles.map(async (tab) => {
|
||||
try {
|
||||
const file = await readWorkspaceFile(tab.path);
|
||||
if (mounted)
|
||||
hydrateMarkdown(tab.id, file.content, file.stat.mtime);
|
||||
} catch (error) {
|
||||
if (mounted)
|
||||
failMarkdown(
|
||||
tab.id,
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: "Failed to read file",
|
||||
);
|
||||
}
|
||||
}),
|
||||
restoredFiles.map((tab) =>
|
||||
loadWorkspaceFile(tab.id, tab.path, () => mounted),
|
||||
),
|
||||
);
|
||||
},
|
||||
);
|
||||
return () => {
|
||||
mounted = false;
|
||||
};
|
||||
}, [failMarkdown, hydrateMarkdown]);
|
||||
}, []);
|
||||
useEffect(() => {
|
||||
hydrateLanguage();
|
||||
}, [hydrateLanguage]);
|
||||
|
|
|
|||
5
reme_studio/cloudflare-env.d.ts
vendored
5
reme_studio/cloudflare-env.d.ts
vendored
|
|
@ -1,5 +0,0 @@
|
|||
declare namespace Cloudflare {
|
||||
interface Env {
|
||||
DB: D1Database;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
import { env } from "cloudflare:workers";
|
||||
import { drizzle } from "drizzle-orm/d1";
|
||||
import * as schema from "./schema";
|
||||
|
||||
export function getDb() {
|
||||
if (!env.DB) {
|
||||
throw new Error(
|
||||
"Cloudflare D1 binding `DB` is unavailable. Set the `d1` field in .openai/hosting.json to `DB` or let your control plane inject the real binding values before using the database.",
|
||||
);
|
||||
}
|
||||
|
||||
return drizzle(env.DB, { schema });
|
||||
}
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
// Intentionally empty by default.
|
||||
// Add Drizzle tables here when the site actually needs a database.
|
||||
// See examples/d1/db/schema.ts for an opt-in example.
|
||||
export {};
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import { defineConfig } from "drizzle-kit";
|
||||
|
||||
export default defineConfig({
|
||||
out: "./drizzle",
|
||||
schema: "./db/schema.ts",
|
||||
dialect: "sqlite",
|
||||
});
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
{
|
||||
"version": "7",
|
||||
"dialect": "sqlite",
|
||||
"entries": []
|
||||
}
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
import { desc } from "drizzle-orm";
|
||||
import { getDb } from "../../../../../db";
|
||||
import { notes } from "../../../db/schema";
|
||||
|
||||
function toRouteErrorMessage(error: unknown) {
|
||||
const message = error instanceof Error ? error.message : "Unexpected error";
|
||||
const detail =
|
||||
error instanceof Error && error.cause instanceof Error
|
||||
? error.cause.message
|
||||
: "";
|
||||
const combined = `${message}\n${detail}`;
|
||||
|
||||
if (combined.includes("no such table") || combined.includes('from "notes"')) {
|
||||
return "The notes table is unavailable. Generate the migration locally with `npm run db:generate`, then deploy so the platform can apply the generated SQL to the real D1 database.";
|
||||
}
|
||||
|
||||
return message;
|
||||
}
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
const db = getDb();
|
||||
const rows = await db
|
||||
.select()
|
||||
.from(notes)
|
||||
.orderBy(desc(notes.createdAt), desc(notes.id))
|
||||
.limit(20);
|
||||
|
||||
return Response.json({ notes: rows });
|
||||
} catch (error) {
|
||||
return Response.json(
|
||||
{ error: toRouteErrorMessage(error) },
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function POST(request: Request) {
|
||||
try {
|
||||
const payload = (await request.json()) as {
|
||||
title?: string;
|
||||
content?: string;
|
||||
};
|
||||
const title = payload.title?.trim() ?? "";
|
||||
const content = payload.content?.trim() ?? "";
|
||||
|
||||
if (!title) {
|
||||
return Response.json({ error: "title is required" }, { status: 400 });
|
||||
}
|
||||
|
||||
const db = getDb();
|
||||
const [note] = await db
|
||||
.insert(notes)
|
||||
.values({ title, content })
|
||||
.returning();
|
||||
return Response.json({ note }, { status: 201 });
|
||||
} catch (error) {
|
||||
return Response.json(
|
||||
{ error: toRouteErrorMessage(error) },
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
import { sql } from "drizzle-orm";
|
||||
import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
|
||||
|
||||
export const notes = sqliteTable("notes", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
title: text("title").notNull(),
|
||||
content: text("content").notNull().default(""),
|
||||
createdAt: text("created_at")
|
||||
.notNull()
|
||||
.default(sql`CURRENT_TIMESTAMP`),
|
||||
});
|
||||
1183
reme_studio/package-lock.json
generated
1183
reme_studio/package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
|
@ -28,14 +28,15 @@
|
|||
"build": "WRANGLER_LOG_PATH=.wrangler/wrangler.log vinext build",
|
||||
"build:static": "vite build --config vite.static.config.ts",
|
||||
"start": "WRANGLER_LOG_PATH=.wrangler/wrangler.log vinext start",
|
||||
"test": "npm run build && npm run build:static && npm run build:demo && node --test tests/*.test.mjs",
|
||||
"test": "npm run test:unit && npm run test:build",
|
||||
"lint": "eslint . --ignore-pattern dist --ignore-pattern dist-static --ignore-pattern dist-demo --ignore-pattern .next",
|
||||
"format": "tsc --noEmit && prettier --write .",
|
||||
"format:check": "tsc --noEmit && prettier --check .",
|
||||
"db:generate": "drizzle-kit generate",
|
||||
"prepack": "npm run build:static",
|
||||
"build:demo": "vite build --config vite.static.config.ts --mode demo",
|
||||
"dev:demo": "vite --config vite.static.config.ts --mode demo"
|
||||
"dev:demo": "vite --config vite.static.config.ts --mode demo",
|
||||
"test:unit": "node --test tests/*.test.mjs",
|
||||
"test:build": "npm run build && npm run build:static && npm run build:demo && node --test tests/distribution/*.test.mjs"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@agentscope-ai/icons": "^1.0.67",
|
||||
|
|
@ -50,8 +51,6 @@
|
|||
"@types/react-dom": "19.2.3",
|
||||
"@vitejs/plugin-react": "6.0.2",
|
||||
"@vitejs/plugin-rsc": "0.5.26",
|
||||
"drizzle-kit": "0.31.10",
|
||||
"drizzle-orm": "0.45.2",
|
||||
"eslint": "9.39.4",
|
||||
"eslint-plugin-jsx-a11y": "6.10.2",
|
||||
"eslint-plugin-react": "7.37.5",
|
||||
|
|
@ -75,9 +74,6 @@
|
|||
"zustand": "^5.0.14"
|
||||
},
|
||||
"overrides": {
|
||||
"@esbuild-kit/core-utils": {
|
||||
"esbuild": "0.25.0"
|
||||
},
|
||||
"dompurify": "3.4.13"
|
||||
},
|
||||
"type": "module"
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
<svg fill="none" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path d="M14.5 13.5V5.41a1 1 0 0 0-.3-.7L9.8.29A1 1 0 0 0 9.08 0H1.5v13.5A2.5 2.5 0 0 0 4 16h8a2.5 2.5 0 0 0 2.5-2.5m-1.5 0v-7H8v-5H3v12a1 1 0 0 0 1 1h8a1 1 0 0 0 1-1M9.5 5V2.12L12.38 5zM5.13 5h-.62v1.25h2.12V5zm-.62 3h7.12v1.25H4.5zm.62 3h-.62v1.25h7.12V11z" clip-rule="evenodd" fill="#666" fill-rule="evenodd"/></svg>
|
||||
|
Before Width: | Height: | Size: 392 B |
|
|
@ -1 +0,0 @@
|
|||
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><g clip-path="url(#a)"><path fill-rule="evenodd" clip-rule="evenodd" d="M10.27 14.1a6.5 6.5 0 0 0 3.67-3.45q-1.24.21-2.7.34-.31 1.83-.97 3.1M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16m.48-1.52a7 7 0 0 1-.96 0H7.5a4 4 0 0 1-.84-1.32q-.38-.89-.63-2.08a40 40 0 0 0 3.92 0q-.25 1.2-.63 2.08a4 4 0 0 1-.84 1.31zm2.94-4.76q1.66-.15 2.95-.43a7 7 0 0 0 0-2.58q-1.3-.27-2.95-.43a18 18 0 0 1 0 3.44m-1.27-3.54a17 17 0 0 1 0 3.64 39 39 0 0 1-4.3 0 17 17 0 0 1 0-3.64 39 39 0 0 1 4.3 0m1.1-1.17q1.45.13 2.69.34a6.5 6.5 0 0 0-3.67-3.44q.65 1.26.98 3.1M8.48 1.5l.01.02q.41.37.84 1.31.38.89.63 2.08a40 40 0 0 0-3.92 0q.25-1.2.63-2.08a4 4 0 0 1 .85-1.32 7 7 0 0 1 .96 0m-2.75.4a6.5 6.5 0 0 0-3.67 3.44 29 29 0 0 1 2.7-.34q.31-1.83.97-3.1M4.58 6.28q-1.66.16-2.95.43a7 7 0 0 0 0 2.58q1.3.27 2.95.43a18 18 0 0 1 0-3.44m.17 4.71q-1.45-.12-2.69-.34a6.5 6.5 0 0 0 3.67 3.44q-.65-1.27-.98-3.1" fill="#666"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h16v16H0z"/></clipPath></defs></svg>
|
||||
|
Before Width: | Height: | Size: 1 KiB |
|
|
@ -1 +0,0 @@
|
|||
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M1.5 2.5h13v10a1 1 0 0 1-1 1h-11a1 1 0 0 1-1-1zM0 1h16v11.5a2.5 2.5 0 0 1-2.5 2.5h-11A2.5 2.5 0 0 1 0 12.5zm3.75 4.5a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5M7 4.75a.75.75 0 1 1-1.5 0 .75.75 0 0 1 1.5 0m1.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5" fill="#666"/></svg>
|
||||
|
Before Width: | Height: | Size: 386 B |
|
|
@ -3,7 +3,7 @@ import { access, readFile, readdir } from "node:fs/promises";
|
|||
import test from "node:test";
|
||||
|
||||
test("the browser demo builds for a subdirectory with no service API or live settings", async () => {
|
||||
const directory = new URL("../dist-demo/", import.meta.url);
|
||||
const directory = new URL("../../dist-demo/", import.meta.url);
|
||||
const html = await readFile(new URL("index.html", directory), "utf8");
|
||||
assert.match(html, /<title>Try ReMe Studio<\/title>/);
|
||||
assert.match(html, /src="\.\/assets\//);
|
||||
|
|
@ -3,7 +3,7 @@ import { readFile } from "node:fs/promises";
|
|||
import test from "node:test";
|
||||
|
||||
async function render() {
|
||||
const workerUrl = new URL("../dist/server/index.js", import.meta.url);
|
||||
const workerUrl = new URL("../../dist/server/index.js", import.meta.url);
|
||||
workerUrl.searchParams.set("test", `${process.pid}-${Date.now()}`);
|
||||
const { default: worker } = await import(workerUrl.href);
|
||||
|
||||
|
|
@ -23,7 +23,7 @@ test("server-renders the ReMe Studio shell", async () => {
|
|||
|
||||
const html = await response.text();
|
||||
const packageJson = JSON.parse(
|
||||
await readFile(new URL("../package.json", import.meta.url), "utf8"),
|
||||
await readFile(new URL("../../package.json", import.meta.url), "utf8"),
|
||||
);
|
||||
assert.match(html, /<title>ReMe Studio<\/title>/i);
|
||||
assert.match(
|
||||
|
|
@ -4,20 +4,22 @@ import test from "node:test";
|
|||
import { fileURLToPath } from "node:url";
|
||||
|
||||
test("static build contains the ReMe workspace entry and assets", async () => {
|
||||
const output = fileURLToPath(new URL("../dist-static/", import.meta.url));
|
||||
const output = fileURLToPath(new URL("../../dist-static/", import.meta.url));
|
||||
const html = await readFile(
|
||||
new URL("../dist-static/index.html", import.meta.url),
|
||||
new URL("../../dist-static/index.html", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
const assets = await readdir(
|
||||
new URL("../dist-static/assets/", import.meta.url),
|
||||
new URL("../../dist-static/assets/", import.meta.url),
|
||||
);
|
||||
const javascript = (
|
||||
await Promise.all(
|
||||
assets
|
||||
.filter((name) => name.endsWith(".js"))
|
||||
.map((name) =>
|
||||
readFile(new URL(`../dist-static/assets/${name}`, import.meta.url)),
|
||||
readFile(
|
||||
new URL(`../../dist-static/assets/${name}`, import.meta.url),
|
||||
),
|
||||
),
|
||||
)
|
||||
).join("\n");
|
||||
|
|
@ -14,7 +14,7 @@ async function exists(path: string): Promise<boolean> {
|
|||
}
|
||||
}
|
||||
|
||||
// Packages Sites metadata and migrations after Vite finishes compiling.
|
||||
// Packages Sites metadata after Vite finishes compiling.
|
||||
export function sites(): Plugin {
|
||||
let root = process.cwd();
|
||||
|
||||
|
|
@ -27,7 +27,6 @@ export function sites(): Plugin {
|
|||
async closeBundle() {
|
||||
const outputDirectory = resolve(root, "dist", ".openai");
|
||||
const hostingConfig = resolve(root, ".openai", "hosting.json");
|
||||
const drizzleSource = resolve(root, "drizzle");
|
||||
|
||||
await rm(outputDirectory, { recursive: true, force: true });
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
|
|
@ -35,11 +34,6 @@ export function sites(): Plugin {
|
|||
if (await exists(hostingConfig)) {
|
||||
await cp(hostingConfig, resolve(outputDirectory, "hosting.json"));
|
||||
}
|
||||
if (await exists(drizzleSource)) {
|
||||
await cp(drizzleSource, resolve(outputDirectory, "drizzle"), {
|
||||
recursive: true,
|
||||
});
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import handler from "vinext/server/app-router-entry";
|
|||
|
||||
interface Env {
|
||||
ASSETS: Fetcher;
|
||||
DB: D1Database;
|
||||
IMAGES: {
|
||||
input(stream: ReadableStream): {
|
||||
transform(options: Record<string, unknown>): {
|
||||
|
|
|
|||
79
tests/unit/file_store_test_support.py
Normal file
79
tests/unit/file_store_test_support.py
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
"""Small shared constructors for local, FAISS, and zvec store contract tests."""
|
||||
|
||||
# pylint: disable=protected-access
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from reme.components.file_store import FaissLocalFileStore, ZvecLocalFileStore
|
||||
from reme.schema import FileChunk, FileNode
|
||||
|
||||
|
||||
class FakeEmbeddingStore:
|
||||
"""Small deterministic embedding provider used by file-store tests."""
|
||||
|
||||
dimensions = 2
|
||||
max_batch_size = 10
|
||||
is_healthy = True
|
||||
|
||||
def _embed(self, text: str) -> np.ndarray:
|
||||
if "beta" in text or "fresh" in text:
|
||||
return np.array([0.0, 1.0], dtype=np.float16)
|
||||
return np.array([1.0, 0.0], dtype=np.float16)
|
||||
|
||||
async def health_check(self, _timeout: float = 2.0) -> bool:
|
||||
"""Report the fake embedding service as healthy."""
|
||||
return True
|
||||
|
||||
async def get_embedding(self, input_text: str, **_kwargs) -> np.ndarray:
|
||||
"""Return a deterministic embedding for a single text."""
|
||||
return self._embed(input_text)
|
||||
|
||||
async def get_node_embeddings(self, nodes: list[FileChunk], **_kwargs) -> list[FileChunk]:
|
||||
"""Attach deterministic embeddings to file chunks."""
|
||||
for chunk_node in nodes:
|
||||
chunk_node.embedding = self._embed(chunk_node.text)
|
||||
return nodes
|
||||
|
||||
async def dump(self) -> None:
|
||||
"""Persist no state for the in-memory fake."""
|
||||
|
||||
|
||||
def node(path: str) -> FileNode:
|
||||
"""Build a minimal file node."""
|
||||
return FileNode(path=path, st_mtime=1.0)
|
||||
|
||||
|
||||
def chunk(chunk_id: str, path: str, text: str, **metadata) -> FileChunk:
|
||||
"""Build a minimal file chunk."""
|
||||
return FileChunk(id=chunk_id, path=path, text=text, start_line=1, end_line=1, metadata=metadata)
|
||||
|
||||
|
||||
def _new_faiss_store(name, **kwargs):
|
||||
"""Construct a FAISS store when the optional backend is installed."""
|
||||
try:
|
||||
store = FaissLocalFileStore(name=name, embedding_store="", **kwargs)
|
||||
except ImportError:
|
||||
pytest.skip("faiss is not installed")
|
||||
return store
|
||||
|
||||
|
||||
def _new_zvec_store(name, **kwargs):
|
||||
"""Construct a zvec store with embedding disabled at bind time."""
|
||||
try:
|
||||
store = ZvecLocalFileStore(name=name, embedding_store="", **kwargs)
|
||||
except ImportError:
|
||||
pytest.skip("zvec is not installed")
|
||||
return store
|
||||
|
||||
|
||||
async def _settle_reindex(store, timeout=5.0):
|
||||
"""Wait until no async reindex is pending or in flight."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while store._reindex_event.is_set() or store._reindex_busy:
|
||||
if time.monotonic() > deadline:
|
||||
raise AssertionError("async reindex did not settle in time")
|
||||
await asyncio.sleep(0.005)
|
||||
|
|
@ -10,6 +10,7 @@ downstream ``update_index_step`` to consume; tests assert against that key.
|
|||
|
||||
# pylint: disable=protected-access,too-many-lines
|
||||
|
||||
from contextlib import chdir as temp_chdir
|
||||
import asyncio
|
||||
import datetime
|
||||
import os
|
||||
|
|
@ -55,22 +56,6 @@ warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba")
|
|||
warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources")
|
||||
|
||||
|
||||
class temp_chdir:
|
||||
"""Context manager to temporarily chdir into a path and restore on exit."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.old = None
|
||||
|
||||
def __enter__(self):
|
||||
self.old = os.getcwd()
|
||||
os.chdir(self.path)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
os.chdir(self.old)
|
||||
|
||||
|
||||
def write_file(path: Path, content: str = "x") -> Path:
|
||||
"""Create parent dirs and write `content` to `path`; return the path."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
|
@ -148,7 +133,6 @@ def test_build_watch_rules_basic():
|
|||
assert rules[0].path == workspace / "daily"
|
||||
assert rules[0].suffixes == ["md"]
|
||||
assert rules[1].path == workspace / "digest"
|
||||
print("✓ test_build_watch_rules_basic passed")
|
||||
|
||||
|
||||
def test_build_watch_rules_multiple_suffixes():
|
||||
|
|
@ -167,7 +151,6 @@ def test_build_watch_rules_multiple_suffixes():
|
|||
assert len(rules) == 2
|
||||
assert rules[0].suffixes == ["md", "jsonl"]
|
||||
assert rules[1].suffixes == ["md", "jsonl"]
|
||||
print("✓ test_build_watch_rules_multiple_suffixes passed")
|
||||
|
||||
|
||||
def test_build_watch_rules_fallback_literal():
|
||||
|
|
@ -176,7 +159,6 @@ def test_build_watch_rules_fallback_literal():
|
|||
workspace = Path("/fake/workspace")
|
||||
rules = build_watch_rules(app_config, workspace, watch_dirs=["custom_dir"], watch_suffixes=["txt"])
|
||||
assert rules[0].path == workspace / "custom_dir"
|
||||
print("✓ test_build_watch_rules_fallback_literal passed")
|
||||
|
||||
|
||||
def test_match_file_suffix():
|
||||
|
|
@ -186,7 +168,6 @@ def test_match_file_suffix():
|
|||
assert match_file("/workspace/daily/sub/note.md", rules)
|
||||
assert not match_file("/workspace/daily/file.txt", rules)
|
||||
assert not match_file("/workspace/other/file.md", rules)
|
||||
print("✓ test_match_file_suffix passed")
|
||||
|
||||
|
||||
def test_match_file_suffix_is_case_insensitive():
|
||||
|
|
@ -203,7 +184,6 @@ def test_match_file_no_suffix_filter():
|
|||
assert match_file("/workspace/resource/anything.xyz", rules)
|
||||
assert match_file("/workspace/resource/sub/deep.pdf", rules)
|
||||
assert not match_file("/workspace/other/file.md", rules)
|
||||
print("✓ test_match_file_no_suffix_filter passed")
|
||||
|
||||
|
||||
def test_collect_existing_filters():
|
||||
|
|
@ -227,7 +207,6 @@ def test_collect_existing_filters():
|
|||
assert str((daily / "ignore.txt").absolute()) not in paths
|
||||
assert str((resource / "data.json").absolute()) in paths
|
||||
assert str((resource / "binary.png").absolute()) not in paths
|
||||
print("✓ test_collect_existing_filters passed")
|
||||
|
||||
|
||||
def test_collect_existing_matches_uppercase_suffixes():
|
||||
|
|
@ -270,7 +249,6 @@ def test_clear_and_scan_flow_includes_jsonl_when_configured():
|
|||
assert paths == {"note.md", "events.jsonl"}
|
||||
finally:
|
||||
await fs.close()
|
||||
print("✓ test_clear_and_scan_defaults_include_jsonl passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -318,7 +296,6 @@ def test_scan_changes_initial_all_added():
|
|||
assert len(ctx["changes"]) == 2
|
||||
finally:
|
||||
await _teardown(fs, chunker)
|
||||
print("✓ test_scan_changes_initial_all_added passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -341,7 +318,6 @@ def test_scan_changes_no_changes():
|
|||
assert ctx["changes"] == []
|
||||
finally:
|
||||
await _teardown(fs, chunker)
|
||||
print("✓ test_scan_changes_no_changes passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -371,7 +347,6 @@ def test_scan_changes_detect_modify_delete():
|
|||
assert counts == {"added": 1, "modified": 1, "deleted": 1}
|
||||
finally:
|
||||
await _teardown(fs, chunker)
|
||||
print("✓ test_scan_changes_detect_modify_delete passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -390,7 +365,6 @@ def test_scan_changes_missing_dir_skipped():
|
|||
assert resp.metadata["counts"] == {"added": 0, "modified": 0, "deleted": 0}
|
||||
finally:
|
||||
await _teardown(fs, chunker)
|
||||
print("✓ test_scan_changes_missing_dir_skipped passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -416,7 +390,6 @@ def test_scan_changes_resource_dir():
|
|||
assert resp.metadata["counts"]["added"] == 2
|
||||
finally:
|
||||
await _teardown(fs, chunker)
|
||||
print("✓ test_scan_changes_resource_dir passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -442,7 +415,6 @@ def test_init_changes_named_file_catalog_monitor():
|
|||
assert ctx["changes"][0]["change"] == "added"
|
||||
finally:
|
||||
await catalog.close()
|
||||
print("✓ test_init_changes_named_file_catalog_monitor passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -461,7 +433,6 @@ def test_bucket_changes_coalesces_by_final_file_state():
|
|||
assert buckets[Change.modified] == [str(p)]
|
||||
assert buckets[Change.added] == []
|
||||
assert buckets[Change.deleted] == []
|
||||
print("✓ test_bucket_changes_coalesces_by_final_file_state passed")
|
||||
|
||||
|
||||
def test_update_catalog_relative_path_uses_workspace():
|
||||
|
|
@ -484,7 +455,6 @@ def test_update_catalog_relative_path_uses_workspace():
|
|||
assert [n.path for n in nodes] == ["daily/a.md"]
|
||||
finally:
|
||||
await catalog.close()
|
||||
print("✓ test_update_catalog_relative_path_uses_workspace passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -1092,7 +1062,6 @@ def test_index_update_loop_init_dispatch_updates_store_across_batches():
|
|||
finally:
|
||||
await chunker.close()
|
||||
await fs.close()
|
||||
print("✓ test_index_update_loop_init_dispatch_updates_store_across_batches passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -1158,7 +1127,6 @@ def test_digest_watch_loop_init_dispatch_updates_named_catalog_and_logs():
|
|||
finally:
|
||||
await resource_catalog.close()
|
||||
await digest_catalog.close()
|
||||
print("✓ test_digest_watch_loop_init_dispatch_updates_named_catalog_and_logs passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -1181,8 +1149,6 @@ def test_watch_changes_default_low_power_timing():
|
|||
assert custom.step == 250
|
||||
assert custom.poll_delay_ms == 3000
|
||||
|
||||
print("✓ test_watch_changes_default_low_power_timing passed")
|
||||
|
||||
|
||||
def test_watch_changes_requires_stop_event():
|
||||
"""Missing stop_event in context raises a clear error."""
|
||||
|
|
@ -1200,7 +1166,6 @@ def test_watch_changes_requires_stop_event():
|
|||
assert "stop_event" in str(e)
|
||||
else:
|
||||
raise AssertionError("expected RuntimeError")
|
||||
print("✓ test_watch_changes_requires_stop_event passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -1221,7 +1186,6 @@ def test_watch_changes_raises_no_valid_paths():
|
|||
assert "No valid watch paths" in str(e)
|
||||
else:
|
||||
raise AssertionError("expected RuntimeError")
|
||||
print("✓ test_watch_changes_raises_no_valid_paths passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -1244,8 +1208,6 @@ def test_watch_changes_filter_matches_rules():
|
|||
assert not step._filter(Change.added, str(workspace / "daily/foo.txt"))
|
||||
assert not step._filter(Change.added, str(workspace / "resource/file.md"))
|
||||
|
||||
print("✓ test_watch_changes_filter_matches_rules passed")
|
||||
|
||||
|
||||
def test_watch_changes_filter_matches_uppercase_suffixes():
|
||||
"""The live watcher accepts uppercase extensions configured in lowercase."""
|
||||
|
|
@ -1267,8 +1229,6 @@ def test_watch_changes_dispatch_steps_list():
|
|||
step = WatchChangesStep(dispatch_steps=["update_catalog_step", "auto_resource_step"])
|
||||
assert step.dispatch_step_specs == ["update_catalog_step", "auto_resource_step"]
|
||||
|
||||
print("✓ test_watch_changes_dispatch_steps_list passed")
|
||||
|
||||
|
||||
def test_auto_resource_batch_deleted_changes():
|
||||
"""AutoResourceStep accepts a batch of change dicts from dispatch_steps."""
|
||||
|
|
@ -1304,7 +1264,6 @@ def test_auto_resource_batch_deleted_changes():
|
|||
assert not note_path.exists()
|
||||
finally:
|
||||
await fs.close()
|
||||
print("✓ test_auto_resource_batch_deleted_changes passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -1620,7 +1579,6 @@ def test_auto_resource_accepts_loose_root_resource():
|
|||
"created": True,
|
||||
"source_path": source.resolve(),
|
||||
}
|
||||
print("✓ test_auto_resource_accepts_loose_root_resource passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -1666,7 +1624,6 @@ def test_auto_resource_loose_root_resource_keeps_existing_dated_resource():
|
|||
"created": True,
|
||||
"source_path": source.resolve(),
|
||||
}
|
||||
print("✓ test_auto_resource_loose_root_resource_keeps_existing_dated_resource passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -1703,7 +1660,6 @@ def test_auto_resource_modified_missing_note_uses_create_tools():
|
|||
assert (cwd / "daily" / "2026-01-01" / "resource-summary.md").exists()
|
||||
finally:
|
||||
await fs.close()
|
||||
print("✓ test_auto_resource_modified_missing_note_uses_create_tools passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -1828,7 +1784,6 @@ def test_auto_resource_sanitizes_invalid_generated_name():
|
|||
assert "name: bad-name" in (cwd / "daily" / "2026-01-01" / "bad-name.md").read_text(encoding="utf-8")
|
||||
finally:
|
||||
await fs.close()
|
||||
print("✓ test_auto_resource_sanitizes_invalid_generated_name passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -1869,7 +1824,6 @@ def test_auto_resource_uniquifies_conflicting_generated_name():
|
|||
assert not (cwd / "daily" / "2026-01-01" / "report.md").exists()
|
||||
finally:
|
||||
await fs.close()
|
||||
print("✓ test_auto_resource_uniquifies_conflicting_generated_name passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -1903,7 +1857,6 @@ def test_auto_resource_update_finds_renamed_note_by_source_resource():
|
|||
assert "Target note path: daily/2026-01-01/generated-name.md" in wrapper.inputs
|
||||
finally:
|
||||
await fs.close()
|
||||
print("✓ test_auto_resource_update_finds_renamed_note_by_source_resource passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -1948,7 +1901,6 @@ def test_auto_resource_update_keeps_existing_renamed_path():
|
|||
assert not (cwd / "daily" / "2026-01-01" / "new-suggestion.md").exists()
|
||||
finally:
|
||||
await fs.close()
|
||||
print("✓ test_auto_resource_update_keeps_existing_renamed_path passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -1981,7 +1933,6 @@ def test_auto_resource_reports_unmodified_when_agent_skips_existing_note():
|
|||
assert resp.metadata["modified"] is False
|
||||
finally:
|
||||
await fs.close()
|
||||
print("✓ test_auto_resource_reports_unmodified_when_agent_skips_existing_note passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -2013,7 +1964,6 @@ def test_auto_resource_preserves_unowned_loose_root_same_stem_note():
|
|||
assert note_path.read_text(encoding="utf-8") == "---\nname: report\n---\nbody\n"
|
||||
finally:
|
||||
await fs.close()
|
||||
print("✓ test_auto_resource_preserves_unowned_loose_root_same_stem_note passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -2044,7 +1994,6 @@ def test_auto_resource_deletes_renamed_note_by_source_resource():
|
|||
assert not note_path.exists()
|
||||
finally:
|
||||
await fs.close()
|
||||
print("✓ test_auto_resource_deletes_renamed_note_by_source_resource passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -2096,7 +2045,6 @@ def test_auto_memory_reports_modified_for_create_and_false_for_skip():
|
|||
assert context["changes"] == []
|
||||
finally:
|
||||
await fs.close()
|
||||
print("✓ test_auto_memory_reports_modified_for_create_and_false_for_skip passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -2148,7 +2096,6 @@ def test_auto_memory_uses_message_day_for_historical_create():
|
|||
assert (cwd / "daily" / historical_day / "memory.md").is_file()
|
||||
finally:
|
||||
await fs.close()
|
||||
print("✓ test_auto_memory_uses_message_day_for_historical_create passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -2194,7 +2141,6 @@ def test_auto_memory_rejects_invalid_explicit_date_before_saving_session():
|
|||
assert wrapper.inputs == ""
|
||||
finally:
|
||||
await fs.close()
|
||||
print("✓ test_auto_memory_rejects_invalid_explicit_date_before_saving_session passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -2237,8 +2183,6 @@ def test_auto_resource_result_hook_is_optional_and_isolated():
|
|||
app_ctx.metadata = {"qwenpaw_memory_result_hook": failing_hook}
|
||||
await step._emit_result_hook(changes=changes, results=results)
|
||||
|
||||
print("✓ test_auto_resource_result_hook_is_optional_and_isolated passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
|
|
@ -2259,48 +2203,5 @@ def test_log_changes_step():
|
|||
resp = await step(ctx)
|
||||
assert resp.success is True
|
||||
assert resp.metadata["count"] == 2
|
||||
print("✓ test_log_changes_step passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n=== Background Steps Tests ===")
|
||||
# _watch_rules
|
||||
test_build_watch_rules_basic()
|
||||
test_build_watch_rules_multiple_suffixes()
|
||||
test_build_watch_rules_fallback_literal()
|
||||
test_match_file_suffix()
|
||||
test_match_file_no_suffix_filter()
|
||||
test_collect_existing_filters()
|
||||
# InitChangesStep
|
||||
test_clear_and_scan_flow_includes_jsonl_when_configured()
|
||||
test_scan_changes_initial_all_added()
|
||||
test_scan_changes_no_changes()
|
||||
test_scan_changes_detect_modify_delete()
|
||||
test_scan_changes_missing_dir_skipped()
|
||||
test_scan_changes_resource_dir()
|
||||
test_index_update_loop_init_dispatch_updates_store_across_batches()
|
||||
test_digest_watch_loop_init_dispatch_updates_named_catalog_and_logs()
|
||||
# WatchChangesStep
|
||||
test_watch_changes_default_low_power_timing()
|
||||
test_watch_changes_requires_stop_event()
|
||||
test_watch_changes_raises_no_valid_paths()
|
||||
test_watch_changes_filter_matches_rules()
|
||||
test_watch_changes_dispatch_steps_list()
|
||||
test_auto_resource_batch_deleted_changes()
|
||||
test_auto_resource_accepts_loose_root_resource()
|
||||
test_auto_resource_loose_root_resource_keeps_existing_dated_resource()
|
||||
test_auto_resource_modified_missing_note_uses_create_tools()
|
||||
test_auto_resource_sanitizes_invalid_generated_name()
|
||||
test_auto_resource_uniquifies_conflicting_generated_name()
|
||||
test_auto_resource_update_finds_renamed_note_by_source_resource()
|
||||
test_auto_resource_update_keeps_existing_renamed_path()
|
||||
test_auto_memory_uses_message_day_for_historical_create()
|
||||
test_auto_memory_rejects_invalid_explicit_date_before_saving_session()
|
||||
test_auto_resource_preserves_unowned_loose_root_same_stem_note()
|
||||
test_auto_resource_deletes_renamed_note_by_source_resource()
|
||||
test_auto_resource_result_hook_is_optional_and_isolated()
|
||||
# LogChangesStep
|
||||
test_log_changes_step()
|
||||
print("\n所有测试通过!")
|
||||
|
|
|
|||
|
|
@ -406,31 +406,3 @@ def test_workspace_metadata_path_no_context():
|
|||
def test_component_metadata_path():
|
||||
comp = StubComponent()
|
||||
assert comp.component_metadata_path.name == ComponentEnum.FILE_CHUNKER.value
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n=== BaseComponent Tests ===")
|
||||
test_dependency_getattr_raises()
|
||||
test_bind_returns_none_for_empty_name()
|
||||
test_bind_returns_dependency_placeholder()
|
||||
test_bind_rejects_base_component_type()
|
||||
test_bind_rejects_no_component_type()
|
||||
test_bind_with_default_factory()
|
||||
test_bind_optional_flag()
|
||||
test_dependencies_lists_unresolved()
|
||||
test_start_close_idempotent()
|
||||
test_restart()
|
||||
test_async_context_manager()
|
||||
test_close_closes_owned_when_parent_close_fails()
|
||||
test_resolve_standalone_optional_becomes_none()
|
||||
test_resolve_standalone_with_default_factory()
|
||||
test_resolve_standalone_required_no_factory_keeps_placeholder()
|
||||
test_owned_components_started_and_closed()
|
||||
test_resolve_from_context()
|
||||
test_resolve_from_context_optional_missing()
|
||||
test_resolve_from_context_required_missing_raises()
|
||||
test_workspace_path_no_context()
|
||||
test_to_workspace_relative_outside_workspace()
|
||||
test_workspace_metadata_path_no_context()
|
||||
test_component_metadata_path()
|
||||
print("\n所有测试通过!")
|
||||
|
|
|
|||
|
|
@ -82,7 +82,6 @@ def test_add_step_coerces_numeric_inputs():
|
|||
assert resp.success is True
|
||||
assert resp.answer == "3.5"
|
||||
assert resp.metadata["result"] == 3.5
|
||||
print("✓ test_add_step_coerces_numeric_inputs passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -95,7 +94,6 @@ def test_add_step_rejects_invalid_inputs():
|
|||
resp = await step(a="one", b=2)
|
||||
assert resp.success is False
|
||||
assert "Invalid add arguments" in resp.answer
|
||||
print("✓ test_add_step_rejects_invalid_inputs passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -111,7 +109,6 @@ def test_python_execute_step_returns_printed_stdout(tmp_path):
|
|||
assert resp.answer == f"{tmp_path.name}\n"
|
||||
assert resp.metadata["returncode"] == 0
|
||||
assert resp.metadata["stderr"] == ""
|
||||
print("✓ test_python_execute_step_returns_printed_stdout passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -126,7 +123,6 @@ def test_python_execute_step_reports_stderr_on_failure():
|
|||
assert resp.metadata["returncode"] != 0
|
||||
assert "RuntimeError: boom" in resp.answer
|
||||
assert "RuntimeError: boom" in resp.metadata["stderr"]
|
||||
print("✓ test_python_execute_step_reports_stderr_on_failure passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -140,7 +136,6 @@ def test_python_execute_step_times_out():
|
|||
assert resp.success is False
|
||||
assert resp.answer == "Python execution timed out after 0.01s"
|
||||
assert resp.metadata["python_timeout"] == 0.01
|
||||
print("✓ test_python_execute_step_times_out passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -154,7 +149,6 @@ def test_python_execute_step_accepts_legacy_timeout_name():
|
|||
assert resp.success is False
|
||||
assert resp.answer == "Python execution timed out after 0.01s"
|
||||
assert resp.metadata["python_timeout"] == 0.01
|
||||
print("✓ test_python_execute_step_accepts_legacy_timeout_name passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -167,7 +161,6 @@ def test_python_execute_step_rejects_invalid_timeout():
|
|||
resp = await step(code="print(1)", python_timeout=0)
|
||||
assert resp.success is False
|
||||
assert resp.answer == "python_timeout must be a positive number"
|
||||
print("✓ test_python_execute_step_rejects_invalid_timeout passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -194,7 +187,6 @@ def test_llm_demo_always_registers_add_tool():
|
|||
assert resp.success is True
|
||||
assert wrapper.last_kwargs["job_tools"] == ["add"]
|
||||
assert "job_tools" not in resp.metadata
|
||||
print("✓ test_llm_demo_always_registers_add_tool passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -217,7 +209,6 @@ def test_file_graph_health_reports_neo4j_cached_counts():
|
|||
assert status["n_nodes"] == 3
|
||||
assert status["n_edges"] == 4
|
||||
assert status["n_virtual"] == 1
|
||||
print("✓ test_file_graph_health_reports_neo4j_cached_counts passed")
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
|
|
@ -248,7 +239,6 @@ def test_traverse_forward_depth_1():
|
|||
assert c_edge["source"] == "a.md"
|
||||
assert c_edge["depth"] == 1
|
||||
await store.close()
|
||||
print("✓ test_traverse_forward_depth_1 passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -292,7 +282,6 @@ def test_traverse_backward_returns_inlinks():
|
|||
("c.md", "b.md"),
|
||||
}
|
||||
await store.close()
|
||||
print("✓ test_traverse_backward_returns_inlinks passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -316,7 +305,6 @@ def test_traverse_depth_2_expands():
|
|||
assert depth_map.get("b.md") == 1
|
||||
assert depth_map.get("c.md") == 2
|
||||
await store.close()
|
||||
print("✓ test_traverse_depth_2_expands passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -339,7 +327,6 @@ def test_traverse_short_seed_yields_empty():
|
|||
# No error, just empty results because "Bob" isn't a graph key.
|
||||
assert payload == []
|
||||
await store.close()
|
||||
print("✓ test_traverse_short_seed_yields_empty passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -355,7 +342,6 @@ def test_traverse_not_found_seed():
|
|||
payload = _edges(step)
|
||||
assert payload == []
|
||||
await store.close()
|
||||
print("✓ test_traverse_not_found_seed passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -380,7 +366,6 @@ def test_traverse_both_directions():
|
|||
("center.md", "downstream.md"),
|
||||
}
|
||||
await store.close()
|
||||
print("✓ test_traverse_both_directions passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -463,17 +448,3 @@ def test_traverse_depth_zero_returns_only_seed_nodes():
|
|||
await store.close()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n=== traverse step tests ===")
|
||||
test_traverse_forward_depth_1()
|
||||
test_traverse_backward_returns_inlinks()
|
||||
test_traverse_depth_2_expands()
|
||||
test_traverse_short_seed_yields_empty()
|
||||
test_traverse_not_found_seed()
|
||||
test_traverse_both_directions()
|
||||
test_traverse_both_preserves_reciprocal_edge_directions()
|
||||
test_traverse_returns_frontmatter_and_unresolved_nodes()
|
||||
test_traverse_depth_zero_returns_only_seed_nodes()
|
||||
print("\n所有测试通过!")
|
||||
|
|
|
|||
|
|
@ -162,20 +162,3 @@ def test_application_registry_copy_remains_mutable():
|
|||
reg.register(_DummyComponent, "runtime-component")
|
||||
assert reg.get(ComponentEnum.FILE_CHUNKER, "runtime-component") is _DummyComponent
|
||||
assert R.get(ComponentEnum.FILE_CHUNKER, "runtime-component") is None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n=== ComponentRegistry Tests ===")
|
||||
test_register_direct_with_explicit_name()
|
||||
test_register_direct_defaults_to_class_name()
|
||||
test_register_decorator()
|
||||
test_register_rejects_missing_component_type()
|
||||
test_register_rejects_empty_name()
|
||||
test_register_rejects_non_class_non_string()
|
||||
test_get_all_returns_copy()
|
||||
test_get_all_unknown_type_returns_empty()
|
||||
test_get_nonexistent_returns_none()
|
||||
test_unregister_existing()
|
||||
test_unregister_missing_returns_false()
|
||||
test_clear()
|
||||
print("\n所有测试通过!")
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ outside the workspace are rejected.
|
|||
|
||||
# pylint: disable=protected-access,redefined-builtin
|
||||
|
||||
from contextlib import chdir as temp_chdir
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
|
|
@ -51,22 +52,6 @@ warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba")
|
|||
warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources")
|
||||
|
||||
|
||||
class temp_chdir:
|
||||
"""Context manager to temporarily chdir into a path and restore on exit."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.old = None
|
||||
|
||||
def __enter__(self):
|
||||
self.old = os.getcwd()
|
||||
os.chdir(self.path)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
os.chdir(self.old)
|
||||
|
||||
|
||||
async def _make_store(files: dict[str, str] | None = None) -> LocalFileStore:
|
||||
"""LocalFileStore seeded with files on disk + registered in the graph."""
|
||||
store = LocalFileStore(name="t", embedding_store="")
|
||||
|
|
@ -130,7 +115,6 @@ def test_stat_indexed_file():
|
|||
assert payload["mime"].startswith("text/")
|
||||
assert payload["frontmatter"] == {"name": "T"}
|
||||
await store.close()
|
||||
print("✓ test_stat_indexed_file passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -149,7 +133,6 @@ def test_stat_directory_fallback():
|
|||
assert payload["type"] == "dir"
|
||||
assert step.context.response.answer == f"stat: topics (dir, mtime {payload['mtime']})"
|
||||
await store.close()
|
||||
print("✓ test_stat_directory_fallback passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -185,7 +168,6 @@ def test_stat_directory_wins_over_same_name_index():
|
|||
assert payload["type"] == "file"
|
||||
assert payload["frontmatter"] == {"name": "index"}
|
||||
await store.close()
|
||||
print("✓ test_stat_directory_wins_over_same_name_index passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -211,7 +193,6 @@ def test_stat_shadowed_extensionless_reports_resolved_path():
|
|||
assert payload["absolute_path"].endswith("LICENSE.md")
|
||||
assert payload["resolved_path"] == "LICENSE.md"
|
||||
await store.close()
|
||||
print("✓ test_stat_shadowed_extensionless_reports_resolved_path passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -232,7 +213,6 @@ def test_stat_extensionless_file_without_md_sibling():
|
|||
assert payload["absolute_path"].endswith("LICENSE")
|
||||
assert "resolved_path" not in payload
|
||||
await store.close()
|
||||
print("✓ test_stat_extensionless_file_without_md_sibling passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -250,7 +230,6 @@ def test_stat_missing_reports_probed_md_path():
|
|||
assert payload["resolved_path"] == "digest/missing.md"
|
||||
assert "digest/missing.md" in str(step.context.response.answer)
|
||||
await store.close()
|
||||
print("✓ test_stat_missing_reports_probed_md_path passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -279,7 +258,6 @@ def test_list_lists_files():
|
|||
assert answer.startswith("Listed 3 file(s) under topics:\n")
|
||||
assert all(f"- {item}" in answer for item in payload["items"])
|
||||
await store.close()
|
||||
print("✓ test_list_lists_files passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -304,7 +282,6 @@ def test_list_empty_directory_has_explicit_answer():
|
|||
assert step.context.response.answer == "No files found under topics."
|
||||
assert _metadata(step) == {"items": [], "count": 0}
|
||||
await store.close()
|
||||
print("✓ test_list_empty_directory_has_explicit_answer passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -327,7 +304,6 @@ def test_list_respects_limit_and_non_recursive():
|
|||
assert payload["count"] == 1
|
||||
assert len(payload["items"]) == 1
|
||||
await store.close()
|
||||
print("✓ test_list_respects_limit_and_non_recursive passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -420,7 +396,6 @@ def test_move_relocates_within_workspace():
|
|||
assert not (Path(tmp) / "daily/2026-05-18/foo/foo.md").exists()
|
||||
assert (Path(tmp) / "knowledge/foo/foo.md").read_text(encoding="utf-8") == "draft"
|
||||
await store.close()
|
||||
print("✓ test_move_relocates_within_workspace passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -442,7 +417,6 @@ def test_move_refuses_overwrite_without_flag():
|
|||
assert "destination exists" in payload.get("error", "")
|
||||
assert (Path(tmp) / "a/x.md").exists()
|
||||
await store.close()
|
||||
print("✓ test_move_refuses_overwrite_without_flag passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -477,7 +451,6 @@ def test_move_default_retargets_inbound_links():
|
|||
assert "[[daily/2026-05-18/draft/draft.md]]" not in notes
|
||||
assert notes.count("[[knowledge/draft/draft.md]]") == 2
|
||||
await store.close()
|
||||
print("✓ test_move_default_retargets_inbound_links passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -508,7 +481,6 @@ def test_move_opt_out_leaves_links_dangling():
|
|||
# link UNCHANGED — caller opted out of retarget
|
||||
assert "[[daily/2026-05-18/draft/draft.md]]" in notes
|
||||
await store.close()
|
||||
print("✓ test_move_opt_out_leaves_links_dangling passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -528,7 +500,6 @@ def test_delete_removes_file():
|
|||
assert payload.get("deleted") is True
|
||||
assert not (Path(tmp) / "knowledge/draft/draft.md").exists()
|
||||
await store.close()
|
||||
print("✓ test_delete_removes_file passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -544,7 +515,6 @@ def test_delete_missing_returns_error():
|
|||
payload = _metadata(step)
|
||||
assert payload["error"] == "not found"
|
||||
await store.close()
|
||||
print("✓ test_delete_missing_returns_error passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -583,7 +553,6 @@ def test_delete_reports_inbound_refs():
|
|||
assert inbound["files_touched"] == 2
|
||||
assert inbound["links_total"] == 3
|
||||
await store.close()
|
||||
print("✓ test_delete_reports_inbound_refs passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -614,7 +583,6 @@ def test_delete_folder_removes_tree():
|
|||
assert not (Path(tmp) / "scratch").exists()
|
||||
assert (Path(tmp) / "keeper/k.md").exists()
|
||||
await store.close()
|
||||
print("✓ test_delete_folder_removes_tree passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -654,7 +622,6 @@ def test_delete_folder_reports_only_external_inbound():
|
|||
b_sources = {row["path"]: row["count"] for row in by_target["doomed/b.md"]["by_file"]}
|
||||
assert b_sources == {"outside/x.md": 1}
|
||||
await store.close()
|
||||
print("✓ test_delete_folder_reports_only_external_inbound passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -676,7 +643,6 @@ def test_delete_folder_empty_has_no_inbound():
|
|||
assert payload["inbound"]["links_total"] == 0
|
||||
assert not (Path(tmp) / "empty").exists()
|
||||
await store.close()
|
||||
print("✓ test_delete_folder_empty_has_no_inbound passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -723,7 +689,6 @@ def test_read_relative_path():
|
|||
assert "# Recipe" in str(resp.answer)
|
||||
assert "flour and water" in str(resp.answer)
|
||||
await store.close()
|
||||
print("✓ test_read_relative_path passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -739,7 +704,6 @@ def test_read_no_suffix_autoappends_md():
|
|||
assert resp.success is True
|
||||
assert "auto-md" in str(resp.answer)
|
||||
await store.close()
|
||||
print("✓ test_read_no_suffix_autoappends_md passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -759,7 +723,6 @@ def test_read_line_range():
|
|||
assert "L1" not in str(resp.answer)
|
||||
assert "L5" not in str(resp.answer)
|
||||
await store.close()
|
||||
print("✓ test_read_line_range passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -775,7 +738,6 @@ def test_read_absolute_path_inside_workspace_accepted():
|
|||
assert resp.success is True
|
||||
assert "x" in str(resp.answer)
|
||||
await store.close()
|
||||
print("✓ test_read_absolute_path_inside_workspace_accepted passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -816,7 +778,6 @@ def test_file_io_ops_reject_outside_workspace_path():
|
|||
assert move_step.context.response.success is False
|
||||
|
||||
await store.close()
|
||||
print("✓ test_file_io_ops_reject_outside_workspace_path passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -832,7 +793,6 @@ def test_read_non_md_degraded():
|
|||
assert resp.success is True
|
||||
assert "plain-text body" in str(resp.answer)
|
||||
await store.close()
|
||||
print("✓ test_read_non_md_degraded passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -847,7 +807,6 @@ def test_read_missing_file():
|
|||
assert resp.success is False
|
||||
assert "does not exist" in str(resp.answer).lower()
|
||||
await store.close()
|
||||
print("✓ test_read_missing_file passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -863,7 +822,6 @@ def test_read_start_after_end():
|
|||
assert resp.success is False
|
||||
assert "start_line" in str(resp.answer)
|
||||
await store.close()
|
||||
print("✓ test_read_start_after_end passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -879,7 +837,6 @@ def test_read_start_line_exceeds_total():
|
|||
assert resp.success is False
|
||||
assert "exceeds" in str(resp.answer).lower()
|
||||
await store.close()
|
||||
print("✓ test_read_start_line_exceeds_total passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -912,7 +869,6 @@ def test_read_trailing_newline_does_not_inflate_total_lines():
|
|||
assert str(in_range.answer).strip() == "line3"
|
||||
|
||||
await store.close()
|
||||
print("✓ test_read_trailing_newline_does_not_inflate_total_lines passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -931,7 +887,6 @@ def test_read_truncation():
|
|||
assert "truncated" in str(resp.answer)
|
||||
assert "start_line=" in str(resp.answer)
|
||||
await store.close()
|
||||
print("✓ test_read_truncation passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -946,7 +901,6 @@ def test_read_empty_path_rejected():
|
|||
assert resp.success is False
|
||||
assert "required" in str(resp.answer).lower()
|
||||
await store.close()
|
||||
print("✓ test_read_empty_path_rejected passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -979,7 +933,6 @@ def test_read_format_session_injected_false_disables_yaml_true():
|
|||
assert "[user @" not in str(step.context.response.answer)
|
||||
|
||||
await store.close()
|
||||
print("✓ test_read_format_session_injected_false_disables_yaml_true passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1033,7 +986,6 @@ def test_write_basic_with_frontmatter():
|
|||
assert "description: a friendly hello note" in on_disk
|
||||
assert "# Hello" in on_disk
|
||||
await store.close()
|
||||
print("✓ test_write_basic_with_frontmatter passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1058,7 +1010,6 @@ def test_write_rejects_invalid_path_components():
|
|||
# Sanity: no `Notes/` directory got created from any of the bad attempts.
|
||||
assert not (Path(tmp) / "Notes").exists()
|
||||
await store.close()
|
||||
print("✓ test_write_rejects_invalid_path_components passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1073,7 +1024,6 @@ def test_write_no_suffix_autoappends_md():
|
|||
assert resp.success is True
|
||||
assert (Path(tmp) / "Notes/My.md").exists()
|
||||
await store.close()
|
||||
print("✓ test_write_no_suffix_autoappends_md passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1093,7 +1043,6 @@ def test_write_overwrites_with_notice():
|
|||
on_disk = (Path(tmp) / "Existing.md").read_text(encoding="utf-8")
|
||||
assert "new" in on_disk and "old" not in on_disk, on_disk
|
||||
await store.close()
|
||||
print("✓ test_write_overwrites_with_notice passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1108,7 +1057,6 @@ def test_write_creates_parent_dirs():
|
|||
assert resp.success is True
|
||||
assert (Path(tmp) / "a/b/c/D.md").exists()
|
||||
await store.close()
|
||||
print("✓ test_write_creates_parent_dirs passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1128,7 +1076,6 @@ def test_write_no_frontmatter_when_all_empty():
|
|||
assert not on_disk.startswith("---"), on_disk
|
||||
assert "# Hello" in on_disk
|
||||
await store.close()
|
||||
print("✓ test_write_no_frontmatter_when_all_empty passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1161,7 +1108,6 @@ def test_write_ignores_arbitrary_extra_fields():
|
|||
assert "tags:" not in on_disk
|
||||
assert "body" in on_disk
|
||||
await store.close()
|
||||
print("✓ test_write_ignores_arbitrary_extra_fields passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1179,7 +1125,6 @@ def test_write_only_description_present():
|
|||
assert "description: just a description" in on_disk
|
||||
assert "name:" not in on_disk
|
||||
await store.close()
|
||||
print("✓ test_write_only_description_present passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1196,7 +1141,6 @@ def test_edit_global_replace():
|
|||
assert "3" in str(resp.answer) # 3 replacements
|
||||
assert (Path(tmp) / "E.md").read_text(encoding="utf-8") == "qux bar qux\nqux\n"
|
||||
await store.close()
|
||||
print("✓ test_edit_global_replace passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1213,7 +1157,6 @@ def test_edit_old_not_found():
|
|||
assert "not found" in str(resp.answer).lower()
|
||||
assert (Path(tmp) / "E.md").read_text(encoding="utf-8") == "hello world\n"
|
||||
await store.close()
|
||||
print("✓ test_edit_old_not_found passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1228,7 +1171,6 @@ def test_edit_missing_file():
|
|||
assert resp.success is False
|
||||
assert "does not exist" in str(resp.answer).lower()
|
||||
await store.close()
|
||||
print("✓ test_edit_missing_file passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1258,7 +1200,6 @@ def test_edit_skips_frontmatter():
|
|||
assert "beta and beta" in on_disk, on_disk
|
||||
assert "alpha and alpha" not in on_disk, on_disk
|
||||
await store.close()
|
||||
print("✓ test_edit_skips_frontmatter passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1277,7 +1218,6 @@ def test_edit_match_only_in_frontmatter_fails():
|
|||
# File untouched.
|
||||
assert (Path(tmp) / "FMOnly.md").read_text(encoding="utf-8") == body
|
||||
await store.close()
|
||||
print("✓ test_edit_match_only_in_frontmatter_fails passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1306,7 +1246,6 @@ def test_write_non_md_skips_frontmatter():
|
|||
assert "name: Greetings" not in on_disk
|
||||
assert on_disk == "# Hello"
|
||||
await store.close()
|
||||
print("✓ test_write_non_md_skips_frontmatter passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1328,7 +1267,6 @@ def test_edit_non_md_full_text():
|
|||
assert "name: replaced" in on_disk, on_disk
|
||||
assert "foo bar foo" in on_disk, on_disk
|
||||
await store.close()
|
||||
print("✓ test_edit_non_md_full_text passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1347,7 +1285,6 @@ def test_read_non_utf8_encoding():
|
|||
assert resp.success is True
|
||||
assert "你好世界" in str(resp.answer)
|
||||
await store.close()
|
||||
print("✓ test_read_non_utf8_encoding passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1369,7 +1306,6 @@ def test_edit_preserves_gbk_encoding():
|
|||
decoded = raw.decode("gbk")
|
||||
assert "替换后" in decoded and "原始内容" not in decoded
|
||||
await store.close()
|
||||
print("✓ test_edit_preserves_gbk_encoding passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1388,7 +1324,6 @@ def test_read_utf8_bom():
|
|||
assert "hello world" in str(resp.answer)
|
||||
assert "" not in str(resp.answer)
|
||||
await store.close()
|
||||
print("✓ test_read_utf8_bom passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -1415,62 +1350,5 @@ def test_all_read_cases_one_store():
|
|||
assert resp.success is True and str(resp.answer).strip() == "L2"
|
||||
|
||||
await store.close()
|
||||
print("✓ test_all_read_cases_one_store passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n=== crud step tests (opaque-byte surface) ===")
|
||||
# stat / list / move / delete
|
||||
test_stat_indexed_file()
|
||||
test_stat_directory_fallback()
|
||||
test_stat_directory_wins_over_same_name_index()
|
||||
test_stat_shadowed_extensionless_reports_resolved_path()
|
||||
test_stat_extensionless_file_without_md_sibling()
|
||||
test_stat_missing_reports_probed_md_path()
|
||||
test_list_lists_files()
|
||||
test_list_respects_limit_and_non_recursive()
|
||||
test_move_relocates_within_workspace()
|
||||
test_move_refuses_overwrite_without_flag()
|
||||
test_move_default_retargets_inbound_links()
|
||||
test_move_opt_out_leaves_links_dangling()
|
||||
test_delete_removes_file()
|
||||
test_delete_missing_returns_error()
|
||||
test_delete_reports_inbound_refs()
|
||||
test_delete_folder_removes_tree()
|
||||
test_delete_folder_reports_only_external_inbound()
|
||||
test_delete_folder_empty_has_no_inbound()
|
||||
print("\n=== crud_md (read) E2E tests ===")
|
||||
test_read_relative_path()
|
||||
test_read_no_suffix_autoappends_md()
|
||||
test_read_line_range()
|
||||
test_read_absolute_path_inside_workspace_accepted()
|
||||
test_file_io_ops_reject_outside_workspace_path()
|
||||
test_read_non_md_degraded()
|
||||
test_read_missing_file()
|
||||
test_read_start_after_end()
|
||||
test_read_start_line_exceeds_total()
|
||||
test_read_trailing_newline_does_not_inflate_total_lines()
|
||||
test_read_truncation()
|
||||
test_read_empty_path_rejected()
|
||||
print("\n=== crud_md (write/edit) E2E tests ===")
|
||||
test_write_basic_with_frontmatter()
|
||||
test_write_no_suffix_autoappends_md()
|
||||
test_write_overwrites_with_notice()
|
||||
test_write_creates_parent_dirs()
|
||||
test_write_no_frontmatter_when_all_empty()
|
||||
test_write_ignores_arbitrary_extra_fields()
|
||||
test_write_only_description_present()
|
||||
test_edit_global_replace()
|
||||
test_edit_old_not_found()
|
||||
test_edit_missing_file()
|
||||
test_edit_skips_frontmatter()
|
||||
test_edit_match_only_in_frontmatter_fails()
|
||||
print("\n=== crud_md (non-md degraded mode) E2E tests ===")
|
||||
test_write_non_md_skips_frontmatter()
|
||||
test_edit_non_md_full_text()
|
||||
test_read_non_utf8_encoding()
|
||||
test_edit_preserves_gbk_encoding()
|
||||
test_read_utf8_bom()
|
||||
print("\n所有测试通过!")
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@ to the plugin layer.
|
|||
|
||||
# pylint: disable=protected-access
|
||||
|
||||
from contextlib import chdir as temp_chdir
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
|
|
@ -42,22 +42,6 @@ warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba")
|
|||
warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources")
|
||||
|
||||
|
||||
class temp_chdir:
|
||||
"""Context manager: chdir into a path on enter, restore on exit."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.old = None
|
||||
|
||||
def __enter__(self):
|
||||
self.old = os.getcwd()
|
||||
os.chdir(self.path)
|
||||
return self
|
||||
|
||||
def __exit__(self, *_):
|
||||
os.chdir(self.old)
|
||||
|
||||
|
||||
def _today() -> str:
|
||||
from reme.steps.evolve import now
|
||||
|
||||
|
|
@ -131,7 +115,6 @@ def test_daily_list_default_date_is_today():
|
|||
assert f"daily/{_today()}/today-a.md" in answer
|
||||
assert f"daily/{_today()}/today-b.md" in answer
|
||||
await store.close()
|
||||
print("✓ test_daily_list_default_date_is_today passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -155,7 +138,6 @@ def test_daily_list_filters_by_date():
|
|||
answer = step.context.response.answer
|
||||
assert "daily/2026-05-18/a.md" in answer
|
||||
await store.close()
|
||||
print("✓ test_daily_list_filters_by_date passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -195,7 +177,6 @@ def test_daily_list_returns_path_and_frontmatter_notes():
|
|||
assert "session-123" in answer
|
||||
assert "[[sessions/dialog/session-123.jsonl]]" in answer
|
||||
await store.close()
|
||||
print("✓ test_daily_list_returns_path_and_frontmatter_notes passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -224,7 +205,6 @@ def test_daily_list_ignores_subdirectories():
|
|||
answer = step.context.response.answer
|
||||
assert "daily/2026-05-18/main.md" in answer
|
||||
await store.close()
|
||||
print("✓ test_daily_list_ignores_subdirectories passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -241,7 +221,6 @@ def test_daily_list_empty_when_no_daily_dir():
|
|||
payload = _metadata(step)
|
||||
assert payload == {"date": "2026-05-18", "count": 0, "notes": []}
|
||||
await store.close()
|
||||
print("✓ test_daily_list_empty_when_no_daily_dir passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -265,7 +244,6 @@ def test_daily_list_does_not_refresh_index():
|
|||
|
||||
assert not index_path.exists(), "daily_list must not refresh the day index — use daily_reindex"
|
||||
await store.close()
|
||||
print("✓ test_daily_list_does_not_refresh_index passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -286,7 +264,6 @@ def test_daily_list_response_shape():
|
|||
assert set(payload.keys()) == {"date", "count", "notes"}
|
||||
assert payload["notes"] == [{"path": "daily/2026-05-18/alpha.md", "name": "alpha"}]
|
||||
await store.close()
|
||||
print("✓ test_daily_list_response_shape passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -328,7 +305,6 @@ def test_daily_write_delegates_to_write_and_refreshes_index():
|
|||
assert index.is_file()
|
||||
assert f"[[daily/{_today()}/project-plan.md]]" in index.read_text(encoding="utf-8")
|
||||
await store.close()
|
||||
print("✓ test_daily_write_delegates_to_write_and_refreshes_index passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -358,7 +334,6 @@ def test_daily_write_accepts_explicit_date():
|
|||
encoding="utf-8",
|
||||
)
|
||||
await store.close()
|
||||
print("✓ test_daily_write_accepts_explicit_date passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -384,7 +359,6 @@ def test_daily_write_rejects_timestamp_as_explicit_date():
|
|||
assert step.context.response.metadata["date"] == "2023-01-19T08:00:00"
|
||||
assert not (Path(tmp) / "daily").exists()
|
||||
await store.close()
|
||||
print("✓ test_daily_write_rejects_timestamp_as_explicit_date passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -418,7 +392,6 @@ def test_daily_write_reserved_metadata_keys_are_overridden():
|
|||
assert post.metadata["source_conversation"] == "[[session/dialog/real-session.jsonl]]"
|
||||
assert post.metadata["priority"] == 2
|
||||
await store.close()
|
||||
print("✓ test_daily_write_reserved_metadata_keys_are_overridden passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -437,7 +410,6 @@ def test_daily_write_rejects_invalid_name_and_session_id():
|
|||
assert step.context.response.success is False
|
||||
assert not (Path(tmp) / "daily" / _today()).exists()
|
||||
await store.close()
|
||||
print("✓ test_daily_write_rejects_invalid_name_and_session_id passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -466,7 +438,6 @@ def test_day_index_lists_each_note():
|
|||
assert "Alpha Project" in text
|
||||
assert "Beta Project" in text
|
||||
await store.close()
|
||||
print("✓ test_day_index_lists_each_note passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -497,7 +468,6 @@ def test_day_index_includes_note_descriptions():
|
|||
)
|
||||
assert "description:" not in text.split("[[daily/2026-05-18/gamma.md]]")[1].split("\n")[0]
|
||||
await store.close()
|
||||
print("✓ test_day_index_includes_note_descriptions passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -531,7 +501,6 @@ def test_day_index_hides_conversation_metadata():
|
|||
assert "source_conversation" not in line
|
||||
assert "[[session/dialog/session-123.jsonl]]" not in line
|
||||
await store.close()
|
||||
print("✓ test_day_index_hides_conversation_metadata passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -554,7 +523,6 @@ def test_day_index_description_is_note_count():
|
|||
assert "description:" in text
|
||||
assert "2 note(s) today." in text
|
||||
await store.close()
|
||||
print("✓ test_day_index_description_is_note_count passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -577,7 +545,6 @@ def test_day_index_description_updates_when_note_count_changes():
|
|||
assert "2 note(s) today." in text
|
||||
assert "1 note(s) today." not in text
|
||||
await store.close()
|
||||
print("✓ test_day_index_description_updates_when_note_count_changes passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -608,7 +575,6 @@ def test_day_index_preserves_user_content_outside_marker():
|
|||
assert "## 我的笔记" in after
|
||||
assert "[[daily/2026-05-18/beta.md]]" in after
|
||||
await store.close()
|
||||
print("✓ test_day_index_preserves_user_content_outside_marker passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -643,7 +609,6 @@ def test_daily_reindex_returns_write_view():
|
|||
assert "[[daily/2026-05-18/alpha.md]]" in text
|
||||
assert "[[daily/2026-05-18/beta.md]]" in text
|
||||
await store.close()
|
||||
print("✓ test_daily_reindex_returns_write_view passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -667,31 +632,5 @@ def test_daily_reindex_created_flag_flips_on_rerun():
|
|||
assert payload_second["created"] is False
|
||||
assert payload_second["notes_count"] == 1
|
||||
await store.close()
|
||||
print("✓ test_daily_reindex_created_flag_flips_on_rerun passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n=== Daily step tests ===")
|
||||
test_daily_list_default_date_is_today()
|
||||
test_daily_list_filters_by_date()
|
||||
test_daily_list_returns_path_and_frontmatter_notes()
|
||||
test_daily_list_ignores_subdirectories()
|
||||
test_daily_list_empty_when_no_daily_dir()
|
||||
test_daily_list_does_not_refresh_index()
|
||||
test_daily_list_response_shape()
|
||||
test_daily_write_delegates_to_write_and_refreshes_index()
|
||||
test_daily_write_accepts_explicit_date()
|
||||
test_daily_write_rejects_timestamp_as_explicit_date()
|
||||
test_daily_write_reserved_metadata_keys_are_overridden()
|
||||
test_daily_write_rejects_invalid_name_and_session_id()
|
||||
test_day_index_lists_each_note()
|
||||
test_day_index_includes_note_descriptions()
|
||||
test_day_index_hides_conversation_metadata()
|
||||
test_day_index_description_is_note_count()
|
||||
test_day_index_description_updates_when_note_count_changes()
|
||||
test_day_index_preserves_user_content_outside_marker()
|
||||
test_daily_reindex_returns_write_view()
|
||||
test_daily_reindex_created_flag_flips_on_rerun()
|
||||
print("\nAll tests passed!")
|
||||
|
|
|
|||
|
|
@ -107,7 +107,6 @@ def test_parse_empty_file():
|
|||
file_node, chunks = await chunker.chunk(temp_path)
|
||||
assert file_node.path == temp_path
|
||||
assert len(chunks) == 0
|
||||
print("✓ test_parse_empty_file passed")
|
||||
finally:
|
||||
os.unlink(temp_path)
|
||||
|
||||
|
|
@ -130,7 +129,6 @@ def test_parse_small_file():
|
|||
assert chunks[0].start_line == 1
|
||||
assert chunks[0].end_line == 3
|
||||
assert chunks[0].text == content
|
||||
print("✓ test_parse_small_file passed")
|
||||
finally:
|
||||
os.unlink(temp_path)
|
||||
|
||||
|
|
@ -154,7 +152,6 @@ def test_parse_chunked_file():
|
|||
assert len(chunks) > 1, f"Expected multiple chunks, got {len(chunks)}"
|
||||
# Verify overlap by checking that consecutive chunks share some content
|
||||
print(f" Created {len(chunks)} chunks")
|
||||
print("✓ test_parse_chunked_file passed")
|
||||
finally:
|
||||
os.unlink(temp_path)
|
||||
|
||||
|
|
@ -175,7 +172,6 @@ def test_parse_with_custom_encoding():
|
|||
_, chunks = await chunker.chunk(temp_path)
|
||||
assert len(chunks) >= 1
|
||||
assert "你好世界" in chunks[0].text
|
||||
print("✓ test_parse_with_custom_encoding passed")
|
||||
finally:
|
||||
os.unlink(temp_path)
|
||||
|
||||
|
|
@ -253,7 +249,6 @@ def test_parse_links_bare():
|
|||
assert link.source_path == "src.md"
|
||||
assert link.target_path == "note"
|
||||
assert link.target_anchor is None
|
||||
print("✓ test_parse_links_bare passed")
|
||||
|
||||
|
||||
def test_parse_links_with_anchor():
|
||||
|
|
@ -262,7 +257,6 @@ def test_parse_links_with_anchor():
|
|||
assert len(links) == 1
|
||||
assert links[0].target_path == "note"
|
||||
assert links[0].target_anchor == "section A"
|
||||
print("✓ test_parse_links_with_anchor passed")
|
||||
|
||||
|
||||
def test_parse_links_alias_dropped():
|
||||
|
|
@ -271,7 +265,6 @@ def test_parse_links_alias_dropped():
|
|||
assert len(links) == 1
|
||||
assert links[0].target_path == "note"
|
||||
assert links[0].target_anchor is None
|
||||
print("✓ test_parse_links_alias_dropped passed")
|
||||
|
||||
|
||||
def test_parse_links_anchor_and_alias():
|
||||
|
|
@ -280,7 +273,6 @@ def test_parse_links_anchor_and_alias():
|
|||
assert len(links) == 1
|
||||
assert links[0].target_path == "note"
|
||||
assert links[0].target_anchor == "sec"
|
||||
print("✓ test_parse_links_anchor_and_alias passed")
|
||||
|
||||
|
||||
def test_parse_links_ignores_legacy_relation_wrappers():
|
||||
|
|
@ -304,7 +296,6 @@ def test_parse_links_multiple_on_one_line():
|
|||
("x", None),
|
||||
("y", "h"),
|
||||
]
|
||||
print("✓ test_parse_links_multiple_on_one_line passed")
|
||||
|
||||
|
||||
def test_parse_wikilink_line_ranges():
|
||||
|
|
@ -336,7 +327,6 @@ def test_parse_links_no_match():
|
|||
assert len(WikilinkHandler.extract_links("no link here :: foo", "src.md")) == 0
|
||||
assert len(WikilinkHandler.extract_links("plain text without brackets", "src.md")) == 0
|
||||
assert len(WikilinkHandler.extract_links("", "src.md")) == 0
|
||||
print("✓ test_parse_links_no_match passed")
|
||||
|
||||
|
||||
def test_parse_links_in_file():
|
||||
|
|
@ -363,7 +353,6 @@ def test_parse_links_in_file():
|
|||
assert file_node.front_matter == FileFrontMatter()
|
||||
assert len(chunks) == 1
|
||||
assert chunks[0].text == content
|
||||
print("✓ test_parse_links_in_file passed")
|
||||
finally:
|
||||
os.unlink(temp_path)
|
||||
|
||||
|
|
@ -388,7 +377,6 @@ def test_parse_links_empty_when_no_content():
|
|||
node2, _ = await chunker.chunk(fm_only_path)
|
||||
assert node1.links == []
|
||||
assert node2.links == []
|
||||
print("✓ test_parse_links_empty_when_no_content passed")
|
||||
finally:
|
||||
os.unlink(empty_path)
|
||||
os.unlink(fm_only_path)
|
||||
|
|
@ -413,28 +401,7 @@ def test_min_chunk_and_overlap_size():
|
|||
try:
|
||||
_, chunks = await chunker.chunk(temp_path)
|
||||
assert len(chunks) == 1
|
||||
print("✓ test_min_chunk_and_overlap_size passed")
|
||||
finally:
|
||||
os.unlink(temp_path)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_parse_empty_file()
|
||||
test_parse_small_file()
|
||||
test_parse_chunked_file()
|
||||
test_parse_with_custom_encoding()
|
||||
test_parse_links_bare()
|
||||
test_parse_links_with_anchor()
|
||||
test_parse_links_alias_dropped()
|
||||
test_parse_links_anchor_and_alias()
|
||||
test_parse_links_ignores_legacy_relation_wrappers()
|
||||
test_parse_links_multiple_on_one_line()
|
||||
test_parse_wikilink_line_ranges()
|
||||
test_parse_markdown_links_are_ignored()
|
||||
test_parse_links_no_match()
|
||||
test_parse_links_in_file()
|
||||
test_parse_links_empty_when_no_content()
|
||||
test_min_chunk_and_overlap_size()
|
||||
print("\n所有测试通过!")
|
||||
|
|
|
|||
|
|
@ -2,93 +2,16 @@
|
|||
|
||||
# pylint: disable=protected-access
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from contextlib import chdir as temp_chdir
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from reme.components.file_store import FaissLocalFileStore, LocalFileStore
|
||||
from reme.schema import FileChunk, FileNode
|
||||
|
||||
from reme.components.file_store import LocalFileStore
|
||||
from reme.steps.index import OptimizeIndexStep
|
||||
|
||||
|
||||
class temp_chdir:
|
||||
"""Temporarily chdir into a test workspace."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.old = None
|
||||
|
||||
def __enter__(self):
|
||||
self.old = os.getcwd()
|
||||
os.chdir(self.path)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
os.chdir(self.old)
|
||||
|
||||
|
||||
class FakeEmbeddingStore:
|
||||
"""Small deterministic embedding provider used by file-store tests."""
|
||||
|
||||
dimensions = 2
|
||||
max_batch_size = 10
|
||||
is_healthy = True
|
||||
|
||||
def _embed(self, text: str) -> np.ndarray:
|
||||
if "beta" in text or "fresh" in text:
|
||||
return np.array([0.0, 1.0], dtype=np.float16)
|
||||
return np.array([1.0, 0.0], dtype=np.float16)
|
||||
|
||||
async def health_check(self, _timeout: float = 2.0) -> bool:
|
||||
"""Report the fake embedding service as healthy."""
|
||||
return True
|
||||
|
||||
async def get_embedding(self, input_text: str, **_kwargs) -> np.ndarray:
|
||||
"""Return a deterministic embedding for a single text."""
|
||||
return self._embed(input_text)
|
||||
|
||||
async def get_node_embeddings(self, nodes: list[FileChunk], **_kwargs) -> list[FileChunk]:
|
||||
"""Attach deterministic embeddings to file chunks."""
|
||||
for chunk_node in nodes:
|
||||
chunk_node.embedding = self._embed(chunk_node.text)
|
||||
return nodes
|
||||
|
||||
|
||||
def run(coro):
|
||||
"""Run an async test body."""
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def node(path: str) -> FileNode:
|
||||
"""Build a minimal file node."""
|
||||
return FileNode(path=path, st_mtime=1.0)
|
||||
|
||||
|
||||
def chunk(chunk_id: str, path: str, text: str, **metadata) -> FileChunk:
|
||||
"""Build a minimal file chunk."""
|
||||
return FileChunk(id=chunk_id, path=path, text=text, start_line=1, end_line=1, metadata=metadata)
|
||||
|
||||
|
||||
def _new_faiss_store(name, **kwargs):
|
||||
"""Construct a FAISS store with embedding disabled at bind time."""
|
||||
try:
|
||||
store = FaissLocalFileStore(name=name, embedding_store="", **kwargs)
|
||||
except ImportError:
|
||||
pytest.skip("faiss is not installed")
|
||||
return store
|
||||
|
||||
|
||||
async def _settle_reindex(store, timeout=5.0):
|
||||
"""Wait until no async reindex is pending or in flight."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while store._reindex_event.is_set() or store._reindex_busy:
|
||||
if time.monotonic() > deadline:
|
||||
raise AssertionError("async reindex did not settle in time")
|
||||
await asyncio.sleep(0.005)
|
||||
from .file_store_test_support import FakeEmbeddingStore, _new_faiss_store, _settle_reindex, chunk, node
|
||||
|
||||
|
||||
async def _seed_unembedded_chunk(store: LocalFileStore, chunk_id: str, path: str, text: str) -> None:
|
||||
|
|
@ -99,259 +22,229 @@ async def _seed_unembedded_chunk(store: LocalFileStore, chunk_id: str, path: str
|
|||
await store.file_graph.upsert_nodes([file_node])
|
||||
|
||||
|
||||
def test_faiss_backfill_adds_incrementally_without_rebuild():
|
||||
@pytest.mark.asyncio
|
||||
async def test_faiss_backfill_adds_incrementally_without_rebuild():
|
||||
"""Backfilled vectors are added to the live index; no full rebuild happens
|
||||
while tombstones stay under the compaction threshold."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_faiss_store("t_faiss_backfill_incr")
|
||||
await store.start()
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
store._faiss_index = store._new_index()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_faiss_store("t_faiss_backfill_incr")
|
||||
await store.start()
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
store._faiss_index = store._new_index()
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await store.upsert([(node("b.md"), [chunk("b", "b.md", "beta text")])])
|
||||
# Same-id text change tombstones the old row (below the 128 floor).
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha updated")])])
|
||||
assert store._tombstones == {0}
|
||||
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await store.upsert([(node("b.md"), [chunk("b", "b.md", "beta text")])])
|
||||
# Same-id text change tombstones the old row (below the 128 floor).
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha updated")])])
|
||||
assert store._tombstones == {0}
|
||||
await _seed_unembedded_chunk(store, "c", "c.md", "alpha extra")
|
||||
await store._backfill_missing_embeddings()
|
||||
|
||||
await _seed_unembedded_chunk(store, "c", "c.md", "alpha extra")
|
||||
await store._backfill_missing_embeddings()
|
||||
|
||||
# The new vector is live; the surviving tombstone proves the index
|
||||
# was extended in place rather than rebuilt.
|
||||
assert set(store._id_to_row) == {"a", "b", "c"}
|
||||
assert store._tombstones == {0}
|
||||
assert store._reindex_worker_task is None
|
||||
assert {c.id for c in await store.vector_search("alpha", 10, {})} == {"a", "b", "c"}
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
# The new vector is live; the surviving tombstone proves the index
|
||||
# was extended in place rather than rebuilt.
|
||||
assert set(store._id_to_row) == {"a", "b", "c"}
|
||||
assert store._tombstones == {0}
|
||||
assert store._reindex_worker_task is None
|
||||
assert {c.id for c in await store.vector_search("alpha", 10, {})} == {"a", "b", "c"}
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_faiss_backfill_compacts_when_tombstones_cross_threshold():
|
||||
@pytest.mark.asyncio
|
||||
async def test_faiss_backfill_compacts_when_tombstones_cross_threshold():
|
||||
"""Tombstone pressure at backfill time still triggers a full rebuild."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_faiss_store("t_faiss_backfill_compact")
|
||||
await store.start()
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
store._faiss_index = store._new_index()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_faiss_store("t_faiss_backfill_compact")
|
||||
await store.start()
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
store._faiss_index = store._new_index()
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha updated")])])
|
||||
assert store._tombstones == {0}
|
||||
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha updated")])])
|
||||
assert store._tombstones == {0}
|
||||
# Lower the threshold only now, so the upsert above did not compact.
|
||||
store.max_tombstones = 1
|
||||
await _seed_unembedded_chunk(store, "c", "c.md", "alpha extra")
|
||||
await store._backfill_missing_embeddings()
|
||||
|
||||
# Lower the threshold only now, so the upsert above did not compact.
|
||||
store.max_tombstones = 1
|
||||
await _seed_unembedded_chunk(store, "c", "c.md", "alpha extra")
|
||||
await store._backfill_missing_embeddings()
|
||||
|
||||
assert store._tombstones == set()
|
||||
assert set(store._id_to_row) == {"a", "c"}
|
||||
assert {c.id for c in await store.vector_search("alpha", 10, {})} == {"a", "c"}
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert store._tombstones == set()
|
||||
assert set(store._id_to_row) == {"a", "c"}
|
||||
assert {c.id for c in await store.vector_search("alpha", 10, {})} == {"a", "c"}
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_faiss_backfill_mass_delta_uses_async_reindex():
|
||||
@pytest.mark.asyncio
|
||||
async def test_faiss_backfill_mass_delta_uses_async_reindex():
|
||||
"""With async_reindex, a delta that rivals the live rows (initial mass
|
||||
backfill into an empty index) goes through the background worker."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_faiss_store("t_faiss_backfill_async_mass", async_reindex=True)
|
||||
await store.start()
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
store._faiss_index = store._new_index()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_faiss_store("t_faiss_backfill_async_mass", async_reindex=True)
|
||||
await store.start()
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
store._faiss_index = store._new_index()
|
||||
await _seed_unembedded_chunk(store, "a", "a.md", "alpha text")
|
||||
await _seed_unembedded_chunk(store, "b", "b.md", "beta text")
|
||||
await store._backfill_missing_embeddings()
|
||||
|
||||
await _seed_unembedded_chunk(store, "a", "a.md", "alpha text")
|
||||
await _seed_unembedded_chunk(store, "b", "b.md", "beta text")
|
||||
await store._backfill_missing_embeddings()
|
||||
|
||||
assert store._reindex_worker_task is not None # routed off-loop
|
||||
await _settle_reindex(store)
|
||||
assert set(store._id_to_row) == {"a", "b"}
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})][0] == "a"
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert store._reindex_worker_task is not None # routed off-loop
|
||||
await _settle_reindex(store)
|
||||
assert set(store._id_to_row) == {"a", "b"}
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})][0] == "a"
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_faiss_backfill_small_delta_adds_inline_in_async_mode():
|
||||
@pytest.mark.asyncio
|
||||
async def test_faiss_backfill_small_delta_adds_inline_in_async_mode():
|
||||
"""With async_reindex, a small delta is added inline without a worker."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_faiss_store("t_faiss_backfill_async_small", async_reindex=True)
|
||||
await store.start()
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
store._faiss_index = store._new_index()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_faiss_store("t_faiss_backfill_async_small", async_reindex=True)
|
||||
await store.start()
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
store._faiss_index = store._new_index()
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await store.upsert([(node("b.md"), [chunk("b", "b.md", "beta text")])])
|
||||
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await store.upsert([(node("b.md"), [chunk("b", "b.md", "beta text")])])
|
||||
await _seed_unembedded_chunk(store, "c", "c.md", "alpha extra")
|
||||
await store._backfill_missing_embeddings()
|
||||
|
||||
await _seed_unembedded_chunk(store, "c", "c.md", "alpha extra")
|
||||
await store._backfill_missing_embeddings()
|
||||
|
||||
# 1 new vector < 2 live rows -> inline add, no background worker.
|
||||
assert store._reindex_worker_task is None
|
||||
assert set(store._id_to_row) == {"a", "b", "c"}
|
||||
assert {c.id for c in await store.vector_search("alpha", 10, {})} >= {"a", "c"}
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
# 1 new vector < 2 live rows -> inline add, no background worker.
|
||||
assert store._reindex_worker_task is None
|
||||
assert set(store._id_to_row) == {"a", "b", "c"}
|
||||
assert {c.id for c in await store.vector_search("alpha", 10, {})} >= {"a", "c"}
|
||||
await store.close()
|
||||
|
||||
|
||||
# -- optimize_index (idle-time maintenance) -------------------------------------
|
||||
|
||||
|
||||
def test_tombstone_threshold_scales_and_honors_override():
|
||||
@pytest.mark.asyncio
|
||||
async def test_tombstone_threshold_scales_and_honors_override():
|
||||
"""Threshold math: full vs half scale, 128 floor, and max_tombstones override."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_faiss_store("t_faiss_threshold")
|
||||
await store.start()
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
store._faiss_index = type("FakeIndex", (), {"ntotal": 1000})()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_faiss_store("t_faiss_threshold")
|
||||
await store.start()
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
store._faiss_index = type("FakeIndex", (), {"ntotal": 1000})()
|
||||
assert store._tombstone_threshold() == 300 # 0.3 * 1000
|
||||
assert store._tombstone_threshold(scale=0.5) == 150 # half the write-path bar
|
||||
store._faiss_index = type("FakeIndex", (), {"ntotal": 100})()
|
||||
assert store._tombstone_threshold(scale=0.5) == 128 # floor dominates
|
||||
|
||||
assert store._tombstone_threshold() == 300 # 0.3 * 1000
|
||||
assert store._tombstone_threshold(scale=0.5) == 150 # half the write-path bar
|
||||
store._faiss_index = type("FakeIndex", (), {"ntotal": 100})()
|
||||
assert store._tombstone_threshold(scale=0.5) == 128 # floor dominates
|
||||
store.max_tombstones = 7
|
||||
assert store._tombstone_threshold() == 7
|
||||
assert store._tombstone_threshold(scale=0.5) == 7 # fixed override ignores scale
|
||||
|
||||
store.max_tombstones = 7
|
||||
assert store._tombstone_threshold() == 7
|
||||
assert store._tombstone_threshold(scale=0.5) == 7 # fixed override ignores scale
|
||||
|
||||
store._faiss_index = None # avoid persisting the fake index on close
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
store._faiss_index = None # avoid persisting the fake index on close
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_local_store_optimize_index_is_noop():
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_store_optimize_index_is_noop():
|
||||
"""LocalFileStore.optimize_index() completes without touching store state."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = LocalFileStore(name="t_local_optimize", embedding_store="")
|
||||
await store.start()
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = LocalFileStore(name="t_local_optimize", embedding_store="")
|
||||
await store.start()
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await store.optimize_index()
|
||||
|
||||
await store.optimize_index()
|
||||
|
||||
assert set(store.file_chunks) == {"a"}
|
||||
assert [c.id for c in await store.keyword_search("alpha", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert set(store.file_chunks) == {"a"}
|
||||
assert [c.id for c in await store.keyword_search("alpha", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_faiss_optimize_index_noop_below_threshold():
|
||||
@pytest.mark.asyncio
|
||||
async def test_faiss_optimize_index_noop_below_threshold():
|
||||
"""optimize_index() keeps tombstones when they are under the half bar."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_faiss_store("t_faiss_optimize_noop")
|
||||
await store.start()
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
store._faiss_index = store._new_index()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_faiss_store("t_faiss_optimize_noop")
|
||||
await store.start()
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
store._faiss_index = store._new_index()
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha updated")])])
|
||||
assert store._tombstones == {0}
|
||||
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha updated")])])
|
||||
assert store._tombstones == {0}
|
||||
await store.optimize_index() # 1 tombstone <= 128 floor -> untouched
|
||||
|
||||
await store.optimize_index() # 1 tombstone <= 128 floor -> untouched
|
||||
|
||||
assert store._tombstones == {0}
|
||||
assert store._reindex_worker_task is None
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert store._tombstones == {0}
|
||||
assert store._reindex_worker_task is None
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_faiss_optimize_index_compacts_inline_when_sync():
|
||||
@pytest.mark.asyncio
|
||||
async def test_faiss_optimize_index_compacts_inline_when_sync():
|
||||
"""optimize_index() rebuilds inline once tombstones exceed the (overridden) bar."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_faiss_store("t_faiss_optimize_sync")
|
||||
await store.start()
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
store._faiss_index = store._new_index()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_faiss_store("t_faiss_optimize_sync")
|
||||
await store.start()
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
store._faiss_index = store._new_index()
|
||||
files = [(node(f"n{i}.md"), [chunk(f"c{i}", f"n{i}.md", "alpha text")]) for i in range(4)]
|
||||
await store.upsert(files)
|
||||
updated = [(node(f"n{i}.md"), [chunk(f"c{i}", f"n{i}.md", f"alpha v2 {i}")]) for i in range(3)]
|
||||
await store.upsert(updated)
|
||||
assert len(store._tombstones) == 3
|
||||
|
||||
files = [(node(f"n{i}.md"), [chunk(f"c{i}", f"n{i}.md", "alpha text")]) for i in range(4)]
|
||||
await store.upsert(files)
|
||||
updated = [(node(f"n{i}.md"), [chunk(f"c{i}", f"n{i}.md", f"alpha v2 {i}")]) for i in range(3)]
|
||||
await store.upsert(updated)
|
||||
assert len(store._tombstones) == 3
|
||||
store.max_tombstones = 2 # lower the bar only for optimize_index
|
||||
await store.optimize_index()
|
||||
|
||||
store.max_tombstones = 2 # lower the bar only for optimize_index
|
||||
await store.optimize_index()
|
||||
|
||||
assert store._tombstones == set()
|
||||
assert set(store._id_to_row) == {"c0", "c1", "c2", "c3"}
|
||||
assert store._reindex_worker_task is None # inline path
|
||||
assert len(await store.vector_search("alpha", 10, {})) == 4
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert store._tombstones == set()
|
||||
assert set(store._id_to_row) == {"c0", "c1", "c2", "c3"}
|
||||
assert store._reindex_worker_task is None # inline path
|
||||
assert len(await store.vector_search("alpha", 10, {})) == 4
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_faiss_optimize_index_uses_worker_when_async():
|
||||
@pytest.mark.asyncio
|
||||
async def test_faiss_optimize_index_uses_worker_when_async():
|
||||
"""optimize_index() submits the rebuild to the background worker under async_reindex."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_faiss_store("t_faiss_optimize_async", async_reindex=True)
|
||||
await store.start()
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
store._faiss_index = store._new_index()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_faiss_store("t_faiss_optimize_async", async_reindex=True)
|
||||
await store.start()
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
store._faiss_index = store._new_index()
|
||||
files = [(node(f"n{i}.md"), [chunk(f"c{i}", f"n{i}.md", "alpha text")]) for i in range(4)]
|
||||
await store.upsert(files)
|
||||
updated = [(node(f"n{i}.md"), [chunk(f"c{i}", f"n{i}.md", f"alpha v2 {i}")]) for i in range(3)]
|
||||
await store.upsert(updated)
|
||||
assert len(store._tombstones) == 3
|
||||
|
||||
files = [(node(f"n{i}.md"), [chunk(f"c{i}", f"n{i}.md", "alpha text")]) for i in range(4)]
|
||||
await store.upsert(files)
|
||||
updated = [(node(f"n{i}.md"), [chunk(f"c{i}", f"n{i}.md", f"alpha v2 {i}")]) for i in range(3)]
|
||||
await store.upsert(updated)
|
||||
assert len(store._tombstones) == 3
|
||||
store.max_tombstones = 2
|
||||
await store.optimize_index()
|
||||
|
||||
store.max_tombstones = 2
|
||||
await store.optimize_index()
|
||||
|
||||
assert store._reindex_worker_task is not None # routed off-loop
|
||||
await _settle_reindex(store)
|
||||
assert store._tombstones == set()
|
||||
assert set(store._id_to_row) == {"c0", "c1", "c2", "c3"}
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert store._reindex_worker_task is not None # routed off-loop
|
||||
await _settle_reindex(store)
|
||||
assert store._tombstones == set()
|
||||
assert set(store._id_to_row) == {"c0", "c1", "c2", "c3"}
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_optimize_index_step_calls_file_store_optimize_index():
|
||||
@pytest.mark.asyncio
|
||||
async def test_optimize_index_step_calls_file_store_optimize_index():
|
||||
"""The cron-facing step delegates to file_store.optimize_index() and reports success."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = LocalFileStore(name="t_optimize_step", embedding_store="")
|
||||
await store.start()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = LocalFileStore(name="t_optimize_step", embedding_store="")
|
||||
await store.start()
|
||||
calls = []
|
||||
original_optimize = store.optimize_index
|
||||
|
||||
calls = []
|
||||
original_optimize = store.optimize_index
|
||||
async def counting_optimize():
|
||||
calls.append(True)
|
||||
await original_optimize()
|
||||
|
||||
async def counting_optimize():
|
||||
calls.append(True)
|
||||
await original_optimize()
|
||||
store.optimize_index = counting_optimize
|
||||
step = OptimizeIndexStep(file_store=store)
|
||||
await step()
|
||||
|
||||
store.optimize_index = counting_optimize
|
||||
step = OptimizeIndexStep(file_store=store)
|
||||
await step()
|
||||
|
||||
assert calls == [True]
|
||||
assert step.context.response.metadata["optimized_index"] is True
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert calls == [True]
|
||||
assert step.context.response.metadata["optimized_index"] is True
|
||||
await store.close()
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@
|
|||
|
||||
# pylint: disable=protected-access
|
||||
|
||||
from contextlib import chdir as temp_chdir
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
|
|
@ -13,22 +13,6 @@ from reme.schema import FileNode
|
|||
from reme.utils.jsonl_zst import write_jsonl_zst
|
||||
|
||||
|
||||
class temp_chdir:
|
||||
"""Context manager to temporarily chdir into a path and restore on exit."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.old = None
|
||||
|
||||
def __enter__(self):
|
||||
self.old = os.getcwd()
|
||||
os.chdir(self.path)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
os.chdir(self.old)
|
||||
|
||||
|
||||
def make_node(path: str, mtime: float = 1.0) -> FileNode:
|
||||
"""Build a FileNode with the given path and mtime for fixture use."""
|
||||
return FileNode(path=path, st_mtime=mtime)
|
||||
|
|
@ -172,14 +156,3 @@ def test_start_failure_does_not_overwrite_persisted_catalog(tmp_path):
|
|||
assert catalog._catalog_file.read_bytes() == original_bytes
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n=== FileCatalog Tests ===")
|
||||
for backend in BACKENDS:
|
||||
test_upsert_and_get_nodes(backend)
|
||||
test_upsert_replaces_existing(backend)
|
||||
test_delete_single_and_list(backend)
|
||||
test_get_nodes_empty_inputs(backend)
|
||||
test_persistence_roundtrip(backend)
|
||||
print("\n所有测试通过!")
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@
|
|||
|
||||
# pylint: disable=protected-access
|
||||
|
||||
from contextlib import chdir as temp_chdir
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
|
|
@ -13,22 +13,6 @@ from reme.enumeration import LinkScopeEnum
|
|||
from reme.schema import FileLink, FileNode
|
||||
|
||||
|
||||
class temp_chdir:
|
||||
"""Context manager to temporarily chdir into a path and restore on exit."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.old = None
|
||||
|
||||
def __enter__(self):
|
||||
self.old = os.getcwd()
|
||||
os.chdir(self.path)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
os.chdir(self.old)
|
||||
|
||||
|
||||
def make_node(path: str, links: list[tuple[str, str | None]] | None = None) -> FileNode:
|
||||
"""Build a FileNode with the given outgoing (target_path, target_anchor) pairs."""
|
||||
return FileNode(
|
||||
|
|
@ -436,19 +420,3 @@ def test_get_nodes_empty_inputs(backend_cls):
|
|||
print(f"✓ test_get_nodes_empty_inputs[{backend_cls.__name__}] passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n=== FileGraph Tests ===")
|
||||
for backend in BACKENDS:
|
||||
test_upsert_and_get_nodes(backend)
|
||||
test_upsert_replaces_old_links(backend)
|
||||
test_outlinks_include_virtual_targets(backend)
|
||||
test_inlinks_visible_for_virtual_target(backend)
|
||||
test_delete_keeps_inbound_view(backend)
|
||||
test_delete_outgoing_links_cleared(backend)
|
||||
test_clear(backend)
|
||||
test_rebuild_links_idempotent(backend)
|
||||
test_persistence_roundtrip(backend)
|
||||
test_get_nodes_empty_inputs(backend)
|
||||
print("\n所有测试通过!")
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -10,7 +10,7 @@ probed path in not-found errors instead of the caller's raw input.
|
|||
|
||||
# pylint: disable=protected-access
|
||||
|
||||
import os
|
||||
from contextlib import chdir as temp_chdir
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -23,22 +23,6 @@ from reme.steps.file_io.frontmatter_read import FrontmatterReadStep
|
|||
from reme.steps.file_io.frontmatter_update import FrontmatterUpdateStep
|
||||
|
||||
|
||||
class temp_chdir:
|
||||
"""Context manager to temporarily chdir into a path and restore on exit."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.old = None
|
||||
|
||||
def __enter__(self):
|
||||
self.old = os.getcwd()
|
||||
os.chdir(self.path)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
os.chdir(self.old)
|
||||
|
||||
|
||||
def _seed(workspace: Path, rel: str, body: str) -> Path:
|
||||
target = workspace / rel
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
|
|
|||
|
|
@ -267,7 +267,3 @@ def test_configs_define_original_jobs_without_daily_variants():
|
|||
{"backend": "auto_memory_cc_step"},
|
||||
{"backend": "auto_tag_step"},
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
|
|
|||
|
|
@ -45,7 +45,6 @@ def test_empty_file():
|
|||
node, chunks = _run(chunker.chunk(path))
|
||||
assert len(chunks) == 0
|
||||
assert node.links == []
|
||||
print("✓ test_empty_file passed")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
|
@ -59,7 +58,6 @@ def test_blank_lines_only():
|
|||
chunker = JsonlFileChunker()
|
||||
_, chunks = _run(chunker.chunk(path))
|
||||
assert len(chunks) == 0
|
||||
print("✓ test_blank_lines_only passed")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
|
@ -75,7 +73,6 @@ def test_single_line():
|
|||
assert chunks[0].start_line == 1
|
||||
assert chunks[0].end_line == 1
|
||||
assert lines[0] in chunks[0].text
|
||||
print("✓ test_single_line passed")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
|
@ -106,7 +103,6 @@ def test_all_lines_fit_in_one_chunk():
|
|||
assert len(chunks) == 1
|
||||
assert chunks[0].start_line == 1
|
||||
assert chunks[0].end_line == 5
|
||||
print("✓ test_all_lines_fit_in_one_chunk passed")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
|
@ -148,7 +144,6 @@ def test_multiple_chunks_no_overlap():
|
|||
assert chunks[0].start_line == 1
|
||||
assert chunks[-1].end_line == 10
|
||||
print(f" Created {len(chunks)} chunks")
|
||||
print("✓ test_multiple_chunks_no_overlap passed")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
|
@ -165,7 +160,6 @@ def test_line_aligned_no_intra_line_split():
|
|||
# Each chunk text should be a concatenation of whole lines.
|
||||
text_lines = c.text.splitlines()
|
||||
assert len(text_lines) == (c.end_line - c.start_line + 1)
|
||||
print("✓ test_line_aligned_no_intra_line_split passed")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
|
@ -193,7 +187,6 @@ def test_overlap_lines_shared():
|
|||
# Last chunk still reaches the end of the file.
|
||||
assert chunks[-1].end_line == 12
|
||||
print(f" Created {len(chunks)} chunks with overlap")
|
||||
print("✓ test_overlap_lines_shared passed")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
|
@ -207,7 +200,6 @@ def test_zero_overlap():
|
|||
_, chunks = _run(chunker.chunk(path))
|
||||
for i in range(len(chunks) - 1):
|
||||
assert chunks[i].end_line < chunks[i + 1].start_line
|
||||
print("✓ test_zero_overlap passed")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
|
@ -228,7 +220,6 @@ def test_overlap_capped_by_max_overlap_chars():
|
|||
overlap_lines = list(range(overlap_start, overlap_end + 1))
|
||||
overlap_text = "".join(lines[ln - 1] for ln in overlap_lines)
|
||||
assert len(overlap_text) <= max_overlap, f"Overlap size {len(overlap_text)} > {max_overlap}"
|
||||
print("✓ test_overlap_capped_by_max_overlap_chars passed")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
|
@ -255,7 +246,6 @@ def test_single_line_exceeds_max():
|
|||
break
|
||||
assert found, "Long line was not found in any chunk"
|
||||
assert len(chunks) >= 2 # At least some splitting happened.
|
||||
print("✓ test_single_line_exceeds_max passed")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
|
@ -290,7 +280,6 @@ def test_byte_mode():
|
|||
f"than char mode ({len(chunks_chars)}) for CJK text"
|
||||
)
|
||||
print(f" char mode: {len(chunks_chars)} chunks, byte mode: {len(chunks_bytes)} chunks")
|
||||
print("✓ test_byte_mode passed")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
|
@ -302,7 +291,6 @@ def test_invalid_mode_raises():
|
|||
assert False, "Should have raised ValueError"
|
||||
except ValueError:
|
||||
pass
|
||||
print("✓ test_invalid_mode_raises passed")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -322,7 +310,6 @@ def test_file_chunk_properties():
|
|||
assert c.end_line >= c.start_line
|
||||
assert c.id and len(c.id) > 0
|
||||
assert c.text.strip()
|
||||
print("✓ test_file_chunk_properties passed")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
|
@ -337,7 +324,6 @@ def test_file_node_properties():
|
|||
assert node.st_mtime > 0
|
||||
assert node.links == []
|
||||
assert node.chunk_ids == [c.id for c in chunks]
|
||||
print("✓ test_file_node_properties passed")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
|
@ -352,7 +338,6 @@ def test_hash_id_deterministic():
|
|||
c2 = JsonlFileChunker(max_chars=100)
|
||||
_, chunks2 = _run(c2.chunk(path))
|
||||
assert [c.id for c in chunks1] == [c.id for c in chunks2]
|
||||
print("✓ test_hash_id_deterministic passed")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
|
@ -366,14 +351,12 @@ def test_max_chars_floor():
|
|||
"""max_chars is clamped to at least 64."""
|
||||
c = JsonlFileChunker(max_chars=1)
|
||||
assert c.max_chars == 64
|
||||
print("✓ test_max_chars_floor passed")
|
||||
|
||||
|
||||
def test_max_overlap_floor():
|
||||
"""Negative max_overlap_chars is clamped to 0."""
|
||||
c = JsonlFileChunker(max_overlap_chars=-10)
|
||||
assert c.max_overlap_chars == 0
|
||||
print("✓ test_max_overlap_floor passed")
|
||||
|
||||
|
||||
def test_full_coverage():
|
||||
|
|
@ -387,7 +370,6 @@ def test_full_coverage():
|
|||
for c in chunks:
|
||||
covered.update(range(c.start_line, c.end_line + 1))
|
||||
assert covered == set(range(1, 26)), f"Missing lines: {set(range(1, 26)) - covered}"
|
||||
print("✓ test_full_coverage passed")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
|
@ -406,7 +388,6 @@ def test_chunk_lines_algorithm_simple():
|
|||
# 3 lines = 63 chars ≤ 64 → chunk 1: (0,3). 4th line alone → chunk 2: (3,4).
|
||||
ranges = chunker._chunk_lines(lines)
|
||||
assert ranges == [(0, 3), (3, 4)], f"Got {ranges}"
|
||||
print("✓ test_chunk_lines_algorithm_simple passed")
|
||||
|
||||
|
||||
def test_chunk_lines_algorithm_with_overlap():
|
||||
|
|
@ -422,34 +403,8 @@ def test_chunk_lines_algorithm_with_overlap():
|
|||
assert ranges[i][1] >= ranges[i + 1][0], f"No overlap between range {i} {ranges[i]} and {i+1} {ranges[i+1]}"
|
||||
# Last range reaches end.
|
||||
assert ranges[-1][1] == 6
|
||||
print("✓ test_chunk_lines_algorithm_with_overlap passed")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Runner
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_empty_file()
|
||||
test_blank_lines_only()
|
||||
test_single_line()
|
||||
test_unicode_line_separator_inside_json_string_is_not_a_record_boundary()
|
||||
test_all_lines_fit_in_one_chunk()
|
||||
test_multiple_chunks_no_overlap()
|
||||
test_line_aligned_no_intra_line_split()
|
||||
test_overlap_lines_shared()
|
||||
test_zero_overlap()
|
||||
test_overlap_capped_by_max_overlap_chars()
|
||||
test_single_line_exceeds_max()
|
||||
test_byte_mode()
|
||||
test_invalid_mode_raises()
|
||||
test_file_chunk_properties()
|
||||
test_file_node_properties()
|
||||
test_hash_id_deterministic()
|
||||
test_max_chars_floor()
|
||||
test_max_overlap_floor()
|
||||
test_full_coverage()
|
||||
test_chunk_lines_algorithm_simple()
|
||||
test_chunk_lines_algorithm_with_overlap()
|
||||
print("\n所有测试通过!")
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -11,8 +11,8 @@ Two pure helpers:
|
|||
|
||||
# pylint: disable=protected-access
|
||||
|
||||
from contextlib import chdir as temp_chdir
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
|
|
@ -26,22 +26,6 @@ warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba")
|
|||
warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources")
|
||||
|
||||
|
||||
class temp_chdir:
|
||||
"""Test helper: chdir to ``path`` on enter, restore previous cwd on exit."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.old = None
|
||||
|
||||
def __enter__(self):
|
||||
self.old = os.getcwd()
|
||||
os.chdir(self.path)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
os.chdir(self.old)
|
||||
|
||||
|
||||
async def _store_with(files: dict[str, dict]) -> LocalFileStore:
|
||||
"""LocalFileStore seeded with files + parsed wikilinks + optional frontmatter meta.
|
||||
|
||||
|
|
@ -89,7 +73,6 @@ def test_expand_links_empty_paths_short_circuits():
|
|||
result = await expand_links(store, [])
|
||||
assert result == {}
|
||||
await store.close()
|
||||
print("✓ test_expand_links_empty_paths_short_circuits passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -123,7 +106,6 @@ def test_expand_links_returns_outlinks_and_inlinks_with_meta():
|
|||
assert result["B.md"]["outlinks"] == []
|
||||
|
||||
await store.close()
|
||||
print("✓ test_expand_links_returns_outlinks_and_inlinks_with_meta passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -149,7 +131,6 @@ def test_expand_links_max_per_direction_caps_neighbors():
|
|||
assert len(out) == 2
|
||||
assert [n["path"] for n in out] == ["a.md", "b.md"]
|
||||
await store.close()
|
||||
print("✓ test_expand_links_max_per_direction_caps_neighbors passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -168,7 +149,6 @@ def test_expand_links_node_without_meta_returns_empty_meta_dict():
|
|||
result = await expand_links(store, ["src.md"])
|
||||
assert result["src.md"]["outlinks"][0]["meta"] == {}
|
||||
await store.close()
|
||||
print("✓ test_expand_links_node_without_meta_returns_empty_meta_dict passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -180,7 +160,6 @@ def test_render_expansion_lines_empty_input_yields_empty_list():
|
|||
"""Both directions empty ⇒ no lines."""
|
||||
assert not render_expansion_lines({})
|
||||
assert not render_expansion_lines({"outlinks": [], "inlinks": []})
|
||||
print("✓ test_render_expansion_lines_empty_input_yields_empty_list passed")
|
||||
|
||||
|
||||
def test_render_expansion_lines_outlinks_only():
|
||||
|
|
@ -200,7 +179,6 @@ def test_render_expansion_lines_outlinks_only():
|
|||
" outlinks (1):",
|
||||
' → B.md name="B" description="beta"',
|
||||
]
|
||||
print("✓ test_render_expansion_lines_outlinks_only passed")
|
||||
|
||||
|
||||
def test_render_expansion_lines_inlinks_only_with_anchor():
|
||||
|
|
@ -221,7 +199,6 @@ def test_render_expansion_lines_inlinks_only_with_anchor():
|
|||
" ← src.md (no meta)",
|
||||
" via anchor=#intro",
|
||||
]
|
||||
print("✓ test_render_expansion_lines_inlinks_only_with_anchor passed")
|
||||
|
||||
|
||||
def test_render_expansion_lines_both_directions_in_order():
|
||||
|
|
@ -239,15 +216,3 @@ def test_render_expansion_lines_both_directions_in_order():
|
|||
assert lines[2] == " inlinks (1):"
|
||||
assert lines[1].lstrip().startswith("→")
|
||||
assert lines[3].lstrip().startswith("←")
|
||||
print("✓ test_render_expansion_lines_both_directions_in_order passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_expand_links_empty_paths_short_circuits()
|
||||
test_expand_links_returns_outlinks_and_inlinks_with_meta()
|
||||
test_expand_links_max_per_direction_caps_neighbors()
|
||||
test_expand_links_node_without_meta_returns_empty_meta_dict()
|
||||
test_render_expansion_lines_empty_input_yields_empty_list()
|
||||
test_render_expansion_lines_outlinks_only()
|
||||
test_render_expansion_lines_inlinks_only_with_anchor()
|
||||
test_render_expansion_lines_both_directions_in_order()
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -8,6 +8,7 @@ a markdown-to-FileNode transformer.
|
|||
|
||||
# pylint: disable=protected-access
|
||||
|
||||
from contextlib import chdir as temp_chdir
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
|
|
@ -18,22 +19,6 @@ import pytest
|
|||
from reme.components.file_chunker import DefaultFileChunker, MarkdownFileChunker
|
||||
|
||||
|
||||
class temp_chdir:
|
||||
"""Context manager to temporarily chdir into a path and restore on exit."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.old = None
|
||||
|
||||
def __enter__(self):
|
||||
self.old = os.getcwd()
|
||||
os.chdir(self.path)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
os.chdir(self.old)
|
||||
|
||||
|
||||
def _write_md(tmpdir: str, name: str, body: str) -> str:
|
||||
"""Drop a markdown file under tmpdir, return its relative path (matches cwd)."""
|
||||
if "/" in name:
|
||||
|
|
@ -54,7 +39,6 @@ def test_parse_empty_file():
|
|||
assert node.path == "x.md"
|
||||
assert chunks == []
|
||||
assert node.links == []
|
||||
print("✓ test_parse_empty_file passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -168,7 +152,6 @@ def test_parse_frontmatter_only():
|
|||
assert node.front_matter.name == "t"
|
||||
assert chunks == []
|
||||
assert node.links == []
|
||||
print("✓ test_parse_frontmatter_only passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -222,7 +205,6 @@ def test_parse_frontmatter_metadata_is_opt_in():
|
|||
"description": "Jon lost his job",
|
||||
"conversation_date": "2023-01-19",
|
||||
}
|
||||
print("✓ test_parse_frontmatter_metadata_is_opt_in passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -282,7 +264,6 @@ def test_parse_frontmatter_metadata_keys_allowlist():
|
|||
)
|
||||
_, chunks = await chunker.chunk(path)
|
||||
assert chunks[0].metadata == {}
|
||||
print("✓ test_parse_frontmatter_metadata_keys_allowlist passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -299,7 +280,6 @@ def test_parse_small_body_one_chunk():
|
|||
assert len(chunks) == 1
|
||||
assert "this is a small body" in chunks[0].text
|
||||
assert node.chunk_ids == [chunks[0].id]
|
||||
print("✓ test_parse_small_body_one_chunk passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -374,7 +354,6 @@ def test_parse_oversized_body_splits():
|
|||
_, chunks = await chunker.chunk(path)
|
||||
assert len(chunks) > 1
|
||||
assert all(len(chunk.text.encode("utf-8")) <= chunker.chunk_byte_size for chunk in chunks)
|
||||
print("✓ test_parse_oversized_body_splits passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -390,7 +369,6 @@ def test_parse_chunk_ids_match_node_chunk_ids():
|
|||
chunker = MarkdownFileChunker(chunk_byte_size=200)
|
||||
node, chunks = await chunker.chunk(path)
|
||||
assert node.chunk_ids == [c.id for c in chunks]
|
||||
print("✓ test_parse_chunk_ids_match_node_chunk_ids passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -410,7 +388,6 @@ def test_parse_links_literal_targets():
|
|||
# source_path always equals the node's own path
|
||||
for link in node.links:
|
||||
assert link.source_path == node.path
|
||||
print("✓ test_parse_links_literal_targets passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -432,7 +409,6 @@ def test_parse_links_short_and_no_ext_kept_literally():
|
|||
node, _ = await chunker.chunk(path)
|
||||
targets = {link.target_path for link in node.links}
|
||||
assert targets == {"Alice", "topics/Alice", "topics/Alice.md"}
|
||||
print("✓ test_parse_links_short_and_no_ext_kept_literally passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -447,7 +423,6 @@ def test_parse_links_legacy_relation_text_is_ignored():
|
|||
chunker = MarkdownFileChunker()
|
||||
node, _ = await chunker.chunk(path)
|
||||
assert {link.target_path for link in node.links} == {"A.md", "B.md"}
|
||||
print("✓ test_parse_links_legacy_relation_text_is_ignored passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -462,7 +437,6 @@ def test_parse_links_deduped():
|
|||
chunker = MarkdownFileChunker()
|
||||
node, _ = await chunker.chunk(path)
|
||||
assert len([link for link in node.links if link.target_path == "A.md"]) == 1
|
||||
print("✓ test_parse_links_deduped passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -471,7 +445,6 @@ def test_parse_min_chunk_byte_size_clamped():
|
|||
"""chunk_byte_size below 100 should be clamped to 100."""
|
||||
chunker = MarkdownFileChunker(chunk_byte_size=10)
|
||||
assert chunker.chunk_byte_size == 100
|
||||
print("✓ test_parse_min_chunk_byte_size_clamped passed")
|
||||
|
||||
|
||||
def test_parse_embed_toc_prefixes_chunk_text():
|
||||
|
|
@ -485,7 +458,6 @@ def test_parse_embed_toc_prefixes_chunk_text():
|
|||
_, chunks = await chunker.chunk(path)
|
||||
# Single small section fits; check that the heading appears in text.
|
||||
assert any("Top" in c.text for c in chunks)
|
||||
print("✓ test_parse_embed_toc_prefixes_chunk_text passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -680,7 +652,6 @@ def test_parse_frontmatter_preserves_original_line_numbers():
|
|||
assert len(chunks) == 1
|
||||
assert chunks[0].start_line == 4
|
||||
assert chunks[0].end_line == 6
|
||||
print("✓ test_parse_frontmatter_preserves_original_line_numbers passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -698,7 +669,6 @@ def test_parse_frontmatter_offsets_split_table_rows():
|
|||
assert len(chunks) > 1
|
||||
assert chunks[0].start_line == 6
|
||||
assert chunks[0].end_line >= chunks[0].start_line
|
||||
print("✓ test_parse_frontmatter_offsets_split_table_rows passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -716,7 +686,6 @@ def test_parse_bad_frontmatter_does_not_abort_chunking():
|
|||
assert len(chunks) == 1
|
||||
assert chunks[0].start_line == 4
|
||||
assert "body" in chunks[0].text
|
||||
print("✓ test_parse_bad_frontmatter_does_not_abort_chunking passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -743,7 +712,6 @@ def test_chunk_does_not_split_wikilink_at_boundary():
|
|||
assert "[[" not in first or "]]" in first, f"first chunk has dangling '[[': {first!r}"
|
||||
# And the link should appear intact in some chunk.
|
||||
assert any(link in c.text for c in chunks), "link was split across all chunks"
|
||||
print("✓ test_chunk_does_not_split_wikilink_at_boundary passed")
|
||||
finally:
|
||||
os.unlink(temp_path)
|
||||
|
||||
|
|
@ -773,7 +741,6 @@ def test_chunk_does_not_split_wikilink_in_overlap():
|
|||
if "]]" in t and "[[" not in t.split("]]", 1)[0]:
|
||||
raise AssertionError(f"chunk starts mid-link: {t[:40]!r}")
|
||||
assert any(link in c.text for c in chunks)
|
||||
print("✓ test_chunk_does_not_split_wikilink_in_overlap passed")
|
||||
finally:
|
||||
os.unlink(temp_path)
|
||||
|
||||
|
|
@ -799,40 +766,7 @@ def test_chunk_falls_back_for_oversize_link():
|
|||
_, chunks = await chunker.chunk(temp_path)
|
||||
# Must terminate (not hang) and cover the whole file.
|
||||
assert len(chunks) >= 2
|
||||
print("✓ test_chunk_falls_back_for_oversize_link passed")
|
||||
finally:
|
||||
os.unlink(temp_path)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n=== MarkdownFileChunker tests ===")
|
||||
test_parse_empty_file()
|
||||
test_parse_frontmatter_only()
|
||||
test_parse_frontmatter_metadata_is_opt_in()
|
||||
test_parse_small_body_one_chunk()
|
||||
test_parse_small_children_are_cached_without_recursive_calls()
|
||||
test_parse_child_cache_flushes_at_exact_limit()
|
||||
test_parse_oversized_child_flushes_parent_cache()
|
||||
test_parse_oversized_body_splits()
|
||||
test_parse_chunk_ids_match_node_chunk_ids()
|
||||
test_parse_links_literal_targets()
|
||||
test_parse_links_short_and_no_ext_kept_literally()
|
||||
test_parse_links_legacy_relation_text_is_ignored()
|
||||
test_parse_links_deduped()
|
||||
test_parse_min_chunk_byte_size_clamped()
|
||||
test_parse_embed_toc_prefixes_chunk_text()
|
||||
test_parse_embed_toc_is_enabled_by_default()
|
||||
test_count_sections_ignores_fenced_headings_and_supports_setext()
|
||||
test_parse_excessive_sections_uses_plain_text_without_ast()
|
||||
test_parse_section_limit_is_inclusive_for_ast()
|
||||
test_parse_small_sections_are_merged_and_headings_preserved()
|
||||
test_parse_embed_toc_uses_breadcrumbs_without_sibling_duplication()
|
||||
test_parse_heading_heavy_output_grows_linearly()
|
||||
test_parse_nested_sections_merge_within_recursive_context()
|
||||
test_parse_breadcrumbs_and_part_markers_share_byte_budget()
|
||||
test_parse_frontmatter_preserves_original_line_numbers()
|
||||
test_parse_frontmatter_offsets_split_table_rows()
|
||||
test_parse_bad_frontmatter_does_not_abort_chunking()
|
||||
print("\n所有测试通过!")
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ Covers:
|
|||
|
||||
# pylint: disable=protected-access
|
||||
|
||||
import os
|
||||
from contextlib import chdir as temp_chdir
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -31,22 +31,6 @@ from reme.steps.file_io.read import ReadStep
|
|||
from reme.steps.file_io.write import WriteStep
|
||||
|
||||
|
||||
class temp_chdir:
|
||||
"""Context manager to temporarily chdir into a path and restore on exit."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.old = None
|
||||
|
||||
def __enter__(self):
|
||||
self.old = os.getcwd()
|
||||
os.chdir(self.path)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
os.chdir(self.old)
|
||||
|
||||
|
||||
def _seed(workspace: Path, rel: str, body: str = "body\n") -> Path:
|
||||
target = workspace / rel
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
|
@ -257,7 +241,3 @@ async def test_no_constraint_allows_all():
|
|||
resp = await _run(WriteStep, store, path="anywhere/new.md", content="ok")
|
||||
assert resp.success is True
|
||||
await store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
|
|
|||
|
|
@ -262,39 +262,3 @@ def test_repr():
|
|||
r = repr(ph)
|
||||
assert "en" in r
|
||||
assert "2" in r
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n=== PromptHandler Tests ===")
|
||||
test_init_filters_non_string_values()
|
||||
test_load_prompt_dict_basic()
|
||||
test_load_prompt_dict_skips_non_string_values()
|
||||
test_load_prompt_dict_overwrite_true()
|
||||
test_load_prompt_dict_overwrite_false()
|
||||
test_load_prompt_dict_none()
|
||||
test_load_prompt_dict_non_dict()
|
||||
test_load_prompt_by_file_yaml()
|
||||
test_load_prompt_by_file_json()
|
||||
test_load_prompt_by_file_none()
|
||||
test_load_prompt_by_file_nonexistent()
|
||||
test_load_prompt_by_file_unsupported_extension()
|
||||
test_get_prompt_bare_key()
|
||||
test_get_prompt_strips()
|
||||
test_get_prompt_missing_raises()
|
||||
test_get_prompt_language_fallback()
|
||||
test_get_prompt_language_fallback_to_bare()
|
||||
test_has_prompt()
|
||||
test_has_prompt_with_language()
|
||||
test_list_prompts_all()
|
||||
test_list_prompts_filtered()
|
||||
test_flag_filter_keeps_matching()
|
||||
test_flag_filter_removes_non_matching()
|
||||
test_flag_filter_default_false()
|
||||
test_flag_filter_unflagged_lines_always_kept()
|
||||
test_format_variables()
|
||||
test_format_missing_variable_raises()
|
||||
test_format_no_variables_no_error()
|
||||
test_format_flags_and_variables_combined()
|
||||
test_format_flags_false_variable_not_needed()
|
||||
test_repr()
|
||||
print("\n所有测试通过!")
|
||||
|
|
|
|||
|
|
@ -13,9 +13,9 @@ Coverage:
|
|||
|
||||
# pylint: disable=protected-access
|
||||
|
||||
from contextlib import chdir as temp_chdir
|
||||
import asyncio
|
||||
import base64
|
||||
import os
|
||||
import tempfile
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
|
|
@ -27,22 +27,6 @@ warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba")
|
|||
warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources")
|
||||
|
||||
|
||||
class temp_chdir:
|
||||
"""Context manager to temporarily chdir into a path and restore on exit."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.old = None
|
||||
|
||||
def __enter__(self):
|
||||
self.old = os.getcwd()
|
||||
os.chdir(self.path)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
os.chdir(self.old)
|
||||
|
||||
|
||||
def _run(coro):
|
||||
asyncio.run(coro)
|
||||
|
||||
|
|
@ -83,7 +67,6 @@ def test_read_image_png():
|
|||
assert resp.metadata["size_bytes"] == len(payload), resp.metadata
|
||||
assert "oversized" not in resp.metadata, resp.metadata
|
||||
await store.close()
|
||||
print("✓ test_read_image_png passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -101,7 +84,6 @@ def test_read_image_jpeg():
|
|||
assert base64.b64decode(resp.answer) == payload
|
||||
assert resp.metadata["mime"] == "image/jpeg", resp.metadata
|
||||
await store.close()
|
||||
print("✓ test_read_image_jpeg passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -127,7 +109,6 @@ def test_read_image_oversized():
|
|||
pass # expected — answer is notice text, not base64
|
||||
assert "exceeds max_bytes" in resp.answer
|
||||
await store.close()
|
||||
print("✓ test_read_image_oversized passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -146,7 +127,6 @@ def test_read_image_unknown_suffix():
|
|||
assert resp.metadata["non_image_warning"] is True, resp.metadata
|
||||
assert resp.metadata["mime"] is None, resp.metadata
|
||||
await store.close()
|
||||
print("✓ test_read_image_unknown_suffix passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -165,7 +145,6 @@ def test_read_image_no_suffix():
|
|||
assert resp.metadata["mime"] is None, resp.metadata
|
||||
assert base64.b64decode(resp.answer) == payload
|
||||
await store.close()
|
||||
print("✓ test_read_image_no_suffix passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -181,7 +160,6 @@ def test_read_image_missing():
|
|||
assert resp.answer.startswith("Error:"), resp
|
||||
assert "does not exist" in resp.answer
|
||||
await store.close()
|
||||
print("✓ test_read_image_missing passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -197,7 +175,6 @@ def test_read_image_is_directory():
|
|||
assert resp.success is False, resp
|
||||
assert "is not a file" in resp.answer, resp
|
||||
await store.close()
|
||||
print("✓ test_read_image_is_directory passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -212,7 +189,6 @@ def test_read_image_path_required():
|
|||
assert resp.success is False, resp
|
||||
assert "`path` is required" in resp.answer, resp
|
||||
await store.close()
|
||||
print("✓ test_read_image_path_required passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -228,6 +204,5 @@ def test_read_image_invalid_max_bytes():
|
|||
assert resp.success is False, resp
|
||||
assert "positive integer" in resp.answer, resp
|
||||
await store.close()
|
||||
print("✓ test_read_image_invalid_max_bytes passed")
|
||||
|
||||
_run(run())
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@ so these tests focus on the *integration* layer:
|
|||
|
||||
# pylint: disable=protected-access
|
||||
|
||||
from contextlib import chdir as temp_chdir
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
|
|
@ -28,22 +28,6 @@ warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba")
|
|||
warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources")
|
||||
|
||||
|
||||
class temp_chdir:
|
||||
"""Context manager to temporarily chdir into a path and restore on exit."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.old = None
|
||||
|
||||
def __enter__(self):
|
||||
self.old = os.getcwd()
|
||||
os.chdir(self.path)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
os.chdir(self.old)
|
||||
|
||||
|
||||
def _run(coro):
|
||||
asyncio.run(coro)
|
||||
|
||||
|
|
@ -102,7 +86,6 @@ def test_read_without_neighbors_default():
|
|||
assert "Related neighbors" not in str(resp.answer)
|
||||
assert "link_expansion" not in resp.metadata
|
||||
await store.close()
|
||||
print("✓ test_read_without_neighbors_default passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -136,7 +119,6 @@ def test_read_with_neighbors_injects_block_and_metadata():
|
|||
assert expansion["A.md"]["outlinks"][0]["path"] == "B.md"
|
||||
assert expansion["A.md"]["outlinks"][0]["meta"] == {"name": "B Doc", "description": "beta"}
|
||||
await store.close()
|
||||
print("✓ test_read_with_neighbors_injects_block_and_metadata passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -177,7 +159,6 @@ def test_read_with_neighbors_no_links_no_block():
|
|||
assert "Related neighbors" not in str(resp.answer)
|
||||
assert "link_expansion" not in resp.metadata
|
||||
await store.close()
|
||||
print("✓ test_read_with_neighbors_no_links_no_block passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -198,6 +179,5 @@ def test_read_with_neighbors_non_md_falls_through():
|
|||
assert "Related neighbors" not in str(resp.answer)
|
||||
assert "link_expansion" not in resp.metadata
|
||||
await store.close()
|
||||
print("✓ test_read_with_neighbors_non_md_falls_through passed")
|
||||
|
||||
_run(run())
|
||||
|
|
|
|||
|
|
@ -156,25 +156,3 @@ def test_custom_response():
|
|||
ctx = RuntimeContext(response=resp)
|
||||
assert ctx.response is resp
|
||||
assert ctx.response.success is False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n=== RuntimeContext Tests ===")
|
||||
test_getitem_setitem()
|
||||
test_getitem_missing_raises()
|
||||
test_contains()
|
||||
test_delitem()
|
||||
test_get_with_default()
|
||||
test_update_merges_and_returns_self()
|
||||
test_from_context_creates_new_when_none()
|
||||
test_from_context_reuses_existing()
|
||||
test_apply_mapping_copies_values()
|
||||
test_apply_mapping_skips_missing_source()
|
||||
test_apply_mapping_empty_is_noop()
|
||||
test_stream_property()
|
||||
test_enqueue_raises_without_queue()
|
||||
test_add_stream_string()
|
||||
test_add_stream_done()
|
||||
test_default_response()
|
||||
test_custom_response()
|
||||
print("\n所有测试通过!")
|
||||
|
|
|
|||
|
|
@ -38,7 +38,6 @@ def test_basic_chinese():
|
|||
|
||||
assert "北京" in jieba_result[0] or "天安门" in jieba_result[0]
|
||||
assert "我" in regex_result[0]
|
||||
print("✓ test_basic_chinese passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -54,7 +53,6 @@ def test_basic_english():
|
|||
|
||||
assert "love" in regex_result[0]
|
||||
assert "beijing" in regex_result[0]
|
||||
print("✓ test_basic_english passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -70,7 +68,6 @@ def test_mixed_chinese_english():
|
|||
|
||||
assert "python" in jieba_result[0]
|
||||
assert "python" in regex_result[0]
|
||||
print("✓ test_mixed_chinese_english passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -92,8 +89,6 @@ def test_open_example():
|
|||
assert "我" in regex_result[0]
|
||||
assert "很" in regex_result[0]
|
||||
|
||||
print("✓ test_open_example passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
|
|
@ -112,8 +107,6 @@ def test_with_stopwords():
|
|||
assert "吗" not in regex_result[0]
|
||||
assert "的" not in jieba_result[0]
|
||||
|
||||
print("✓ test_with_stopwords passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
|
|
@ -130,7 +123,6 @@ def test_multiple_texts():
|
|||
|
||||
assert len(jieba_result) == 3
|
||||
assert len(regex_result) == 3
|
||||
print("✓ test_multiple_texts passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -152,8 +144,6 @@ def test_tokenizer_lifecycle():
|
|||
assert not tokenizer.is_started
|
||||
assert len(tokenizer.stopwords) == 0
|
||||
|
||||
print("✓ test_tokenizer_lifecycle passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
|
|
@ -166,16 +156,3 @@ def test_jieba_tokenizer_requires_start():
|
|||
assert "Call start() first" in str(exc)
|
||||
else:
|
||||
raise AssertionError("expected RuntimeError when JiebaTokenizer is used before start()")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n=== Tokenizer Tests ===")
|
||||
test_basic_chinese()
|
||||
test_basic_english()
|
||||
test_mixed_chinese_english()
|
||||
test_open_example()
|
||||
test_with_stopwords()
|
||||
test_multiple_texts()
|
||||
test_tokenizer_lifecycle()
|
||||
test_jieba_tokenizer_requires_start()
|
||||
print("\n所有测试通过!")
|
||||
|
|
|
|||
|
|
@ -1,21 +0,0 @@
|
|||
"""Focused tests for TuShare source selection."""
|
||||
|
||||
# pylint: disable=protected-access
|
||||
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
from reme.utils.tushare import create_tushare_api
|
||||
|
||||
|
||||
def test_tushare_uses_configured_mirror_or_sdk_default(monkeypatch):
|
||||
"""A mirror overrides the SDK endpoint while an empty setting leaves it unchanged."""
|
||||
api = SimpleNamespace(_DataApi__http_url="https://api.tushare.pro")
|
||||
monkeypatch.setitem(sys.modules, "tushare", SimpleNamespace(pro_api=lambda _token: api))
|
||||
|
||||
monkeypatch.delenv("TUSHARE_MIRROR_URL", raising=False)
|
||||
assert create_tushare_api("token")._DataApi__http_url == "https://api.tushare.pro"
|
||||
assert api._DataApi__timeout == 600
|
||||
|
||||
monkeypatch.setenv("TUSHARE_MIRROR_URL", "http://112.124.63.173:4000/tushare/")
|
||||
assert create_tushare_api("token")._DataApi__http_url == "http://112.124.63.173:4000/tushare"
|
||||
|
|
@ -14,8 +14,8 @@ short ``[[x]]`` and no-ext ``[[topics/x]]`` are left alone by design.
|
|||
|
||||
# pylint: disable=protected-access
|
||||
|
||||
from contextlib import chdir as temp_chdir
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
|
|
@ -28,22 +28,6 @@ warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba")
|
|||
warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources")
|
||||
|
||||
|
||||
class temp_chdir:
|
||||
"""Test helper: chdir to ``path`` on enter, restore previous cwd on exit."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.old = None
|
||||
|
||||
def __enter__(self):
|
||||
self.old = os.getcwd()
|
||||
os.chdir(self.path)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
os.chdir(self.old)
|
||||
|
||||
|
||||
async def _store_with(files: dict[str, str]) -> LocalFileStore:
|
||||
"""LocalFileStore seeded with (rel → content) files: written to disk
|
||||
AND registered in the file_graph with wikilinks parsed from the body.
|
||||
|
|
@ -98,7 +82,6 @@ def test_retarget_exact_full_path_match():
|
|||
assert "[[topics/Alice.md]]" not in body
|
||||
assert "[[topics/Bob.md]]" in body
|
||||
await store.close()
|
||||
print("✓ test_retarget_exact_full_path_match passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -123,7 +106,6 @@ def test_retarget_short_and_no_ext_forms_ignored():
|
|||
assert "[[Alice]]" in body # short form untouched
|
||||
assert "[[topics/Alice]]" in body # no-ext form untouched
|
||||
await store.close()
|
||||
print("✓ test_retarget_short_and_no_ext_forms_ignored passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -140,7 +122,6 @@ def test_retarget_anchor_preserved():
|
|||
body = (root / "note.md").read_text(encoding="utf-8")
|
||||
assert "[[people/Alice.md#intro]]" in body
|
||||
await store.close()
|
||||
print("✓ test_retarget_anchor_preserved passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -157,7 +138,6 @@ def test_retarget_alias_preserved():
|
|||
body = (root / "note.md").read_text(encoding="utf-8")
|
||||
assert "[[people/Alice.md|Alice the Architect]]" in body
|
||||
await store.close()
|
||||
print("✓ test_retarget_alias_preserved passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -174,7 +154,6 @@ def test_retarget_anchor_and_alias_together():
|
|||
body = (root / "note.md").read_text(encoding="utf-8")
|
||||
assert "[[people/Alice.md#bio|her bio]]" in body
|
||||
await store.close()
|
||||
print("✓ test_retarget_anchor_and_alias_together passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -191,7 +170,6 @@ def test_retarget_image_marker_preserved():
|
|||
body = (root / "note.md").read_text(encoding="utf-8")
|
||||
assert "![[diagrams/diagram.md]]" in body
|
||||
await store.close()
|
||||
print("✓ test_retarget_image_marker_preserved passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -212,7 +190,6 @@ def test_retarget_preserves_surrounding_relation_text():
|
|||
assert "colleague:: [[people/Alice.md]]" in body
|
||||
assert "[负责:: [[people/Alice.md]]]" in body
|
||||
await store.close()
|
||||
print("✓ test_retarget_preserves_surrounding_relation_text passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -256,7 +233,6 @@ def test_retarget_multiple_files_aggregate_counts():
|
|||
by_file = {row["path"]: row["count"] for row in payload["by_file"]}
|
||||
assert by_file == {"a.md": 2, "sub/b.md": 1}
|
||||
await store.close()
|
||||
print("✓ test_retarget_multiple_files_aggregate_counts passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -280,7 +256,6 @@ def test_retarget_dry_run_does_not_write():
|
|||
assert payload["files_touched"] == 1
|
||||
assert (root / "note.md").read_text(encoding="utf-8") == original
|
||||
await store.close()
|
||||
print("✓ test_retarget_dry_run_does_not_write passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -310,7 +285,6 @@ def test_retarget_scope_limits_sweep():
|
|||
assert "[[people/Alice.md]]" in in_body
|
||||
assert "[[topics/Alice.md]]" in out_body # untouched
|
||||
await store.close()
|
||||
print("✓ test_retarget_scope_limits_sweep passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -325,7 +299,6 @@ def test_retarget_src_eq_dst_is_noop():
|
|||
assert payload["links_changed"] == 0
|
||||
assert payload["files_touched"] == 0
|
||||
await store.close()
|
||||
print("✓ test_retarget_src_eq_dst_is_noop passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -340,7 +313,6 @@ def test_retarget_empty_src_or_dst_errors():
|
|||
payload = await WikilinkHandler.retarget_links(store, **kwargs)
|
||||
assert "error" in payload, f"expected error for {kwargs}"
|
||||
await store.close()
|
||||
print("✓ test_retarget_empty_src_or_dst_errors passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -355,7 +327,6 @@ def test_retarget_dst_with_forbidden_chars_errors():
|
|||
payload = await WikilinkHandler.retarget_links(store, src="topics/Alice.md", dst=bad)
|
||||
assert "error" in payload, f"expected error for dst={bad!r}, got {payload}"
|
||||
await store.close()
|
||||
print("✓ test_retarget_dst_with_forbidden_chars_errors passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -373,7 +344,6 @@ def test_retarget_absolute_path_rejected():
|
|||
payload = await WikilinkHandler.retarget_links(store, **kwargs)
|
||||
assert "error" in payload, f"expected error for {kwargs}"
|
||||
await store.close()
|
||||
print("✓ test_retarget_absolute_path_rejected passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
|
@ -396,27 +366,5 @@ def test_find_inbound_counts_references():
|
|||
by_file = {row["path"]: row["count"] for row in payload["by_file"]}
|
||||
assert by_file == {"a.md": 2, "b.md": 1}
|
||||
await store.close()
|
||||
print("✓ test_find_inbound_counts_references passed")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n=== wikilink helper tests ===")
|
||||
test_retarget_exact_full_path_match()
|
||||
test_retarget_short_and_no_ext_forms_ignored()
|
||||
test_retarget_anchor_preserved()
|
||||
test_retarget_alias_preserved()
|
||||
test_retarget_anchor_and_alias_together()
|
||||
test_retarget_image_marker_preserved()
|
||||
test_retarget_preserves_surrounding_relation_text()
|
||||
test_markdown_links_are_ignored()
|
||||
test_retarget_multiple_files_aggregate_counts()
|
||||
test_retarget_dry_run_does_not_write()
|
||||
test_retarget_scope_limits_sweep()
|
||||
test_retarget_src_eq_dst_is_noop()
|
||||
test_retarget_empty_src_or_dst_errors()
|
||||
test_retarget_dst_with_forbidden_chars_errors()
|
||||
test_retarget_absolute_path_rejected()
|
||||
test_find_inbound_counts_references()
|
||||
print("\n所有测试通过!")
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ interleaving).
|
|||
|
||||
# pylint: disable=protected-access
|
||||
|
||||
from contextlib import chdir as temp_chdir
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
|
|
@ -29,22 +29,6 @@ warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba")
|
|||
warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources")
|
||||
|
||||
|
||||
class temp_chdir:
|
||||
"""Context manager to temporarily chdir into a path and restore on exit."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.old = None
|
||||
|
||||
def __enter__(self):
|
||||
self.old = os.getcwd()
|
||||
os.chdir(self.path)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
os.chdir(self.old)
|
||||
|
||||
|
||||
def _run(coro):
|
||||
asyncio.run(coro)
|
||||
|
||||
|
|
@ -87,7 +71,6 @@ def test_write_metadata_extends_frontmatter():
|
|||
assert post.metadata["priority"] == 3
|
||||
assert post.content.strip() == "body text"
|
||||
await store.close()
|
||||
print("✓ test_write_metadata_extends_frontmatter passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -112,7 +95,6 @@ def test_write_metadata_reserved_keys_ignored():
|
|||
assert post.metadata["description"] == "ExplicitDesc"
|
||||
assert post.metadata["tag"] == "kept"
|
||||
await store.close()
|
||||
print("✓ test_write_metadata_reserved_keys_ignored passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -134,7 +116,6 @@ def test_write_no_metadata_preserves_legacy_shape():
|
|||
post = frontmatter.loads((Path(tmp) / "note.md").read_text(encoding="utf-8"))
|
||||
assert set(post.metadata.keys()) == {"name", "description"}
|
||||
await store.close()
|
||||
print("✓ test_write_no_metadata_preserves_legacy_shape passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -157,7 +138,6 @@ def test_write_non_md_drops_metadata():
|
|||
on_disk = (Path(tmp) / "note.txt").read_text(encoding="utf-8")
|
||||
assert on_disk == "plain body"
|
||||
await store.close()
|
||||
print("✓ test_write_non_md_drops_metadata passed")
|
||||
|
||||
_run(run())
|
||||
|
||||
|
|
@ -202,6 +182,5 @@ def test_write_lock_serializes_concurrent_writes():
|
|||
idx_body = post.content.strip().split("-", 1)[1]
|
||||
assert idx_name == idx_desc == idx_body, (idx_name, idx_desc, idx_body)
|
||||
await store.close()
|
||||
print("✓ test_write_lock_serializes_concurrent_writes passed")
|
||||
|
||||
_run(run())
|
||||
|
|
|
|||
|
|
@ -2,9 +2,8 @@
|
|||
|
||||
# pylint: disable=protected-access
|
||||
|
||||
import asyncio
|
||||
from contextlib import chdir as temp_chdir
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import tomllib
|
||||
import warnings
|
||||
|
|
@ -13,66 +12,11 @@ from pathlib import Path
|
|||
import numpy as np
|
||||
import pytest
|
||||
|
||||
|
||||
from reme.components.file_store import ZvecLocalFileStore
|
||||
from reme.schema import FileChunk, FileNode
|
||||
from reme.schema import FileChunk
|
||||
|
||||
|
||||
class temp_chdir:
|
||||
"""Temporarily chdir into a test workspace."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.old = None
|
||||
|
||||
def __enter__(self):
|
||||
self.old = os.getcwd()
|
||||
os.chdir(self.path)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
os.chdir(self.old)
|
||||
|
||||
|
||||
class FakeEmbeddingStore:
|
||||
"""Small deterministic embedding provider used by file-store tests."""
|
||||
|
||||
dimensions = 2
|
||||
max_batch_size = 10
|
||||
is_healthy = True
|
||||
|
||||
def _embed(self, text: str) -> np.ndarray:
|
||||
if "beta" in text or "fresh" in text:
|
||||
return np.array([0.0, 1.0], dtype=np.float16)
|
||||
return np.array([1.0, 0.0], dtype=np.float16)
|
||||
|
||||
async def health_check(self, _timeout: float = 2.0) -> bool:
|
||||
"""Report the fake embedding service as healthy."""
|
||||
return True
|
||||
|
||||
async def get_embedding(self, input_text: str, **_kwargs) -> np.ndarray:
|
||||
"""Return a deterministic embedding for a single text."""
|
||||
return self._embed(input_text)
|
||||
|
||||
async def get_node_embeddings(self, nodes: list[FileChunk], **_kwargs) -> list[FileChunk]:
|
||||
"""Attach deterministic embeddings to file chunks."""
|
||||
for chunk_node in nodes:
|
||||
chunk_node.embedding = self._embed(chunk_node.text)
|
||||
return nodes
|
||||
|
||||
|
||||
def run(coro):
|
||||
"""Run an async test body."""
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def node(path: str) -> FileNode:
|
||||
"""Build a minimal file node."""
|
||||
return FileNode(path=path, st_mtime=1.0)
|
||||
|
||||
|
||||
def chunk(chunk_id: str, path: str, text: str, **metadata) -> FileChunk:
|
||||
"""Build a minimal file chunk."""
|
||||
return FileChunk(id=chunk_id, path=path, text=text, start_line=1, end_line=1, metadata=metadata)
|
||||
from .file_store_test_support import FakeEmbeddingStore, _new_zvec_store, chunk, node
|
||||
|
||||
|
||||
def embedded_chunk(chunk_id: str, path: str, text: str, embedding: list[float]) -> FileChunk:
|
||||
|
|
@ -104,15 +48,6 @@ def _tamper_with_collection(store: ZvecLocalFileStore, mutate) -> None:
|
|||
del collection
|
||||
|
||||
|
||||
def _new_zvec_store(name, **kwargs):
|
||||
"""Construct a zvec store with embedding disabled at bind time."""
|
||||
try:
|
||||
store = ZvecLocalFileStore(name=name, embedding_store="", **kwargs)
|
||||
except ImportError:
|
||||
pytest.skip("zvec is not installed")
|
||||
return store
|
||||
|
||||
|
||||
async def _started_store(name, **kwargs) -> ZvecLocalFileStore:
|
||||
"""Start a store with a fake embedding provider and a live collection."""
|
||||
store = _new_zvec_store(name, **kwargs)
|
||||
|
|
@ -134,454 +69,400 @@ async def _seed_unembedded_chunk(store: ZvecLocalFileStore, chunk_id: str, path:
|
|||
# -- CRUD / search ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_zvec_upsert_and_vector_search_ranks_by_similarity():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_upsert_and_vector_search_ranks_by_similarity():
|
||||
"""Upserted chunks are searchable; results are ranked by cosine similarity."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_basic")
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_basic")
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await store.upsert([(node("b.md"), [chunk("b", "b.md", "beta text")])])
|
||||
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await store.upsert([(node("b.md"), [chunk("b", "b.md", "beta text")])])
|
||||
|
||||
results = await store.vector_search("alpha", 10, {})
|
||||
assert [c.id for c in results] == ["a", "b"]
|
||||
# Identical vector -> cosine distance 0 -> similarity score 1.
|
||||
assert results[0].scores["vector"] == pytest.approx(1.0, abs=1e-5)
|
||||
assert results[0].scores["score"] == results[0].scores["vector"]
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
results = await store.vector_search("alpha", 10, {})
|
||||
assert [c.id for c in results] == ["a", "b"]
|
||||
# Identical vector -> cosine distance 0 -> similarity score 1.
|
||||
assert results[0].scores["vector"] == pytest.approx(1.0, abs=1e-5)
|
||||
assert results[0].scores["score"] == results[0].scores["vector"]
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_zvec_same_id_text_change_updates_vector_in_place():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_same_id_text_change_updates_vector_in_place():
|
||||
"""A same-id text change replaces the vector via native zvec upsert."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_update")
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_update")
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})] == ["a"]
|
||||
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})] == ["a"]
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "beta text")])])
|
||||
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "beta text")])])
|
||||
|
||||
assert store._indexed_ids == {"a"}
|
||||
results = await store.vector_search("beta", 5, {})
|
||||
assert [c.id for c in results] == ["a"]
|
||||
assert results[0].scores["vector"] == pytest.approx(1.0, abs=1e-5)
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert store._indexed_ids == {"a"}
|
||||
results = await store.vector_search("beta", 5, {})
|
||||
assert [c.id for c in results] == ["a"]
|
||||
assert results[0].scores["vector"] == pytest.approx(1.0, abs=1e-5)
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_zvec_explicit_embedding_change_updates_collection():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_explicit_embedding_change_updates_collection():
|
||||
"""A same-id, same-text update with a new explicit embedding replaces the vector."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_explicit_embedding")
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_explicit_embedding")
|
||||
await store.upsert([(node("a.md"), [embedded_chunk("same", "a.md", "same text", [1.0, 0.0])])])
|
||||
await store.upsert([(node("a.md"), [embedded_chunk("same", "a.md", "same text", [0.0, 1.0])])])
|
||||
|
||||
await store.upsert([(node("a.md"), [embedded_chunk("same", "a.md", "same text", [1.0, 0.0])])])
|
||||
await store.upsert([(node("a.md"), [embedded_chunk("same", "a.md", "same text", [0.0, 1.0])])])
|
||||
# "beta" embeds to [0, 1]; the collection must hold the new vector.
|
||||
results = await store.vector_search("beta", 5, {})
|
||||
assert [c.id for c in results] == ["same"]
|
||||
assert results[0].scores["vector"] == pytest.approx(1.0, abs=1e-5)
|
||||
await store.close()
|
||||
|
||||
# "beta" embeds to [0, 1]; the collection must hold the new vector.
|
||||
results = await store.vector_search("beta", 5, {})
|
||||
assert [c.id for c in results] == ["same"]
|
||||
assert results[0].scores["vector"] == pytest.approx(1.0, abs=1e-5)
|
||||
await store.close()
|
||||
# The stale vector must not survive a restart either.
|
||||
reopened = _new_zvec_store("t_zvec_explicit_embedding")
|
||||
reopened.embedding_store = FakeEmbeddingStore()
|
||||
rebuilds = _track_rebuilds(reopened)
|
||||
await reopened.start()
|
||||
|
||||
# The stale vector must not survive a restart either.
|
||||
reopened = _new_zvec_store("t_zvec_explicit_embedding")
|
||||
reopened.embedding_store = FakeEmbeddingStore()
|
||||
rebuilds = _track_rebuilds(reopened)
|
||||
await reopened.start()
|
||||
|
||||
assert not rebuilds # the collection was already in sync, no repair needed
|
||||
results = await reopened.vector_search("beta", 5, {})
|
||||
assert [c.id for c in results] == ["same"]
|
||||
assert results[0].scores["vector"] == pytest.approx(1.0, abs=1e-5)
|
||||
await reopened.close()
|
||||
|
||||
run(go())
|
||||
assert not rebuilds # the collection was already in sync, no repair needed
|
||||
results = await reopened.vector_search("beta", 5, {})
|
||||
assert [c.id for c in results] == ["same"]
|
||||
assert results[0].scores["vector"] == pytest.approx(1.0, abs=1e-5)
|
||||
await reopened.close()
|
||||
|
||||
|
||||
def test_zvec_upsert_removes_stale_chunks_from_collection():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_upsert_removes_stale_chunks_from_collection():
|
||||
"""Re-upserting a path deletes vectors of chunks that no longer exist."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_stale")
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_stale")
|
||||
await store.upsert([(node("a.md"), [chunk("old", "a.md", "alpha old")])])
|
||||
await store.upsert([(node("a.md"), [chunk("new", "a.md", "alpha new")])])
|
||||
|
||||
await store.upsert([(node("a.md"), [chunk("old", "a.md", "alpha old")])])
|
||||
await store.upsert([(node("a.md"), [chunk("new", "a.md", "alpha new")])])
|
||||
|
||||
assert store._indexed_ids == {"new"}
|
||||
assert [c.id for c in await store.vector_search("alpha", 10, {})] == ["new"]
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert store._indexed_ids == {"new"}
|
||||
assert [c.id for c in await store.vector_search("alpha", 10, {})] == ["new"]
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_zvec_delete_removes_vectors():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_delete_removes_vectors():
|
||||
"""Deleting a path removes its chunk vectors from the collection."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_delete")
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_delete")
|
||||
await store.upsert(
|
||||
[
|
||||
(node("a.md"), [chunk("a", "a.md", "alpha text")]),
|
||||
(node("b.md"), [chunk("b", "b.md", "beta text")]),
|
||||
],
|
||||
)
|
||||
await store.delete("a.md")
|
||||
|
||||
await store.upsert(
|
||||
[
|
||||
(node("a.md"), [chunk("a", "a.md", "alpha text")]),
|
||||
(node("b.md"), [chunk("b", "b.md", "beta text")]),
|
||||
],
|
||||
)
|
||||
await store.delete("a.md")
|
||||
|
||||
assert store._indexed_ids == {"b"}
|
||||
assert [c.id for c in await store.vector_search("alpha", 10, {})] == ["b"]
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert store._indexed_ids == {"b"}
|
||||
assert [c.id for c in await store.vector_search("alpha", 10, {})] == ["b"]
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_zvec_vector_search_applies_post_filter():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_vector_search_applies_post_filter():
|
||||
"""The shared search_filter semantics apply on top of ANN results."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_filter")
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_filter")
|
||||
await store.upsert(
|
||||
[
|
||||
(node("a.md"), [chunk("a", "a.md", "alpha one", kind="x")]),
|
||||
(node("b.md"), [chunk("b", "b.md", "alpha two", kind="y")]),
|
||||
],
|
||||
)
|
||||
|
||||
await store.upsert(
|
||||
[
|
||||
(node("a.md"), [chunk("a", "a.md", "alpha one", kind="x")]),
|
||||
(node("b.md"), [chunk("b", "b.md", "alpha two", kind="y")]),
|
||||
],
|
||||
)
|
||||
|
||||
assert [c.id for c in await store.vector_search("alpha", 10, {"path": "b.md"})] == ["b"]
|
||||
assert [c.id for c in await store.vector_search("alpha", 10, {"kind": "x"})] == ["a"]
|
||||
assert await store.vector_search("alpha", 10, {"path": "c.md"}) == []
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert [c.id for c in await store.vector_search("alpha", 10, {"path": "b.md"})] == ["b"]
|
||||
assert [c.id for c in await store.vector_search("alpha", 10, {"kind": "x"})] == ["a"]
|
||||
assert await store.vector_search("alpha", 10, {"path": "c.md"}) == []
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_zvec_vector_search_uses_current_query_api():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_vector_search_uses_current_query_api():
|
||||
"""The ANN query path must not rely on zvec's deprecated VectorQuery alias."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_query_api")
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_query_api")
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error", DeprecationWarning)
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error", DeprecationWarning)
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_zvec_clear_resets_collection():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_clear_resets_collection():
|
||||
"""clear() drops the collection, sidecar, and indexed-id state."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_clear")
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_clear")
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await store.dump()
|
||||
assert store.zvec_sidecar_path.exists()
|
||||
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await store.dump()
|
||||
assert store.zvec_sidecar_path.exists()
|
||||
await store.clear()
|
||||
|
||||
await store.clear()
|
||||
|
||||
assert store._indexed_ids == set()
|
||||
assert not store.zvec_sidecar_path.exists()
|
||||
assert await store.vector_search("alpha", 10, {}) == []
|
||||
# The store stays usable after clear.
|
||||
await store.upsert([(node("b.md"), [chunk("b", "b.md", "beta text")])])
|
||||
assert [c.id for c in await store.vector_search("beta", 10, {})] == ["b"]
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert store._indexed_ids == set()
|
||||
assert not store.zvec_sidecar_path.exists()
|
||||
assert await store.vector_search("alpha", 10, {}) == []
|
||||
# The store stays usable after clear.
|
||||
await store.upsert([(node("b.md"), [chunk("b", "b.md", "beta text")])])
|
||||
assert [c.id for c in await store.vector_search("beta", 10, {})] == ["b"]
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_zvec_keyword_only_mode_keeps_keyword_search_working():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_keyword_only_mode_keeps_keyword_search_working():
|
||||
"""Without an embedding store, vector search is empty but keyword search works."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_zvec_store("t_zvec_keyword_only")
|
||||
await store.start()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = _new_zvec_store("t_zvec_keyword_only")
|
||||
await store.start()
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "uniquezvecword only")])])
|
||||
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "uniquezvecword only")])])
|
||||
|
||||
assert store._collection is None
|
||||
assert await store.vector_search("uniquezvecword", 5, {}) == []
|
||||
assert [c.id for c in await store.keyword_search("uniquezvecword", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert store._collection is None
|
||||
assert await store.vector_search("uniquezvecword", 5, {}) == []
|
||||
assert [c.id for c in await store.keyword_search("uniquezvecword", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
|
||||
# -- embedding backfill ---------------------------------------------------------
|
||||
|
||||
|
||||
def test_zvec_backfill_adds_incrementally():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_backfill_adds_incrementally():
|
||||
"""Backfilled vectors are upserted into the live collection without a rebuild."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_backfill")
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_backfill")
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await _seed_unembedded_chunk(store, "c", "c.md", "alpha extra")
|
||||
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await _seed_unembedded_chunk(store, "c", "c.md", "alpha extra")
|
||||
rebuilds = []
|
||||
original_rebuild = store._rebuild_collection
|
||||
store._rebuild_collection = lambda: rebuilds.append(True) or original_rebuild()
|
||||
|
||||
rebuilds = []
|
||||
original_rebuild = store._rebuild_collection
|
||||
store._rebuild_collection = lambda: rebuilds.append(True) or original_rebuild()
|
||||
await store._backfill_missing_embeddings()
|
||||
|
||||
await store._backfill_missing_embeddings()
|
||||
|
||||
assert not rebuilds
|
||||
assert store._indexed_ids == {"a", "c"}
|
||||
assert {c.id for c in await store.vector_search("alpha", 10, {})} == {"a", "c"}
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert not rebuilds
|
||||
assert store._indexed_ids == {"a", "c"}
|
||||
assert {c.id for c in await store.vector_search("alpha", 10, {})} == {"a", "c"}
|
||||
await store.close()
|
||||
|
||||
|
||||
# -- persistence ----------------------------------------------------------------
|
||||
|
||||
|
||||
def test_zvec_persistence_round_trip_reopens_without_rebuild():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_persistence_round_trip_reopens_without_rebuild():
|
||||
"""dump() + a fresh store reattaches the persisted collection via the sidecar."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
seed = await _started_store("t_zvec_persist")
|
||||
await seed.upsert(
|
||||
[
|
||||
(node("a.md"), [chunk("a", "a.md", "alpha text")]),
|
||||
(node("b.md"), [chunk("b", "b.md", "beta text")]),
|
||||
],
|
||||
)
|
||||
await seed.close()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
seed = await _started_store("t_zvec_persist")
|
||||
await seed.upsert(
|
||||
[
|
||||
(node("a.md"), [chunk("a", "a.md", "alpha text")]),
|
||||
(node("b.md"), [chunk("b", "b.md", "beta text")]),
|
||||
],
|
||||
)
|
||||
await seed.close()
|
||||
store = _new_zvec_store("t_zvec_persist")
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
|
||||
store = _new_zvec_store("t_zvec_persist")
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
rebuilds = []
|
||||
original_rebuild = store._rebuild_collection
|
||||
store._rebuild_collection = lambda: rebuilds.append(True) or original_rebuild()
|
||||
|
||||
rebuilds = []
|
||||
original_rebuild = store._rebuild_collection
|
||||
store._rebuild_collection = lambda: rebuilds.append(True) or original_rebuild()
|
||||
await store.start()
|
||||
|
||||
await store.start()
|
||||
|
||||
assert not rebuilds
|
||||
assert store._indexed_ids == {"a", "b"}
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})][0] == "a"
|
||||
assert [c.id for c in await store.vector_search("beta", 5, {})][0] == "b"
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert not rebuilds
|
||||
assert store._indexed_ids == {"a", "b"}
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})][0] == "a"
|
||||
assert [c.id for c in await store.vector_search("beta", 5, {})][0] == "b"
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_zvec_digest_mismatch_triggers_rebuild():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_digest_mismatch_triggers_rebuild():
|
||||
"""A sidecar whose digest fell behind the chunk JSONL forces a clean rebuild."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
seed = await _started_store("t_zvec_digest")
|
||||
await seed.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await seed.close()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
seed = await _started_store("t_zvec_digest")
|
||||
await seed.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await seed.close()
|
||||
sidecar = json.loads(seed.zvec_sidecar_path.read_text())
|
||||
sidecar["digest"] = "0" * 64
|
||||
seed.zvec_sidecar_path.write_text(json.dumps(sidecar))
|
||||
|
||||
sidecar = json.loads(seed.zvec_sidecar_path.read_text())
|
||||
sidecar["digest"] = "0" * 64
|
||||
seed.zvec_sidecar_path.write_text(json.dumps(sidecar))
|
||||
store = _new_zvec_store("t_zvec_digest")
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
|
||||
store = _new_zvec_store("t_zvec_digest")
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
rebuilds = []
|
||||
original_rebuild = store._rebuild_collection
|
||||
store._rebuild_collection = lambda: rebuilds.append(True) or original_rebuild()
|
||||
|
||||
rebuilds = []
|
||||
original_rebuild = store._rebuild_collection
|
||||
store._rebuild_collection = lambda: rebuilds.append(True) or original_rebuild()
|
||||
await store.start()
|
||||
|
||||
await store.start()
|
||||
|
||||
assert rebuilds == [True]
|
||||
assert store._indexed_ids == {"a"}
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert rebuilds == [True]
|
||||
assert store._indexed_ids == {"a"}
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_zvec_missing_sidecar_triggers_rebuild():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_missing_sidecar_triggers_rebuild():
|
||||
"""A collection directory without its sidecar cannot be trusted and is rebuilt."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
seed = await _started_store("t_zvec_no_sidecar")
|
||||
await seed.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await seed.close()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
seed = await _started_store("t_zvec_no_sidecar")
|
||||
await seed.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await seed.close()
|
||||
seed.zvec_sidecar_path.unlink()
|
||||
|
||||
seed.zvec_sidecar_path.unlink()
|
||||
store = _new_zvec_store("t_zvec_no_sidecar")
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
await store.start()
|
||||
|
||||
store = _new_zvec_store("t_zvec_no_sidecar")
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
await store.start()
|
||||
|
||||
assert store._indexed_ids == {"a"}
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert store._indexed_ids == {"a"}
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_zvec_hnsw_m_mismatch_triggers_rebuild():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_hnsw_m_mismatch_triggers_rebuild():
|
||||
"""A persisted collection built with a different HNSW M is rebuilt."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
seed = await _started_store("t_zvec_m_mismatch", hnsw_m=16)
|
||||
await seed.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await seed.close()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
seed = await _started_store("t_zvec_m_mismatch", hnsw_m=16)
|
||||
await seed.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await seed.close()
|
||||
store = _new_zvec_store("t_zvec_m_mismatch", hnsw_m=48)
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
|
||||
store = _new_zvec_store("t_zvec_m_mismatch", hnsw_m=48)
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
rebuilds = []
|
||||
original_rebuild = store._rebuild_collection
|
||||
store._rebuild_collection = lambda: rebuilds.append(True) or original_rebuild()
|
||||
|
||||
rebuilds = []
|
||||
original_rebuild = store._rebuild_collection
|
||||
store._rebuild_collection = lambda: rebuilds.append(True) or original_rebuild()
|
||||
await store.start()
|
||||
|
||||
await store.start()
|
||||
|
||||
assert rebuilds == [True]
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert rebuilds == [True]
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_zvec_missing_collection_document_triggers_rebuild():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_missing_collection_document_triggers_rebuild():
|
||||
"""A collection that lost a document is rebuilt even though the sidecar matches."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
seed = await _started_store("t_zvec_missing_doc")
|
||||
await seed.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await seed.close()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
seed = await _started_store("t_zvec_missing_doc")
|
||||
await seed.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await seed.close()
|
||||
_tamper_with_collection(seed, lambda _zvec, collection: collection.delete(ids=["a"]))
|
||||
# The sidecar still claims the chunk is indexed.
|
||||
assert json.loads(seed.zvec_sidecar_path.read_text())["ids"] == ["a"]
|
||||
|
||||
_tamper_with_collection(seed, lambda _zvec, collection: collection.delete(ids=["a"]))
|
||||
# The sidecar still claims the chunk is indexed.
|
||||
assert json.loads(seed.zvec_sidecar_path.read_text())["ids"] == ["a"]
|
||||
store = _new_zvec_store("t_zvec_missing_doc")
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
rebuilds = _track_rebuilds(store)
|
||||
await store.start()
|
||||
|
||||
store = _new_zvec_store("t_zvec_missing_doc")
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
rebuilds = _track_rebuilds(store)
|
||||
await store.start()
|
||||
|
||||
assert rebuilds == [True]
|
||||
assert store._indexed_ids == {"a"}
|
||||
assert store._collection.stats.doc_count == 1
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert rebuilds == [True]
|
||||
assert store._indexed_ids == {"a"}
|
||||
assert store._collection.stats.doc_count == 1
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_zvec_unexpected_collection_document_triggers_rebuild():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_unexpected_collection_document_triggers_rebuild():
|
||||
"""A collection holding a document no chunk owns is rebuilt."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
seed = await _started_store("t_zvec_extra_doc")
|
||||
await seed.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await seed.close()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
seed = await _started_store("t_zvec_extra_doc")
|
||||
await seed.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await seed.close()
|
||||
_tamper_with_collection(
|
||||
seed,
|
||||
lambda zvec, collection: collection.upsert(
|
||||
[zvec.Doc(id="ghost", vectors={"embedding": [0.0, 1.0]})],
|
||||
),
|
||||
)
|
||||
|
||||
_tamper_with_collection(
|
||||
seed,
|
||||
lambda zvec, collection: collection.upsert(
|
||||
[zvec.Doc(id="ghost", vectors={"embedding": [0.0, 1.0]})],
|
||||
),
|
||||
)
|
||||
store = _new_zvec_store("t_zvec_extra_doc")
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
rebuilds = _track_rebuilds(store)
|
||||
await store.start()
|
||||
|
||||
store = _new_zvec_store("t_zvec_extra_doc")
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
rebuilds = _track_rebuilds(store)
|
||||
await store.start()
|
||||
|
||||
assert rebuilds == [True]
|
||||
assert store._indexed_ids == {"a"}
|
||||
assert store._collection.stats.doc_count == 1
|
||||
assert [c.id for c in await store.vector_search("beta", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert rebuilds == [True]
|
||||
assert store._indexed_ids == {"a"}
|
||||
assert store._collection.stats.doc_count == 1
|
||||
assert [c.id for c in await store.vector_search("beta", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_zvec_corrupted_collection_vector_triggers_rebuild():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_corrupted_collection_vector_triggers_rebuild():
|
||||
"""An expected id whose stored vector was replaced is rebuilt from the chunks."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
seed = await _started_store("t_zvec_bad_vector")
|
||||
await seed.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await seed.close()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
seed = await _started_store("t_zvec_bad_vector")
|
||||
await seed.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await seed.close()
|
||||
_tamper_with_collection(
|
||||
seed,
|
||||
lambda zvec, collection: collection.upsert(
|
||||
[zvec.Doc(id="a", vectors={"embedding": [0.0, 1.0]})],
|
||||
),
|
||||
)
|
||||
|
||||
_tamper_with_collection(
|
||||
seed,
|
||||
lambda zvec, collection: collection.upsert(
|
||||
[zvec.Doc(id="a", vectors={"embedding": [0.0, 1.0]})],
|
||||
),
|
||||
)
|
||||
store = _new_zvec_store("t_zvec_bad_vector")
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
rebuilds = _track_rebuilds(store)
|
||||
await store.start()
|
||||
|
||||
store = _new_zvec_store("t_zvec_bad_vector")
|
||||
store.embedding_store = FakeEmbeddingStore()
|
||||
rebuilds = _track_rebuilds(store)
|
||||
await store.start()
|
||||
|
||||
assert rebuilds == [True]
|
||||
# The rebuilt collection carries the vector of the authoritative chunk.
|
||||
results = await store.vector_search("alpha", 5, {})
|
||||
assert [c.id for c in results] == ["a"]
|
||||
assert results[0].scores["vector"] == pytest.approx(1.0, abs=1e-5)
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert rebuilds == [True]
|
||||
# The rebuilt collection carries the vector of the authoritative chunk.
|
||||
results = await store.vector_search("alpha", 5, {})
|
||||
assert [c.id for c in results] == ["a"]
|
||||
assert results[0].scores["vector"] == pytest.approx(1.0, abs=1e-5)
|
||||
await store.close()
|
||||
|
||||
|
||||
def test_zvec_stale_embedding_dimension_rebuilds_from_backfill():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_stale_embedding_dimension_rebuilds_from_backfill():
|
||||
"""Persisted vectors with a stale dimension are dropped and re-embedded."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
seed = await _started_store("t_zvec_stale_dim")
|
||||
await seed.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await seed.close()
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
seed = await _started_store("t_zvec_stale_dim")
|
||||
await seed.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await seed.close()
|
||||
class WideEmbeddingStore(FakeEmbeddingStore):
|
||||
"""Same provider with a different dimension."""
|
||||
|
||||
class WideEmbeddingStore(FakeEmbeddingStore):
|
||||
"""Same provider with a different dimension."""
|
||||
dimensions = 4
|
||||
|
||||
dimensions = 4
|
||||
def _embed(self, text: str) -> np.ndarray:
|
||||
base = super()._embed(text)
|
||||
return np.concatenate([base, base]).astype(np.float16)
|
||||
|
||||
def _embed(self, text: str) -> np.ndarray:
|
||||
base = super()._embed(text)
|
||||
return np.concatenate([base, base]).astype(np.float16)
|
||||
store = _new_zvec_store("t_zvec_stale_dim")
|
||||
store.embedding_store = WideEmbeddingStore()
|
||||
await store.start()
|
||||
# Startup backfill re-embeds the stale chunk in the background.
|
||||
await store._embedding_backfill_task
|
||||
|
||||
store = _new_zvec_store("t_zvec_stale_dim")
|
||||
store.embedding_store = WideEmbeddingStore()
|
||||
await store.start()
|
||||
# Startup backfill re-embeds the stale chunk in the background.
|
||||
await store._embedding_backfill_task
|
||||
|
||||
assert store._collection.schema.vectors[0].dimension == 4
|
||||
assert store._indexed_ids == {"a"}
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert store._collection.schema.vectors[0].dimension == 4
|
||||
assert store._indexed_ids == {"a"}
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
|
||||
# -- packaging ------------------------------------------------------------
|
||||
|
|
@ -599,17 +480,14 @@ def test_zvec_declared_as_installable_dependency():
|
|||
# -- maintenance ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_zvec_optimize_index_runs_native_optimize():
|
||||
@pytest.mark.asyncio
|
||||
async def test_zvec_optimize_index_runs_native_optimize():
|
||||
"""optimize_index() delegates to Collection.optimize() and keeps search intact."""
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_optimize")
|
||||
|
||||
async def go():
|
||||
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
|
||||
store = await _started_store("t_zvec_optimize")
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await store.optimize_index()
|
||||
|
||||
await store.upsert([(node("a.md"), [chunk("a", "a.md", "alpha text")])])
|
||||
await store.optimize_index()
|
||||
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
||||
run(go())
|
||||
assert [c.id for c in await store.vector_search("alpha", 5, {})] == ["a"]
|
||||
await store.close()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue