ReMe/reme4/steps/file_io/list.py
jinliyl a4efc0f776
refactor(reme4): restructure steps packages (#258)
* fix(bm25_index): 修正BM25索引计算中的文档长度归一化问题

修复了在计算BM25相似度时对文档长度进行不正确归一化的bug,确保所有查询都能得到准确的相关性评分。

* up

* up

* up

* up

* up

* up

* up

* up

* up

* up

* up

* up

* up

* up

* up

* up

* up

* refactor(steps): Rename and adjust indexing step logic

- Rename `scan_changes.py` and `reindex.py` to `clear_and_scan.py`
- Update implementation details of `ScanChangesStep` and `ClearAndScanStep`
- Modify the scheduling mechanism in `WatchChangesStep`
- Adjust step registration and parameter configuration in config files
- Update related tests to align with the new interface changes

* up

* feat(daily): replace daily CRUD operations with slug provisioning approach

* refactor(tests): migrate CRUD step tests from HTTP server to direct LocalFileStore

* up

* up

* up

* up

---------

Co-authored-by: huangsen <huangsen.huang@alibaba-inc.com>
2026-05-28 14:30:30 +08:00

104 lines
4.3 KiB
Python

"""``file_list`` — enumerate files under a directory in the vault.
Reads directly from the filesystem (``Path.iterdir`` / ``Path.rglob``),
**not** the file_store index. The store may lag behind disk during
indexing or after rapid mutations; the on-disk walk is the source of truth.
Parameters:
path — dir to list under (relative to the vault or absolute). Empty = vault root.
limit — cap on the number of returned items (default 100, must be > 0).
recursive — descend into subdirectories. Default False = direct children only.
No frontmatter is read. Callers needing frontmatter-based filtering
should iterate the result and call ``frontmatter_read`` per candidate.
"""
from pathlib import Path
from typing import Iterable
from ..base_step import BaseStep
from ...components import R
# Default cap on returned items so huge vaults don't blow up the response.
DEFAULT_LIMIT = 100
@R.register("list_step")
class ListStep(BaseStep):
"""Enumerate files under a directory in the vault."""
def _fail(self, message: str, **meta) -> None:
"""Set a failed response (matches the read/edit/... fail envelope)."""
assert self.context is not None
self.context.response.success = False
self.context.response.answer = f"Error: {message}"
if meta:
self.context.response.metadata.update(meta)
def _collect_params(self) -> tuple[str, bool, int]:
"""Read ``path`` / ``recursive`` / ``limit`` from context; coerce permissively."""
assert self.context is not None
path = str(self.context.get("path") or "")
recursive = bool(self.context.get("recursive", False))
raw_limit = self.context.get("limit")
# Strings like "50" are accepted; bad/non-positive values fall back to default.
try:
limit = int(raw_limit) if raw_limit is not None else DEFAULT_LIMIT
except (TypeError, ValueError):
limit = DEFAULT_LIMIT
return path, recursive, limit if limit > 0 else DEFAULT_LIMIT
@staticmethod
def _resolve_target_dir(vault_dir: Path, path: str) -> Path:
"""Empty → vault root; absolute → as-is; relative → joined under vault_dir."""
if not path:
return vault_dir
candidate = Path(path)
return candidate.resolve() if candidate.is_absolute() else (vault_dir / candidate).resolve()
@staticmethod
def _walk_files(target_dir: Path, recursive: bool, limit: int) -> list[Path]:
"""Return up to ``limit`` regular files under ``target_dir``; short-circuits at the cap."""
entries: Iterable[Path] = target_dir.rglob("*") if recursive else target_dir.iterdir()
files: list[Path] = []
for entry in entries:
if not entry.is_file(): # skip dirs, sockets, broken links, etc.
continue
files.append(entry)
if len(files) >= limit:
break
return files
@staticmethod
def _format_relative(files: list[Path], vault_dir: Path) -> list[str]:
"""Render as vault-relative paths; fall back to absolute when outside the vault."""
out: list[str] = []
for entry in files:
try:
out.append(str(entry.relative_to(vault_dir)))
except ValueError:
out.append(str(entry))
return out
async def execute(self):
assert self.context is not None
path, recursive, limit = self._collect_params()
vault_dir = Path(self.file_store.vault_path or ".").resolve()
target_dir = self._resolve_target_dir(vault_dir, path)
if not target_dir.exists():
self._fail(f"directory {target_dir} does not exist", path=str(target_dir))
return None
if not target_dir.is_dir():
self._fail(f"path {target_dir} is not a directory", path=str(target_dir))
return None
items = self._format_relative(self._walk_files(target_dir, recursive, limit), vault_dir)
self.context.response.success = True
self.context.response.answer = f"Listed {len(items)} file(s) under {path or '.'}"
self.context.response.metadata.update({"items": items, "count": len(items)})
self.logger.info(
f"[{self.name}] listed dir={target_dir} recursive={recursive} count={len(items)} limit={limit}",
)
return self.context.response