Merge 23b161f391 into ebcb154e37

local-deploy/Start-ReMe.ps1

@@ -0,0 +1,73 @@
+[CmdletBinding()]
+param()
+
+$ErrorActionPreference = 'Stop'
+
+$ReMeRoot = 'D:\projects\reme'
+$DataRoot = 'D:\projects\reme-data'
+$Executable = Join-Path $ReMeRoot '.venv\Scripts\reme.exe'
+$Config = Join-Path $ReMeRoot 'local-deploy\safe.yaml'
+$LogRoot = Join-Path $ReMeRoot 'local-deploy\logs'
+$PidFile = Join-Path $LogRoot 'reme.pid'
+$Port = 2333
+
+if (-not (Test-Path -LiteralPath $Executable)) {
+    throw "ReMe executable not found: $Executable"
+}
+if (-not (Test-Path -LiteralPath $Config)) {
+    throw "ReMe safe config not found: $Config"
+}
+
+$listeners = @(Get-NetTCPConnection -State Listen -LocalPort $Port -ErrorAction SilentlyContinue)
+if ($listeners.Count -gt 0) {
+    $owners = $listeners | Select-Object -ExpandProperty OwningProcess -Unique
+    foreach ($owner in $owners) {
+        $process = Get-CimInstance Win32_Process -Filter "ProcessId = $owner"
+        if ($process.CommandLine -like "*$ReMeRoot*" -and $process.CommandLine -match '\bstart\b') {
+            Write-Output "ReMe is already running on 127.0.0.1:$Port (PID $owner)."
+            exit 0
+        }
+    }
+    throw "Port $Port is already owned by another process."
+}
+
+New-Item -ItemType Directory -Force -Path $DataRoot, $LogRoot | Out-Null
+
+# Keep optional model-backed jobs unavailable in this deployment process.
+foreach ($name in @('LLM_API_KEY', 'EMBEDDING_API_KEY', 'CLAUDE_CODE_API_KEY')) {
+    Remove-Item -LiteralPath "Env:$name" -ErrorAction SilentlyContinue
+}
+$env:NO_PROXY = '127.0.0.1,localhost,::1'
+$env:PYTHONUTF8 = '1'
+$env:PYTHONIOENCODING = 'utf-8'
+
+$arguments = @(
+    'start'
+    "config=$Config"
+    "workspace_dir=$DataRoot"
+    'service.host=127.0.0.1'
+    "service.port=$Port"
+)
+
+$process = Start-Process `
+    -FilePath $Executable `
+    -ArgumentList $arguments `
+    -WorkingDirectory $ReMeRoot `
+    -WindowStyle Hidden `
+    -RedirectStandardOutput (Join-Path $LogRoot 'reme.out.log') `
+    -RedirectStandardError (Join-Path $LogRoot 'reme.err.log') `
+    -PassThru
+
+Set-Content -LiteralPath $PidFile -Value $process.Id -Encoding ascii
+Start-Sleep -Milliseconds 750
+if ($process.HasExited) {
+    $errorLog = Join-Path $LogRoot 'reme.err.log'
+    $details = if (Test-Path -LiteralPath $errorLog) {
+        (Get-Content -LiteralPath $errorLog -Tail 20 -Encoding UTF8) -join [Environment]::NewLine
+    } else {
+        'No error log was produced.'
+    }
+    throw "ReMe exited during startup.$([Environment]::NewLine)$details"
+}
+
+Write-Output "Started ReMe on 127.0.0.1:$Port (PID $($process.Id))."

local-deploy/Stop-ReMe.ps1

@@ -0,0 +1,32 @@
+[CmdletBinding()]
+param()
+
+$ErrorActionPreference = 'Stop'
+
+$ReMeRoot = 'D:\projects\reme'
+$LogRoot = Join-Path $ReMeRoot 'local-deploy\logs'
+$PidFile = Join-Path $LogRoot 'reme.pid'
+$Port = 2333
+$listeners = @(Get-NetTCPConnection -State Listen -LocalPort $Port -ErrorAction SilentlyContinue)
+
+if ($listeners.Count -eq 0) {
+    Write-Output "No listener is running on port $Port."
+    exit 0
+}
+
+$owners = $listeners | Select-Object -ExpandProperty OwningProcess -Unique
+foreach ($owner in $owners) {
+    $process = Get-CimInstance Win32_Process -Filter "ProcessId = $owner"
+    if ($null -eq $process) {
+        continue
+    }
+    if ($process.CommandLine -notlike "*$ReMeRoot*" -or $process.CommandLine -notmatch '\bstart\b') {
+        throw "Refusing to stop PID $owner because it is not the local ReMe service."
+    }
+    Stop-Process -Id $owner -Force
+    Write-Output "Stopped ReMe process $owner."
+}
+
+if (Test-Path -LiteralPath $PidFile) {
+    Remove-Item -LiteralPath $PidFile -Force
+}

local-deploy/Test-ReMe.ps1

@@ -0,0 +1,68 @@
+[CmdletBinding()]
+param(
+    [ValidateRange(1, 600)]
+    [int]$TimeoutSeconds = 120
+)
+
+$ErrorActionPreference = 'Stop'
+
+$ReMeRoot = 'D:\projects\reme'
+$Executable = Join-Path $ReMeRoot '.venv\Scripts\reme.exe'
+$Port = 2333
+$Uri = "http://127.0.0.1:$Port/version"
+$env:NO_PROXY = '127.0.0.1,localhost,::1'
+$env:PYTHONUTF8 = '1'
+$env:PYTHONIOENCODING = 'utf-8'
+$deadline = [DateTime]::UtcNow.AddSeconds($TimeoutSeconds)
+$version = $null
+
+do {
+    try {
+        $version = Invoke-RestMethod -Method Post -Uri $Uri -ContentType 'application/json' -Body '{}'
+        break
+    } catch {
+        if ([DateTime]::UtcNow -ge $deadline) {
+            throw "ReMe did not become ready within $TimeoutSeconds seconds: $($_.Exception.Message)"
+        }
+        Start-Sleep -Milliseconds 500
+    }
+} while ($true)
+
+$listeners = @(Get-NetTCPConnection -State Listen -LocalPort $Port -ErrorAction Stop)
+$invalid = @($listeners | Where-Object { $_.LocalAddress -ne '127.0.0.1' })
+if ($invalid.Count -gt 0) {
+    $addresses = ($invalid | Select-Object -ExpandProperty LocalAddress -Unique) -join ', '
+    throw "ReMe is listening outside loopback: $addresses"
+}
+
+$owners = $listeners | Select-Object -ExpandProperty OwningProcess -Unique
+foreach ($owner in $owners) {
+    $process = Get-CimInstance Win32_Process -Filter "ProcessId = $owner"
+    if ($null -eq $process -or $process.CommandLine -notlike "*$ReMeRoot*") {
+        throw "Port $Port is not owned by the expected ReMe deployment."
+    }
+}
+
+foreach ($action in @('shell', 'auto_memory', 'auto_resource', 'auto_dream')) {
+    $blocked = Invoke-WebRequest `
+        -SkipHttpErrorCheck `
+        -Method Post `
+        -Uri "http://127.0.0.1:$Port/$action" `
+        -ContentType 'application/json' `
+        -Body '{}'
+    if ($blocked.StatusCode -ne 404) {
+        throw "Unsafe ReMe action remains exposed: $action returned HTTP $($blocked.StatusCode)."
+    }
+}
+
+$health = & $Executable health_check 2>&1
+if ($LASTEXITCODE -ne 0) {
+    throw "ReMe health_check failed: $($health -join [Environment]::NewLine)"
+}
+
+[pscustomobject]@{
+    Address = "127.0.0.1:$Port"
+    Version = $version
+    ListenerPids = @($owners)
+    HealthCheck = ($health -join [Environment]::NewLine)
+}

local-deploy/build_safe_config.py

@@ -0,0 +1,89 @@
+"""Build the loopback deployment's allowlisted ReMe configuration."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+REME_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_CONFIG = REME_ROOT / "reme" / "config" / "default.yaml"
+OUTPUT_CONFIG = Path(__file__).resolve().with_name("safe.yaml")
+
+ALLOWED_JOBS = (
+    "index_update_loop",
+    "version",
+    "health_check",
+    "status",
+    "help",
+    "traverse",
+    "reindex",
+    "search",
+    "node_search",
+    "daily_list",
+    "daily_reindex",
+    "frontmatter_delete",
+    "frontmatter_read",
+    "frontmatter_update",
+    "stat",
+    "list",
+    "move",
+    "delete",
+    "read",
+    "read_image",
+    "write",
+    "daily_write",
+    "edit",
+)
+
+ALLOWED_COMPONENTS = (
+    "tokenizer",
+    "file_graph",
+    "file_catalog",
+    "file_chunker",
+    "keyword_index",
+    "file_store",
+)
+
+
+def select(mapping: dict, names: tuple[str, ...], label: str) -> dict:
+    """Return the allowlisted entries and reject missing defaults."""
+    missing = [name for name in names if name not in mapping]
+    if missing:
+        raise RuntimeError(f"Default config is missing required {label}: {missing}")
+    return {name: mapping[name] for name in names}
+
+
+def build_safe_config() -> dict:
+    """Build the deployment config from the pinned upstream defaults."""
+    with DEFAULT_CONFIG.open(encoding="utf-8") as source:
+        config = yaml.safe_load(source)
+
+    config["jobs"] = select(config["jobs"], ALLOWED_JOBS, "jobs")
+    config["components"] = select(config["components"], ALLOWED_COMPONENTS, "components")
+    return config
+
+
+def validate_safe_config(config: dict) -> None:
+    """Verify the generated jobs and components match the allowlists."""
+    if tuple(config.get("jobs", {})) != ALLOWED_JOBS:
+        raise RuntimeError("Generated job set does not match the deployment allowlist")
+    if tuple(config.get("components", {})) != ALLOWED_COMPONENTS:
+        raise RuntimeError("Generated component set does not match the deployment allowlist")
+
+
+def main() -> None:
+    """Write and validate the loopback deployment config."""
+    config = build_safe_config()
+    validate_safe_config(config)
+    OUTPUT_CONFIG.write_text(
+        yaml.safe_dump(config, sort_keys=False, allow_unicode=True),
+        encoding="utf-8",
+    )
+    with OUTPUT_CONFIG.open(encoding="utf-8") as generated:
+        validate_safe_config(yaml.safe_load(generated))
+    print(f"Wrote {OUTPUT_CONFIG}")
+
+
+if __name__ == "__main__":
+    main()

local-deploy/deployment.json

@@ -0,0 +1,14 @@
+{
+  "upstream": "https://github.com/agentscope-ai/ReMe",
+  "commit": "c3b1e9391840537e769c32985f1ae4c6523be87b",
+  "remeVersion": "0.4.1.1",
+  "pythonVersion": "3.13.0",
+  "serviceAddress": "http://127.0.0.1:2333",
+  "workspace": "D:\\projects\\reme-data",
+  "config": "D:\\projects\\reme\\local-deploy\\safe.yaml",
+  "safeConfigSha256": "C822ECBFDC53D56C8E1981947087C552A7FB3B8B076637953D16608BC0E172C6",
+  "exposedJobCount": 23,
+  "blockedActions": ["shell", "auto_memory", "auto_resource", "auto_dream"],
+  "llmFeatures": "disabled-no-credentials",
+  "verifiedAtUtc": "2026-07-15T18:56:22.4536541Z"
+}

local-deploy/safe.yaml

@@ -0,0 +1,464 @@
+service:
+  backend: http
+jobs:
+  index_update_loop:
+    backend: background
+    max_file_bytes: 20971520
+    watch_dirs:
+    - daily_dir
+    - digest_dir
+    - resource_dir
+    watch_suffixes:
+    - md
+    - jsonl
+    steps:
+    - backend: init_changes_step
+      monitor_type: file_store
+      monitor_name: default
+      dispatch_steps:
+      - update_index_step
+    - backend: watch_changes_step
+      dispatch_steps:
+      - backend: update_index_step
+        persist: false
+  version:
+    backend: base
+    description: return reme package version
+    parameters:
+      type: object
+      properties: {}
+    steps:
+    - backend: version_step
+  health_check:
+    backend: base
+    description: return a concise health-check snapshot of reme components
+    parameters:
+      type: object
+      properties: {}
+    steps:
+    - backend: health_check_step
+  status:
+    backend: base
+    description: report memory estimates for stateful data components and process
+      RSS
+    parameters:
+      type: object
+      properties: {}
+    steps:
+    - backend: status_step
+  help:
+    backend: base
+    description: list all registered jobs with their metadata
+    parameters:
+      type: object
+      properties: {}
+    steps:
+    - backend: help_step
+  traverse:
+    backend: base
+    description: Walk the wikilink graph from a path.
+    parameters:
+      type: object
+      properties:
+        path:
+          type: string
+          description: path
+        depth:
+          type: integer
+          description: hop limit
+          default: 1
+        direction:
+          type: string
+          enum:
+          - forward
+          - backward
+          - both
+          default: both
+      required:
+      - path
+    steps:
+    - backend: traverse_step
+  reindex:
+    backend: base
+    max_file_bytes: 20971520
+    description: wipe the file store and rebuild it from the existing files
+    watch_dirs:
+    - daily_dir
+    - digest_dir
+    - resource_dir
+    watch_suffixes:
+    - md
+    - jsonl
+    parameters:
+      type: object
+      properties: {}
+    steps:
+    - backend: clear_store_step
+    - backend: init_changes_step
+      monitor_type: file_store
+      monitor_name: default
+      dispatch_steps:
+      - update_index_step
+  search:
+    backend: base
+    description: Hybrid workspace search (vector + BM25, RRF-fused).
+    parameters:
+      type: object
+      properties:
+        query:
+          type: string
+          description: search query
+        limit:
+          type: integer
+          description: max results
+          default: 5
+        min_score:
+          type: number
+          description: min fused score
+          default: 0.0
+        start_date:
+          type: string
+          description: optional inclusive start date filter (YYYY-MM-DD); results
+            earlier than this date are excluded
+        end_date:
+          type: string
+          description: optional inclusive end date filter (YYYY-MM-DD); results later
+            than this date are excluded
+      required:
+      - query
+    steps:
+    - backend: search_step
+      vector_weight: 0.7
+      candidate_multiplier: 5.0
+      expand_links: true
+      max_links_per_direction: 10
+  node_search:
+    backend: base
+    description: Digest node recall — given a candidate abstraction's name+description,
+      surface existing digest nodes similar enough to either dedup against or link
+      to as related.
+    parameters:
+      type: object
+      properties:
+        query:
+          type: string
+          description: search query
+        limit:
+          type: integer
+          description: max digest nodes to return
+          default: 20
+      required:
+      - query
+    steps:
+    - backend: node_search_step
+      vector_weight: 0.7
+      candidate_multiplier: 5.0
+  daily_list:
+    backend: base
+    description: List notes under a single day.
+    parameters:
+      type: object
+      properties:
+        date:
+          type: string
+          description: YYYY-MM-DD; empty = today
+          default: ''
+    steps:
+    - backend: daily_list_step
+  daily_reindex:
+    backend: base
+    description: Rebuild the day-index page daily/<date>.md.
+    parameters:
+      type: object
+      properties:
+        date:
+          type: string
+          description: YYYY-MM-DD; empty = today
+          default: ''
+    steps:
+    - backend: daily_reindex_step
+  frontmatter_delete:
+    backend: base
+    description: Drop keys from a file's frontmatter.
+    parameters:
+      type: object
+      properties:
+        path:
+          type: string
+          description: workspace-relative path
+        keys:
+          type: array
+          description: keys to remove
+          items:
+            type: string
+      required:
+      - path
+      - keys
+    steps:
+    - backend: frontmatter_delete_step
+  frontmatter_read:
+    backend: base
+    description: Read a file's frontmatter as a dict.
+    parameters:
+      type: object
+      properties:
+        path:
+          type: string
+          description: workspace-relative path
+      required:
+      - path
+    steps:
+    - backend: frontmatter_read_step
+  frontmatter_update:
+    backend: base
+    description: Merge key-values into a file's frontmatter.
+    parameters:
+      type: object
+      properties:
+        path:
+          type: string
+          description: workspace-relative path
+        metadata:
+          type: object
+          description: key-values to merge
+      required:
+      - path
+      - metadata
+    steps:
+    - backend: frontmatter_update_step
+  stat:
+    backend: base
+    description: Stat path (size, mtime, exists, is_dir, is_file).
+    parameters:
+      type: object
+      properties:
+        path:
+          type: string
+          description: workspace-relative path
+      required:
+      - path
+    steps:
+    - backend: stat_step
+  list:
+    backend: base
+    description: List files under a workspace path.
+    parameters:
+      type: object
+      properties:
+        path:
+          type: string
+          description: workspace-relative dir; empty = root
+          default: ''
+        recursive:
+          type: boolean
+          description: recurse
+          default: false
+        limit:
+          type: integer
+          description: max results
+          default: 100
+    steps:
+    - backend: list_step
+  move:
+    backend: base
+    description: Move / rename a workspace file; rewrites inbound wikilinks by default.
+    parameters:
+      type: object
+      properties:
+        src_path:
+          type: string
+          description: workspace-relative source
+        dst_path:
+          type: string
+          description: workspace-relative destination
+        overwrite:
+          type: boolean
+          description: overwrite if dst exists
+          default: false
+        retarget:
+          type: boolean
+          description: rewrite [[src]] → [[dst]] across the workspace
+          default: true
+      required:
+      - src_path
+      - dst_path
+    steps:
+    - backend: move_step
+  delete:
+    backend: base
+    description: Delete a workspace file or folder; returns surviving inbound wikilinks.
+    parameters:
+      type: object
+      properties:
+        path:
+          type: string
+          description: workspace-relative path
+      required:
+      - path
+    steps:
+    - backend: delete_step
+  read:
+    backend: base
+    description: Read a markdown file under the workspace.
+    parameters:
+      type: object
+      properties:
+        path:
+          type: string
+          description: workspace-relative path; markdown only
+        start_line:
+          type: integer
+          description: first line (1-based, inclusive)
+        end_line:
+          type: integer
+          description: last line (1-based, inclusive)
+      required:
+      - path
+    steps:
+    - backend: read_step
+      with_neighbors: false
+      max_neighbors_per_direction: 10
+  read_image:
+    backend: base
+    description: Read an image file as base64 (workspace-relative path).
+    parameters:
+      type: object
+      properties:
+        path:
+          type: string
+          description: workspace-relative path; common image formats supported (png/jpg/jpeg/webp/gif/bmp/tiff/heic)
+      required:
+      - path
+    steps:
+    - backend: read_image_step
+      max_bytes: 5242880
+  write:
+    backend: base
+    description: Write a markdown file (create or overwrite) with name/description
+      frontmatter.
+    parameters:
+      type: object
+      properties:
+        path:
+          type: string
+          description: workspace-relative path; markdown only
+        name:
+          type: string
+          description: frontmatter name
+        description:
+          type: string
+          description: frontmatter description
+        content:
+          type: string
+          description: body
+        metadata:
+          type: object
+          description: Optional extra frontmatter fields (md only).
+      required:
+      - path
+      - name
+      - description
+      - content
+    steps:
+    - backend: write_step
+  daily_write:
+    backend: base
+    description: Write a daily markdown note with conversation source frontmatter.
+    parameters:
+      type: object
+      properties:
+        name:
+          type: string
+          description: daily note filename stem and frontmatter name
+        description:
+          type: string
+          description: frontmatter description
+        session_id:
+          type: string
+          description: source conversation session identifier
+        content:
+          type: string
+          description: body
+        date:
+          type: string
+          description: YYYY-MM-DD daily note date; empty = today
+          default: ''
+        metadata:
+          type: object
+          description: Optional extra frontmatter fields.
+      required:
+      - name
+      - description
+      - session_id
+      - content
+    steps:
+    - backend: daily_write_step
+  edit:
+    backend: base
+    description: Find-and-replace in a markdown file (all occurrences).
+    parameters:
+      type: object
+      properties:
+        path:
+          type: string
+          description: workspace-relative path
+        old:
+          type: string
+          description: text to find
+        new:
+          type: string
+          description: replacement
+          default: ''
+      required:
+      - path
+      - old
+      - new
+    steps:
+    - backend: edit_step
+components:
+  tokenizer:
+    default:
+      backend: regex
+  file_graph:
+    default:
+      backend: local
+  file_catalog:
+    default:
+      backend: local
+    resource:
+      backend: local
+    digest:
+      backend: local
+    dream:
+      backend: local
+  file_chunker:
+    markdown:
+      backend: markdown
+      supported_extensions:
+      - md
+      include_frontmatter_in_metadata: false
+      include_frontmatter_keys_in_metadata: []
+    json:
+      backend: json
+      supported_extensions:
+      - json
+    jsonl:
+      backend: jsonl
+      supported_extensions:
+      - jsonl
+    default:
+      backend: default
+      supported_extensions:
+      - txt
+      - log
+  keyword_index:
+    default:
+      backend: bm25
+      tokenizer: default
+  file_store:
+    default:
+      backend: local
+      store_name: local
+      embedding_store: ''
+      keyword_index: default
+      file_graph: default