diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 88de942c..82a80a4a 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -23,6 +23,20 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: website/package-lock.json + - name: Build web workspace + working-directory: website + run: | + npm ci + npm run build:static + rm -rf ../reme/web + mkdir -p ../reme/web + cp -R dist-static/. ../reme/web/ - name: Set up Python uses: actions/setup-python@v6 with: @@ -38,6 +52,7 @@ jobs: WHEEL="$(ls dist/*.whl)" pip install "${WHEEL}[core]" python -c "import reme; print(reme.__version__)" + python -c "from importlib.resources import files; assert (files('reme') / 'web' / 'index.html').is_file()" - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/README.md b/README.md index aa54b69d..ea9a122c 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,10 @@ reme start service.port=8181 After startup, check the service status. If you use a custom port, replace `2333` in the URL below with that port. +The HTTP service also serves the bundled ReMe Workspace at when the web build is available. +Set `service.web_enabled=false` to disable it, or use `service.web_static_dir` / `REME_WEB_STATIC_DIR` to provide a custom +static build. + ```bash reme version curl -s http://127.0.0.1:2333/version -H 'Content-Type: application/json' -d '{}' diff --git a/README_ZH.md b/README_ZH.md index 212566cb..3a888bb1 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -121,6 +121,9 @@ reme start service.port=8181 启动后可以检查服务状态；如果使用了自定义端口，请将下面 URL 中的 `2333` 替换为对应端口。 +如果安装包中包含 Web 构建产物，HTTP 服务还会在 提供 ReMe Workspace。可以设置 +`service.web_enabled=false` 关闭，或通过 `service.web_static_dir` / `REME_WEB_STATIC_DIR` 指定自定义静态目录。 + ```bash reme version curl -s http://127.0.0.1:2333/version -H 'Content-Type: application/json' -d '{}' diff --git a/pyproject.toml b/pyproject.toml index b863e98b..c8f6600f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,6 +81,7 @@ include-package-data = true [tool.setuptools.package-data] "*" = ["py.typed", "**/*.yaml", "**/*.json"] +"reme" = ["web/**"] "reme.components.tokenizer" = ["stopwords"] [tool.setuptools.dynamic] diff --git a/reme/components/service/base_service.py b/reme/components/service/base_service.py index c1afd2cf..fed150e2 100644 --- a/reme/components/service/base_service.py +++ b/reme/components/service/base_service.py @@ -44,6 +44,9 @@ class BaseService(BaseComponent): def start_service(self, app: "Application") -> None: """Block on serving requests until shutdown.""" + def finalize_service(self, app: "Application") -> None: + """Register routes that must be added after every configured job.""" + # ----- Shared helpers ------------------------------------------------ def _lifespan(self, app: "Application", host: str, port: int): @@ -97,4 +100,5 @@ class BaseService(BaseComponent): """Build the service, register jobs, then start serving (blocking).""" self.build_service(app) self.add_jobs(app) + self.finalize_service(app) self.start_service(app) diff --git a/reme/components/service/http_service.py b/reme/components/service/http_service.py index d8e010c6..a61369c8 100644 --- a/reme/components/service/http_service.py +++ b/reme/components/service/http_service.py @@ -3,19 +3,21 @@ import asyncio import warnings from collections.abc import AsyncGenerator +from pathlib import Path from typing import TYPE_CHECKING import uvicorn -from fastapi import FastAPI +from fastapi import FastAPI, HTTPException from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import StreamingResponse +from fastapi.responses import FileResponse, StreamingResponse +from fastapi.staticfiles import StaticFiles from .base_service import BaseService from ..component_registry import R from ..job import BaseJob, StreamJob from ...constants import REME_DEFAULT_HOST, REME_DEFAULT_PORT from ...schema import Request, Response -from ...utils import execute_stream_task +from ...utils import execute_stream_task, resolve_web_static_dir if TYPE_CHECKING: from ...application import Application @@ -33,10 +35,19 @@ _WEBSOCKET_DEPRECATION_PATTERNS = ( class HttpService(BaseService): """Map non-stream jobs to JSON POST endpoints and StreamJobs to SSE endpoints.""" - def __init__(self, host: str = REME_DEFAULT_HOST, port: int = REME_DEFAULT_PORT, **kwargs): + def __init__( + self, + host: str = REME_DEFAULT_HOST, + port: int = REME_DEFAULT_PORT, + web_enabled: bool = True, + web_static_dir: str | None = None, + **kwargs, + ): super().__init__(**kwargs) self.host: str = host self.port: int = port + self.web_enabled = web_enabled + self.web_static_dir = web_static_dir # ----- BaseService contract ------------------------------------------ @@ -69,6 +80,52 @@ class HttpService(BaseService): warnings.filterwarnings("ignore", category=DeprecationWarning, message=pattern) uvicorn.run(self.service, host=self.host, port=self.port, **self.kwargs) + def finalize_service(self, app: "Application") -> None: + """Serve the optional workspace UI after all job endpoints are registered.""" + del app + if not self.web_enabled: + return + + static_dir = resolve_web_static_dir(self.web_static_dir) + if static_dir is None: + self.logger.info("Web workspace is unavailable; no static build was found") + return + + index_file = static_dir / "index.html" + assets_dir = static_dir / "assets" + if assets_dir.is_dir(): + self.service.mount( + "/assets", + StaticFiles(directory=str(assets_dir)), + name="web-assets", + ) + + no_cache_headers = {"Cache-Control": "no-cache, no-store, must-revalidate"} + post_only_paths = { + route.path + for route in self.service.routes + if "POST" in (getattr(route, "methods", None) or set()) + and "GET" not in (getattr(route, "methods", None) or set()) + } + + @self.service.get("/{full_path:path}", include_in_schema=False) + async def workspace_spa(full_path: str): + if full_path in {"docs", "redoc", "openapi.json"}: + raise HTTPException(status_code=404, detail="Not Found") + if f"/{full_path}" in post_only_paths: + raise HTTPException( + status_code=405, + detail="Method Not Allowed", + headers={"Allow": "POST"}, + ) + + if full_path and not Path(full_path).is_absolute(): + static_file = (static_dir / full_path).resolve() + if static_file.is_relative_to(static_dir) and static_file.is_file(): + return FileResponse(static_file) + + return FileResponse(index_file, headers=no_cache_headers) + # ----- Endpoint factories -------------------------------------------- def _add_json_job(self, job: BaseJob) -> None: diff --git a/reme/config/default.yaml b/reme/config/default.yaml index a5f61f5a..95a0bad5 100644 --- a/reme/config/default.yaml +++ b/reme/config/default.yaml @@ -1,5 +1,6 @@ service: backend: http + web_enabled: true jobs: index_update_loop: diff --git a/reme/utils/__init__.py b/reme/utils/__init__.py index c9b7aaf3..2f7570fd 100644 --- a/reme/utils/__init__.py +++ b/reme/utils/__init__.py @@ -15,6 +15,7 @@ from .logo_utils import print_logo from .service_utils import find_reme, locate_reme, precheck_start, cli_find_reme, running_service_config from .similarity_utils import cosine_similarity, batch_cosine_similarity from .token_utils import estimate_token_count +from .web_static import REME_WEB_STATIC_DIR, resolve_web_static_dir from .agent_state_io import AsStateHandler from .counter import ( global_counter_add, @@ -46,6 +47,8 @@ __all__ = [ "cosine_similarity", "batch_cosine_similarity", "estimate_token_count", + "REME_WEB_STATIC_DIR", + "resolve_web_static_dir", "AsStateHandler", "global_counter_add", "global_counter_add_many", diff --git a/reme/utils/web_static.py b/reme/utils/web_static.py new file mode 100644 index 00000000..97d38a3e --- /dev/null +++ b/reme/utils/web_static.py @@ -0,0 +1,30 @@ +"""Resolve the optional ReMe workspace frontend build.""" + +from __future__ import annotations + +import os +from pathlib import Path + +REME_WEB_STATIC_DIR = "REME_WEB_STATIC_DIR" + + +def resolve_web_static_dir(configured_dir: str | None = None) -> Path | None: + """Return the first directory containing a built workspace ``index.html``.""" + package_dir = Path(__file__).resolve().parent.parent + repository_dir = package_dir.parent + cwd = Path.cwd() + candidates = [ + configured_dir, + os.getenv(REME_WEB_STATIC_DIR), + package_dir / "web", + repository_dir / "website" / "dist-static", + cwd / "website" / "dist-static", + cwd / "web_dist", + ] + for candidate in candidates: + if not candidate: + continue + path = Path(candidate).expanduser().resolve() + if path.is_dir() and (path / "index.html").is_file(): + return path + return None diff --git a/tests/unit/test_http_web_workspace.py b/tests/unit/test_http_web_workspace.py new file mode 100644 index 00000000..96472a5b --- /dev/null +++ b/tests/unit/test_http_web_workspace.py @@ -0,0 +1,102 @@ +"""HTTP service coverage for the optional bundled web workspace.""" + +from pathlib import Path +from types import SimpleNamespace + +import pytest +from fastapi.testclient import TestClient + +from reme.components.service.http_service import HttpService +from reme.utils import REME_WEB_STATIC_DIR, resolve_web_static_dir + + +class _FakeApplication: + def __init__(self) -> None: + self.config = SimpleNamespace(app_name="ReMe test") + self.started = False + self.closed = False + + async def start(self) -> None: + """Record that the application lifespan started.""" + self.started = True + + async def close(self) -> None: + """Record that the application lifespan closed.""" + self.closed = True + + +def _static_build(tmp_path: Path) -> Path: + static_dir = tmp_path / "web" + assets_dir = static_dir / "assets" + assets_dir.mkdir(parents=True) + (static_dir / "index.html").write_text("
ReMe workspace
", encoding="utf-8") + (static_dir / "favicon.svg").write_text("", encoding="utf-8") + (assets_dir / "app.js").write_text("console.log('reme')", encoding="utf-8") + return static_dir + + +def test_http_service_serves_workspace_without_shadowing_jobs(tmp_path: Path) -> None: + """Serve workspace files and preserve explicitly registered job routes.""" + static_dir = _static_build(tmp_path) + app = _FakeApplication() + service = HttpService(web_static_dir=str(static_dir)) + service.build_service(app) # type: ignore[arg-type] + + @service.service.post("/status") + async def status(): + return {"success": True} + + service.finalize_service(app) # type: ignore[arg-type] + + with TestClient(service.service) as client: + assert client.get("/").text == "
ReMe workspace
" + assert client.get("/memory/topic").text == "
ReMe workspace
" + assert client.get("/favicon.svg").text == "" + assert client.get("/assets/app.js").text == "console.log('reme')" + assert client.post("/status").json() == {"success": True} + assert client.get("/status").status_code == 405 + assert client.get("/status").headers["allow"] == "POST" + assert client.get("/").headers["cache-control"] == "no-cache, no-store, must-revalidate" + + assert app.started is True + assert app.closed is True + + +def test_http_service_can_disable_workspace(tmp_path: Path) -> None: + """Leave the root route unregistered when workspace serving is disabled.""" + app = _FakeApplication() + service = HttpService(web_enabled=False, web_static_dir=str(_static_build(tmp_path))) + service.build_service(app) # type: ignore[arg-type] + service.finalize_service(app) # type: ignore[arg-type] + + with TestClient(service.service) as client: + assert client.get("/").status_code == 404 + + +def test_http_service_does_not_serve_symlinks_outside_static_dir(tmp_path: Path) -> None: + """Do not expose files reached through symlinks outside the static build.""" + static_dir = _static_build(tmp_path) + secret_file = tmp_path / "secret.txt" + secret_file.write_text("private", encoding="utf-8") + try: + (static_dir / "escape.txt").symlink_to(secret_file) + except OSError as error: + pytest.skip(f"Symbolic links are unavailable: {error}") + + app = _FakeApplication() + service = HttpService(web_static_dir=str(static_dir)) + service.build_service(app) # type: ignore[arg-type] + service.finalize_service(app) # type: ignore[arg-type] + + with TestClient(service.service) as client: + assert client.get("/escape.txt").text == "
ReMe workspace
" + + +def test_static_dir_configuration_precedes_environment(monkeypatch, tmp_path: Path) -> None: + """Prefer an explicit static directory over the environment setting.""" + configured = _static_build(tmp_path / "configured") + environment = _static_build(tmp_path / "environment") + monkeypatch.setenv(REME_WEB_STATIC_DIR, str(environment)) + + assert resolve_web_static_dir(str(configured)) == configured.resolve() + assert resolve_web_static_dir() == environment.resolve() diff --git a/website/.env.example b/website/.env.example index 70373034..56463b30 100644 --- a/website/.env.example +++ b/website/.env.example @@ -1,2 +1,4 @@ NEXT_PUBLIC_REME_API_URL=http://127.0.0.1:2333 NEXT_PUBLIC_REME_WORKSPACE_EXTENSIONS=md,txt +VITE_REME_API_URL=http://127.0.0.1:2333 +VITE_REME_WORKSPACE_EXTENSIONS=md,txt diff --git a/website/.gitignore b/website/.gitignore index 6c92d9d8..3d2f4055 100644 --- a/website/.gitignore +++ b/website/.gitignore @@ -37,6 +37,7 @@ yarn-error.log* .vercel /dist/ +/dist-static/ /.wrangler/ /outputs/ /work/ diff --git a/website/.prettierignore b/website/.prettierignore index b4a797c1..10f02850 100644 --- a/website/.prettierignore +++ b/website/.prettierignore @@ -2,6 +2,7 @@ .next/ .vinext/ dist/ +dist-static/ out/ # Dependencies diff --git a/website/README.md b/website/README.md index b9b7fbb8..90072a98 100644 --- a/website/README.md +++ b/website/README.md @@ -24,6 +24,27 @@ Open . The frontend connects to NEXT_PUBLIC_REME_API_URL=http://127.0.0.1:8000 npm run dev ``` +## ReMe-hosted static build + +ReMe can serve the same workspace from its FastAPI process. Build the static +variant and restart ReMe: + +```bash +cd website +npm ci +npm run build:static +cd .. +reme start +``` + +Open . The static build uses same-origin requests by +default. For standalone static development, run `npm run dev:static` and set +`VITE_REME_API_URL` to the running ReMe service URL when necessary. + +The regular `npm run build` command remains the vinext/Sites deployment build; +`npm run build:static` creates `dist-static/` exclusively for FastAPI and Python +package distribution. + The workspace hides dotfiles and dot-directories. It displays only Markdown and text files by default. Configure the allowed extensions as a comma-separated list in `.env.local`: @@ -38,5 +59,6 @@ Useful checks: npm run lint npx tsc --noEmit npm run build +npm run build:static npm test ``` diff --git a/website/app/api-endpoint.ts b/website/app/api-endpoint.ts new file mode 100644 index 00000000..f5d5a421 --- /dev/null +++ b/website/app/api-endpoint.ts @@ -0,0 +1,10 @@ +export function normalizeReMeApiUrl(value: string): string { + return value.replace(/\/$/, ""); +} + +export function displayReMeApiEndpoint( + apiUrl: string, + origin?: string, +): string { + return apiUrl || origin || "/"; +} diff --git a/website/app/api.ts b/website/app/api.ts index 0de232e2..1ed6f29c 100644 --- a/website/app/api.ts +++ b/website/app/api.ts @@ -12,10 +12,15 @@ import { workspaceFileListing, type WorkspaceFileListing, } from "./workspace-files"; +import { displayReMeApiEndpoint, normalizeReMeApiUrl } from "./api-endpoint"; -export const REME_API_URL = ( - process.env.NEXT_PUBLIC_REME_API_URL || "http://127.0.0.1:2333" -).replace(/\/$/, ""); +export const REME_API_URL = normalizeReMeApiUrl( + process.env.NEXT_PUBLIC_REME_API_URL || "http://127.0.0.1:2333", +); +export const REME_API_ENDPOINT = displayReMeApiEndpoint( + REME_API_URL, + typeof window === "undefined" ? undefined : window.location.origin, +); const message = (key: TranslationKey, status: number) => translate(useLanguageStore.getState().language, key, { status: String(status), diff --git a/website/app/files-workspace/FilesNavigator.tsx b/website/app/files-workspace/FilesNavigator.tsx index f018c976..b6877e82 100644 --- a/website/app/files-workspace/FilesNavigator.tsx +++ b/website/app/files-workspace/FilesNavigator.tsx @@ -22,7 +22,7 @@ import { getAppConfig, listWorkspaceFiles, readWorkspaceFile, - REME_API_URL, + REME_API_ENDPOINT, } from "../api"; import { useI18n } from "../i18n"; import { useWorkspaceStore } from "../store"; @@ -54,9 +54,9 @@ const loadWorkspace = () => Promise.all([getAppConfig(), listWorkspaceFiles([...extensions])]); const remeEndpoint = (() => { try { - return new URL(REME_API_URL).host; + return new URL(REME_API_ENDPOINT).host; } catch { - return REME_API_URL; + return REME_API_ENDPOINT; } })(); @@ -340,7 +340,7 @@ export default function FilesNavigator({ /> ))} -
+
diff --git a/website/app/settings-center.tsx b/website/app/settings-center.tsx index 2489bc9e..f7efa2b8 100644 --- a/website/app/settings-center.tsx +++ b/website/app/settings-center.tsx @@ -17,7 +17,7 @@ import { getReMeStatus, getReMeVersion, rebuildReMeIndex, - REME_API_URL, + REME_API_ENDPOINT, } from "./api"; import { useI18n } from "./i18n"; import type { AppConfig, ReMeResponse } from "./types"; @@ -357,7 +357,7 @@ export default function SettingsCenter({
{t("apiEndpoint")} - {REME_API_URL} + {REME_API_ENDPOINT}
diff --git a/website/index.html b/website/index.html new file mode 100644 index 00000000..ca774ebd --- /dev/null +++ b/website/index.html @@ -0,0 +1,31 @@ + + + + + + + + ReMe Workspace + + + +
+ + + diff --git a/website/package.json b/website/package.json index 89ff0eb1..ce1a940d 100644 --- a/website/package.json +++ b/website/package.json @@ -7,10 +7,12 @@ }, "scripts": { "dev": "WRANGLER_LOG_PATH=.wrangler/wrangler.log vinext dev --force", + "dev:static": "vite --config vite.static.config.ts", "build": "WRANGLER_LOG_PATH=.wrangler/wrangler.log vinext build", + "build:static": "vite build --config vite.static.config.ts", "start": "WRANGLER_LOG_PATH=.wrangler/wrangler.log vinext start", - "test": "npm run build && node --test tests/*.test.mjs", - "lint": "eslint . --ignore-pattern dist --ignore-pattern .next", + "test": "npm run build && npm run build:static && node --test tests/*.test.mjs", + "lint": "eslint . --ignore-pattern dist --ignore-pattern dist-static --ignore-pattern .next", "format": "tsc --noEmit && prettier --write .", "format:check": "tsc --noEmit && prettier --check .", "db:generate": "drizzle-kit generate" diff --git a/website/static/main.tsx b/website/static/main.tsx new file mode 100644 index 00000000..67071416 --- /dev/null +++ b/website/static/main.tsx @@ -0,0 +1,11 @@ +import React from "react"; +import ReactDOM from "react-dom/client"; + +import "../app/globals.css"; +import { ReMeWorkspace } from "../app/workspace"; + +ReactDOM.createRoot(document.getElementById("root")!).render( + + + , +); diff --git a/website/static/next-dynamic.tsx b/website/static/next-dynamic.tsx new file mode 100644 index 00000000..e1b204ea --- /dev/null +++ b/website/static/next-dynamic.tsx @@ -0,0 +1,26 @@ +import { + lazy, + Suspense, + type ComponentType, + type LazyExoticComponent, +} from "react"; + +type DynamicOptions = { + ssr?: boolean; +}; + +export default function dynamic( + loader: () => Promise<{ default: ComponentType }>, + options?: DynamicOptions, +): ComponentType { + void options; + const LazyComponent: LazyExoticComponent> = lazy(loader); + + return function DynamicComponent(props: Props) { + return ( + + + + ); + }; +} diff --git a/website/tests/api-endpoint.test.mjs b/website/tests/api-endpoint.test.mjs new file mode 100644 index 00000000..dcd92a0f --- /dev/null +++ b/website/tests/api-endpoint.test.mjs @@ -0,0 +1,27 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + displayReMeApiEndpoint, + normalizeReMeApiUrl, +} from "../app/api-endpoint.ts"; + +test("same-origin API requests retain a visible endpoint", () => { + const apiUrl = normalizeReMeApiUrl("/"); + + assert.equal(apiUrl, ""); + assert.equal( + displayReMeApiEndpoint(apiUrl, "http://127.0.0.1:2333"), + "http://127.0.0.1:2333", + ); + assert.equal(displayReMeApiEndpoint(apiUrl), "/"); +}); + +test("configured API endpoint is preferred over the browser origin", () => { + const apiUrl = normalizeReMeApiUrl("http://localhost:8181/"); + + assert.equal(apiUrl, "http://localhost:8181"); + assert.equal( + displayReMeApiEndpoint(apiUrl, "http://localhost:3000"), + "http://localhost:8181", + ); +}); diff --git a/website/tests/static-build.test.mjs b/website/tests/static-build.test.mjs new file mode 100644 index 00000000..d6c4d00c --- /dev/null +++ b/website/tests/static-build.test.mjs @@ -0,0 +1,37 @@ +import assert from "node:assert/strict"; +import { readFile, readdir } from "node:fs/promises"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +test("static build contains the ReMe workspace entry and assets", async () => { + const output = fileURLToPath(new URL("../dist-static/", import.meta.url)); + const html = await readFile( + new URL("../dist-static/index.html", import.meta.url), + "utf8", + ); + const assets = await readdir( + new URL("../dist-static/assets/", import.meta.url), + ); + const javascript = ( + await Promise.all( + assets + .filter((name) => name.endsWith(".js")) + .map((name) => + readFile(new URL(`../dist-static/assets/${name}`, import.meta.url)), + ), + ) + ).join("\n"); + + assert.match(html, /ReMe Workspace<\/title>/i); + assert.match(html, /<div id="root"><\/div>/i); + assert.match(html, /<script type="module"[^>]+src="\/assets\//i); + assert.ok( + assets.some((name) => name.endsWith(".js")), + output, + ); + assert.ok( + assets.some((name) => name.endsWith(".css")), + output, + ); + assert.doesNotMatch(javascript, /http:\/\/127\.0\.0\.1:2333/); +}); diff --git a/website/vite.static.config.ts b/website/vite.static.config.ts new file mode 100644 index 00000000..c8261a63 --- /dev/null +++ b/website/vite.static.config.ts @@ -0,0 +1,32 @@ +import react from "@vitejs/plugin-react"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { defineConfig, loadEnv } from "vite"; + +const staticDir = fileURLToPath(new URL("./static/", import.meta.url)); + +export default defineConfig(({ mode }) => { + const env = loadEnv(mode, process.cwd(), ""); + + return { + plugins: [react()], + resolve: { + alias: { + "next/dynamic": path.resolve(staticDir, "next-dynamic.tsx"), + }, + }, + define: { + "process.env.NEXT_PUBLIC_REME_API_URL": JSON.stringify( + env.VITE_REME_API_URL || "/", + ), + "process.env.NEXT_PUBLIC_REME_WORKSPACE_EXTENSIONS": JSON.stringify( + env.VITE_REME_WORKSPACE_EXTENSIONS ?? "", + ), + }, + build: { + outDir: "dist-static", + emptyOutDir: true, + sourcemap: mode !== "production", + }, + }; +});