From 4c54c2b650038eff2a5d0d77aaa61b3e836f1b18 Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:54:58 +0800 Subject: [PATCH] feat(docker): add container deployment and multi-platform release workflow (#582) * feat(docker): add container deployment and multi-platform release workflow * fix(llm): initialize providers on model access and preserve runtime injection * test(llm): keep rejected model updates independent of credentials --- .dockerignore | 40 ++++ .github/workflows/docker.yml | 221 ++++++++++++++++++ Dockerfile | 54 +++++ README.md | 13 ++ README_ZH.md | 12 + deploy/docker/example.env | 18 ++ deploy/docker/reme_container.py | 100 ++++++++ docker-compose.yml | 24 ++ docs/.vitepress/config.mts | 1 + docs/en/docker.md | 156 +++++++++++++ docs/zh/docker.md | 148 ++++++++++++ github-pages/tests/generated-content.test.mjs | 1 + reme/application.py | 8 +- reme/components/as_llm/__init__.py | 25 +- reme/utils/service_utils.py | 62 +++-- scripts/test_docker_image.py | 187 +++++++++++++++ tests/unit/test_as_llm_lazy.py | 100 ++++++++ tests/unit/test_auto_dream.py | 55 ++++- tests/unit/test_docker_runtime.py | 146 ++++++++++++ tests/unit/test_embedded_consumer_compat.py | 59 ++++- tests/unit/test_service_utils.py | 53 +++++ 21 files changed, 1452 insertions(+), 31 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/docker.yml create mode 100644 Dockerfile create mode 100644 deploy/docker/example.env create mode 100644 deploy/docker/reme_container.py create mode 100644 docker-compose.yml create mode 100644 docs/en/docker.md create mode 100644 docs/zh/docker.md create mode 100644 scripts/test_docker_image.py create mode 100644 tests/unit/test_as_llm_lazy.py create mode 100644 tests/unit/test_docker_runtime.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..1300569a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,40 @@ +.git +**/.DS_Store +**/.ssh +**/.codex +**/.claude +**/private* +**/.env +**/.env.* +**/.reme +**/.venv +**/venv +**/__pycache__ +**/*.py[cod] +**/.pytest_cache +**/.mypy_cache +**/.ruff_cache +**/.coverage* +**/htmlcov +**/node_modules +**/dist +**/dist-static +**/.generated +**/.next +**/.vite +**/.wrangler +**/.cache +**/*.egg-info +**/build +**/logs +**/*.log +**/*.tmp +reme_studio/src/reme_studio/static +benchmark +cookbook +docs +github-pages +integrations +plugins +skills +tests diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..f90af94f --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,221 @@ +name: CI and Release / Docker + +on: + pull_request: + branches: [main] + paths: + - '.github/workflows/docker.yml' + - 'Dockerfile' + - '.dockerignore' + - 'docker-compose.yml' + - 'deploy/docker/**' + - 'pyproject.toml' + - 'README.md' + - 'LICENSE' + - 'reme/**' + - 'reme_studio/**' + - 'scripts/package_studio.py' + - 'scripts/test_docker_image.py' + push: + branches: [main] + paths: + - '.github/workflows/docker.yml' + - 'Dockerfile' + - '.dockerignore' + - 'docker-compose.yml' + - 'deploy/docker/**' + - 'pyproject.toml' + - 'README.md' + - 'LICENSE' + - 'reme/**' + - 'reme_studio/**' + - 'scripts/package_studio.py' + - 'scripts/test_docker_image.py' + release: + types: [published] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + PUBLISH_IMAGE: ${{ github.event_name == 'release' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main') }} + +jobs: + build: + name: Build and test / ${{ matrix.arch }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.11' + + - name: Validate package version + env: + RELEASE_VERSION: ${{ github.event.release.tag_name }} + run: | + python -m pip install packaging + if [ -n "$RELEASE_VERSION" ]; then + python scripts/bump_version.py --check --expected-version "$RELEASE_VERSION" + else + python scripts/bump_version.py --check + fi + + - name: Normalize image name + id: image + env: + REPOSITORY: ${{ github.repository }} + run: echo "name=ghcr.io/${REPOSITORY,,}" >> "$GITHUB_OUTPUT" + + - name: Validate Compose + run: docker compose config --quiet + + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 + + - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6 + id: metadata + with: + images: ${{ steps.image.outputs.name }} + flavor: latest=${{ github.event_name == 'release' && !github.event.release.prerelease && 'auto' || 'false' }} + tags: | + type=raw,value=main,enable=${{ github.ref == 'refs/heads/main' }} + type=pep440,pattern={{version}},value=${{ github.event.release.tag_name }},enable=${{ github.event_name == 'release' }} + type=sha + + - name: Build local image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7 + with: + context: . + platforms: linux/${{ matrix.arch }} + load: true + tags: reme:smoke + labels: ${{ steps.metadata.outputs.labels }} + cache-from: type=gha,scope=reme-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=reme-${{ matrix.arch }} + + - name: Test installed image and persistent workspace + run: python scripts/test_docker_image.py --image reme:smoke + + - name: Log in to GHCR + if: env.PUBLISH_IMAGE == 'true' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish tested architecture by digest + id: publish + if: env.PUBLISH_IMAGE == 'true' + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7 + with: + context: . + platforms: linux/${{ matrix.arch }} + outputs: type=image,name=${{ steps.image.outputs.name }},push-by-digest=true,name-canonical=true,push=true + labels: ${{ steps.metadata.outputs.labels }} + cache-from: type=gha,scope=reme-${{ matrix.arch }} + provenance: mode=max + sbom: true + + - name: Record image digest + if: env.PUBLISH_IMAGE == 'true' + env: + IMAGE_DIGEST: ${{ steps.publish.outputs.digest }} + run: | + mkdir -p "$RUNNER_TEMP/digests" + touch "$RUNNER_TEMP/digests/${IMAGE_DIGEST#sha256:}" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: env.PUBLISH_IMAGE == 'true' + with: + name: docker-digest-${{ matrix.arch }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 + + manifest: + name: Publish multi-platform tags + needs: build + if: github.event_name == 'release' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main') + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + packages: write + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: docker-digest-* + merge-multiple: true + path: ${{ runner.temp }}/digests + + - name: Normalize image name + id: image + env: + REPOSITORY: ${{ github.repository }} + run: echo "name=ghcr.io/${REPOSITORY,,}" >> "$GITHUB_OUTPUT" + + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 + + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6 + id: metadata + with: + images: ${{ steps.image.outputs.name }} + flavor: latest=${{ github.event_name == 'release' && !github.event.release.prerelease && 'auto' || 'false' }} + tags: | + type=raw,value=main,enable=${{ github.ref == 'refs/heads/main' }} + type=pep440,pattern={{version}},value=${{ github.event.release.tag_name }},enable=${{ github.event_name == 'release' }} + type=sha + + - name: Assemble and verify tags + env: + IMAGE_NAME: ${{ steps.image.outputs.name }} + IMAGE_TAGS: ${{ steps.metadata.outputs.tags }} + run: | + set -euo pipefail + image_refs=() + for digest_file in "$RUNNER_TEMP"/digests/*; do + image_refs+=("${IMAGE_NAME}@sha256:$(basename "$digest_file")") + done + [ "${#image_refs[@]}" -eq 2 ] + tag_args=() + while IFS= read -r tag; do + tag_args+=(--tag "$tag") + done <<< "$IMAGE_TAGS" + docker buildx imagetools create "${tag_args[@]}" "${image_refs[@]}" + while IFS= read -r tag; do + manifest=$(docker buildx imagetools inspect "$tag" --raw) + IMAGE_MANIFEST="$manifest" python - <<'PY' + import json + import os + manifest = json.loads(os.environ["IMAGE_MANIFEST"]) + platforms = {(item["platform"]["os"], item["platform"]["architecture"]) for item in manifest["manifests"]} + assert {("linux", "amd64"), ("linux", "arm64")} <= platforms, platforms + PY + done <<< "$IMAGE_TAGS" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..7a5fd438 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,54 @@ +# syntax=docker/dockerfile:1 + +FROM node:22-bookworm-slim AS studio-builder +WORKDIR /build/reme_studio +COPY reme_studio/package.json reme_studio/package-lock.json ./ +RUN --mount=type=cache,target=/root/.npm npm ci +COPY reme_studio/ ./ +RUN npm run build:static && test -f dist-static/index.html + +FROM python:3.11-slim-bookworm AS python-builder +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /build +COPY pyproject.toml README.md LICENSE ./ +COPY reme/ reme/ +COPY reme_studio/pyproject.toml reme_studio/README.md reme_studio/LICENSE reme_studio/ +COPY reme_studio/src/ reme_studio/src/ +COPY --from=studio-builder /build/reme_studio/dist-static/ reme_studio/dist-static/ +COPY scripts/package_studio.py scripts/package_studio.py +RUN python scripts/package_studio.py && python -m venv /opt/venv +ENV PATH="/opt/venv/bin:${PATH}" +RUN --mount=type=cache,target=/root/.cache/pip \ + python -m pip install --upgrade pip \ + && python -m pip install ./reme_studio ".[core,image-heif]" \ + && python -m pip check +# Check installed resources away from the checkout, so source files cannot mask +# incomplete wheels or a Studio package fetched accidentally from PyPI. +WORKDIR /tmp +RUN python -I -c "import reme; from reme_studio import static_dir; from reme.config import resolve_app_config; assert (static_dir() / 'index.html').is_file(); assert resolve_app_config(log_config=False)['service']['backend'] == 'http'" + +FROM python:3.11-slim-bookworm AS runtime +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates git libgomp1 libstdc++6 tini tzdata \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --gid 1000 reme \ + && useradd --uid 1000 --gid reme --no-create-home reme \ + && mkdir -p /app /data \ + && chown reme:reme /app /data +COPY --from=python-builder /opt/venv /opt/venv +COPY deploy/docker/reme_container.py /usr/local/lib/reme_container.py +ENV PATH="/opt/venv/bin:${PATH}" \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + HOME=/tmp/reme-home \ + REME_WORKSPACE_DIR=/data \ + REME_HOST=0.0.0.0 +WORKDIR /app +USER reme +EXPOSE 2333 +HEALTHCHECK --interval=30s --timeout=5s --start-period=120s --retries=3 \ + CMD ["python", "/usr/local/lib/reme_container.py", "--healthcheck"] +ENTRYPOINT ["/usr/bin/tini", "--", "python", "/usr/local/lib/reme_container.py"] +CMD ["start"] diff --git a/README.md b/README.md index e6a1ebbd..b45ef826 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,19 @@ cd .. The static build requires Node.js 22.13 or newer and makes Studio available from the source tree. +### Docker + +With Docker and Compose 2.24.0+, build and start ReMe with the bundled Studio: + +```bash +mkdir -p .reme +docker compose up --build -d +``` + +Open . The complete workspace persists in `./.reme`. On Linux, set `REME_UID` and `REME_GID` to your +user's IDs when they differ from 1000. See [Docker deployment](https://reme.agentscope.io/en/docker) for model credentials, +custom paths, published images, and upgrades. + ### Environment Variables Configure environment variables when you want LLM-powered memory evolution or embedding retrieval. Embeddings are diff --git a/README_ZH.md b/README_ZH.md index 3c6b0468..ef1d211e 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -96,6 +96,18 @@ cd .. 静态构建要求 Node.js 22.13 或更高版本，并让源码安装可以直接使用 Studio。 +### Docker + +使用 Docker 和 Compose 2.24.0+，构建并启动包含 Studio 的 ReMe： + +```bash +mkdir -p .reme +docker compose up --build -d +``` + +打开 ，完整工作区保存在宿主机的 `./.reme`。Linux 用户的 UID/GID 不是 1000 时，请设置对应的 +`REME_UID` 和 `REME_GID`。模型凭证、自定义路径、发布镜像和升级方式见 [Docker 部署](https://reme.agentscope.io/zh/docker)。 + ### 环境变量配置 如果需要 LLM 驱动的记忆演化或 embedding 检索，请在启动服务前配置环境变量。embedding 默认关闭，因此默认配置不会启动 diff --git a/deploy/docker/example.env b/deploy/docker/example.env new file mode 100644 index 00000000..bd546d20 --- /dev/null +++ b/deploy/docker/example.env @@ -0,0 +1,18 @@ +# Copy to the repository's .env for Docker Compose; keep real credentials private. +# File operations and BM25 search work without model credentials. +LLM_API_KEY= +LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 +LLM_MODEL_NAME=qwen3.7-plus + +# Host settings: create this directory before starting Compose. +REME_DATA_DIR=./.reme +REME_PUBLISHED_PORT=2333 + +# On Linux, set these to the outputs of `id -u` and `id -g` so files remain yours. +REME_UID=1000 +REME_GID=1000 + +# Optional container settings. +# REME_TIMEZONE=Asia/Shanghai +# REME_CONFIG=/etc/reme/config.yaml +# REME_IMAGE=ghcr.io/agentscope-ai/reme:main diff --git a/deploy/docker/reme_container.py b/deploy/docker/reme_container.py new file mode 100644 index 00000000..0b1e1b55 --- /dev/null +++ b/deploy/docker/reme_container.py @@ -0,0 +1,100 @@ +"""Container startup and health checks using ReMe's existing CLI contract.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import sys +from urllib.request import ProxyHandler, Request, build_opener + +HEALTH_STATE_PATH = Path("/tmp/reme-health.json") +_ENV_OVERRIDES = { + "REME_CONFIG": "config", + "REME_WORKSPACE_DIR": "workspace_dir", + "REME_HOST": "service.host", + "REME_PORT": "service.port", + "REME_TIMEZONE": "timezone", +} + + +def start_command(arguments: list[str], environment: dict[str, str]) -> list[str]: + """Add container defaults only to `start`; explicit CLI arguments win.""" + from reme.config import deep_merge_config, parse_kwargs + + defaults = parse_kwargs( + "log_to_file=false", + *[f"{key}={json.dumps(environment[name])}" for name, key in _ENV_OVERRIDES.items() if environment.get(name)], + ) + # Docker environment values are strings; the HTTP service expects an integer port. + if environment.get("REME_PORT"): + defaults["service"]["port"] = int(environment["REME_PORT"]) + overrides = deep_merge_config(defaults, parse_kwargs(*arguments)) + return ["reme", "start", *[f"{key}={json.dumps(value, ensure_ascii=False)}" for key, value in overrides.items()]] + + +def write_health_state(command: list[str], state_path: Path = HEALTH_STATE_PATH) -> None: + """Record only the effective HTTP address, never credentials or user data.""" + from reme.components.service.cli_service import prepare_start_config + from reme.config import parse_kwargs + from reme.constants import REME_DEFAULT_HOST, REME_DEFAULT_PORT, normalize_connect_host + from reme.plugin import resolve_plugin_runtime + + config = resolve_plugin_runtime(prepare_start_config(parse_kwargs(*command[2:]))).config + service = config.get("service") or {} + if service.get("backend") != "http": + return + host = normalize_connect_host(service.get("host") or REME_DEFAULT_HOST) + if host == "::": + host = "::1" + port = int(service.get("port", REME_DEFAULT_PORT)) + if not 1 <= port <= 65535: + raise ValueError("service.port must be between 1 and 65535") + host = f"[{host}]" if ":" in host else host + with state_path.open("w", encoding="utf-8") as state_file: + os.chmod(state_path, 0o600) + json.dump({"url": f"http://{host}:{port}/health_check"}, state_file) + + +def healthcheck(state_path: Path = HEALTH_STATE_PATH) -> int: + """Require both a successful Job and a healthy component snapshot.""" + try: + state = json.loads(state_path.read_text(encoding="utf-8")) + request = Request(state["url"], data=b"{}", headers={"Content-Type": "application/json"}, method="POST") + # A deployment's outbound proxy must not intercept its local probe. + with build_opener(ProxyHandler({})).open(request, timeout=4) as response: + payload = json.load(response) + healthy = payload.get("metadata", {}).get("health", {}).get("healthy") + if payload.get("success") is True and healthy is True: + return 0 + except (OSError, ValueError, KeyError, TypeError, AttributeError): + pass + print("ReMe HTTP health check failed; inspect the container logs and POST /health_check", file=sys.stderr) + return 1 + + +def main() -> int: + """Prepare startup, then replace this process so signals reach ReMe.""" + arguments = sys.argv[1:] + if arguments == ["--healthcheck"]: + return healthcheck() + + HEALTH_STATE_PATH.unlink(missing_ok=True) + Path(os.environ.get("HOME", "/tmp/reme-home")).mkdir(parents=True, exist_ok=True) + if not arguments: + arguments = ["start"] + start_actions = {"start", "-start", "--start"} + if len(arguments) >= 2 and arguments[0] == "reme" and arguments[1] in start_actions: + arguments = arguments[1:] + if arguments[0] in start_actions: + from reme.utils import load_env + + load_env() + arguments = start_command(arguments[1:], dict(os.environ)) + write_health_state(arguments) + os.execvp(arguments[0], arguments) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..764c6a7e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,24 @@ +services: + reme: + image: ${REME_IMAGE:-reme:local} + build: + context: . + init: false # The image already runs tini. + user: "${REME_UID:-1000}:${REME_GID:-1000}" + ports: + - "${REME_BIND_ADDRESS:-127.0.0.1}:${REME_PUBLISHED_PORT:-2333}:${REME_PORT:-2333}" + volumes: + - type: bind + source: ${REME_DATA_DIR:-./.reme} + target: /data + bind: + create_host_path: false + env_file: + - path: ${REME_ENV_FILE:-.env} + required: false + environment: + REME_HOST: 0.0.0.0 + REME_PORT: ${REME_PORT:-2333} + REME_WORKSPACE_DIR: /data + restart: unless-stopped + stop_grace_period: 60s diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index afe0b1f9..32dd70d4 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -126,6 +126,7 @@ function docsSidebar(language: "zh" | "en"): DefaultTheme.SidebarItem[] { { text: zh ? "快速开始" : "Quick Start", link: `/${language}/quick_start` }, { text: zh ? "基础配置" : "Configuration", link: `/${language}/configuration` }, { text: zh ? "服务与部署" : "Services and Deployment", link: `/${language}/services` }, + { text: zh ? "Docker 部署" : "Docker Deployment", link: `/${language}/docker` }, { text: "ReMe Studio", link: `/${language}/workspace/studio` }, ], }, diff --git a/docs/en/docker.md b/docs/en/docker.md new file mode 100644 index 00000000..08518ce2 --- /dev/null +++ b/docs/en/docker.md @@ -0,0 +1,156 @@ +--- +title: Docker Deployment +description: Run ReMe and Studio in Docker with a user-owned persistent workspace. +--- + +# Docker Deployment + +The image includes ReMe's `core` and HEIF image dependencies and the Studio static frontend. One HTTP process serves the API, +Studio at `/`, and MCP at `/mcp`. The default configuration keeps embeddings disabled; file operations and BM25 search do +not require model credentials. + +## Build and run with Compose + +Use Docker Engine or Docker Desktop with Compose **2.24.0 or newer**. From the repository root: + +```bash +mkdir -p .reme +docker compose up --build -d +docker compose logs -f reme +``` + +Open . Compose binds the host port to loopback and mounts `./.reme` at `/data`. The source checkout and +Studio assets are not mounted over the installed application. + +On Linux, if your workspace is not owned by UID/GID 1000, set the process identity before starting: + +```bash +export REME_UID=$(id -u) +export REME_GID=$(id -g) +docker compose up --build -d +``` + +For model-powered memory evolution, copy `deploy/docker/example.env` to `.env` if you do not already have one, then fill in +your model credentials. Compose injects this optional file at runtime; Docker builds exclude `.env` files. Compose's +`--env-file` controls variable interpolation; `REME_ENV_FILE` selects the file injected into the container. + +## Use a published image + +The Docker workflow publishes `ghcr.io/agentscope-ai/reme:main` after successful main-branch checks. Stable GitHub releases +publish their package version and `latest`; prereleases do not update `latest`. Both Linux amd64 and arm64 images are tested +before their combined tags are published. Publication starts when the workflow is enabled in the repository. + +```bash +mkdir -p "$HOME/.reme" +docker run -d --name reme \ + --user "$(id -u):$(id -g)" \ + -p 127.0.0.1:2333:2333 \ + --mount "type=bind,source=$HOME/.reme,target=/data" \ + --restart unless-stopped \ + ghcr.io/agentscope-ai/reme:main +``` + +Add `--env-file /path/to/model.env` before the image name when using model credentials. For reproducible deployments, +replace `main` with a released version or image digest. To use the image with Compose, set `REME_IMAGE`, then run +`docker compose pull` and `docker compose up -d --no-build`. + +## Paths, configuration, and ports + +The image runs as UID/GID 1000 by default. `/data` contains the entire workspace: source sessions, resources, daily notes, +digest notes, and rebuildable metadata. Create the host directory yourself and make it writable by the configured user. +Mounting only `metadata/` does not preserve the source memories. Paths in a custom configuration refer to the container's +filesystem; additional paths need additional mounts. + +| Setting | Meaning | +|---|---| +| `REME_WORKSPACE_DIR` | Container workspace; image default `/data`, fixed to `/data` by Compose | +| `REME_CONFIG` | Existing config name or mounted YAML/JSON path; unset uses the built-in default | +| `REME_HOST` | HTTP bind address; image and Compose use `0.0.0.0` | +| `REME_PORT` | Container port override; Compose defaults to `2333` | +| `REME_TIMEZONE` | Optional application timezone override; otherwise the application default applies | +| `REME_DATA_DIR` | Compose host workspace directory; default `./.reme` | +| `REME_PUBLISHED_PORT` | Compose host port; default `2333`, independent of the container port | +| `REME_BIND_ADDRESS` | Compose host bind address; default `127.0.0.1` | +| `REME_UID`, `REME_GID` | Compose process identity; both default to `1000` | +| `REME_ENV_FILE` | Optional Compose runtime environment file; default `.env` | + +Explicit `start key=value` arguments override container environment settings, which override the loaded configuration for +those keys. Other keys retain ReMe's normal deep merge behavior. File logging defaults to off in the image; use container +logs. `log_to_file=true` explicitly enables file logs under `/app/logs`, which requires a separate mount to persist them. +The temporary home directory `/tmp/reme-home` and probe address file are disposable, not workspace storage. + +To customize the full job/component configuration, copy `reme/config/default.yaml` to `reme.yaml`, edit it, set +`REME_CONFIG=/etc/reme/config.yaml` in `.env`, and add `compose.override.yaml`: + +```yaml +services: + reme: + volumes: + - ./reme.yaml:/etc/reme/config.yaml:ro +``` + +Keep the `health_check` Job enabled and in `service.jobs` if you use an allowlist. The image's probe uses HTTP; when +overriding the service to CLI or MCP stdio, disable the Docker health check with `--no-healthcheck` (or Compose +`healthcheck: {disable: true}`). + +To override startup without changing the image: + +```bash +docker run --rm -p 127.0.0.1:2444:2444 \ + --mount "type=bind,source=$HOME/.reme,target=/data" \ + reme:local start service.port=2444 timezone=UTC +docker compose exec reme reme health_check +docker compose exec reme reme status +``` + +Other commands pass through unchanged, including `reme start job=version` for a one-shot job or `python` for diagnostics. +From a host CLI, supply the published address explicitly, for example `reme health_check host=127.0.0.1 port=2444`. +Host process discovery cannot reconstruct a container's startup arguments. Configure agent integrations to use the +published HTTP or MCP endpoint instead of starting a second native ReMe on the same workspace. + +## Networking and optional tools + +`127.0.0.1` inside a container refers to that container. A model server on the host needs a reachable host address, such as +`host.docker.internal` on Docker Desktop. On Linux, add `extra_hosts: ["host.docker.internal:host-gateway"]` to the service +and configure the model URL accordingly. Another Compose service is reachable by its service name. + +The HTTP action API has no built-in authentication and includes write/delete operations. Keep the default loopback port +publication. For access from another machine, place an authenticated TLS proxy in front of the service and restrict direct +access to its port; the same restriction must cover Studio, HTTP Jobs, and MCP. + +The image includes the configured agent SDK dependencies, but host OAuth files, transcripts, plugins, external MCP +executables, and host workspace paths are not automatically available. Mount required data explicitly and install extra +plugins/tools in a derived image so that recreating the container preserves the installation. Keep credentials out of +Docker build arguments and layers. Optional FAISS/zvec backends also depend on the capabilities of the target machine. + +## Health, upgrade, and recovery + +Docker posts to the existing `/health_check` Job and requires both `success=true` and `metadata.health.healthy=true`. +The probe address follows effective configuration and CLI port overrides, and bypasses outbound proxy settings. +Initialization has a 120-second health grace period; larger workspaces may need a longer Compose `healthcheck.start_period`. +This reports component health, not whether a remote model will accept a future request. An unhealthy Docker status alone +does not trigger `restart: unless-stopped`; that policy restarts exited processes. + +Before upgrading, stop writes and back up the **complete** host workspace and your deployment configuration. Then: + +```bash +docker compose stop +# Back up the configured host workspace here. +docker compose pull +docker compose up -d --no-build +docker compose exec reme reme health_check +``` + +For a locally built deployment, replace `pull` and `up --no-build` with `docker compose up --build -d`. Compose allows +60 seconds for orderly shutdown. Recreating containers leaves the bind-mounted workspace intact; use one ReMe writer +process per workspace. See [backup and recovery](./operations.md) for restoring derived state without deleting memory. + +For container validation after a local build: + +```bash +docker build -t reme:local . +python scripts/test_docker_image.py --image reme:local +``` + +The smoke check uses a disposable workspace and no model credentials. It verifies Studio, HTTP and MCP, file containment, +non-root execution, graceful shutdown, and memory search after replacing a container on a different port. diff --git a/docs/zh/docker.md b/docs/zh/docker.md new file mode 100644 index 00000000..ff47bdf8 --- /dev/null +++ b/docs/zh/docker.md @@ -0,0 +1,148 @@ +--- +title: Docker 部署 +description: 使用 Docker 运行 ReMe 和 Studio，并将持久记忆保存在用户拥有的工作区中。 +--- + +# Docker 部署 + +镜像包含 ReMe 的 `core`、HEIF 图片依赖和 Studio 静态前端。一个 HTTP 进程同时提供 API、根路径 `/` 上的 Studio 和 +`/mcp` 上的 MCP。默认配置关闭 embedding；基础文件操作和 BM25 检索不需要模型凭证。 + +## 使用 Compose 构建并启动 + +需要 Docker Engine 或 Docker Desktop，以及 **2.24.0 或更新版本**的 Compose。在仓库根目录运行： + +```bash +mkdir -p .reme +docker compose up --build -d +docker compose logs -f reme +``` + +打开 。Compose 默认只开放宿主机回环地址，并将 `./.reme` 挂载到容器的 `/data`。 +源码目录和 Studio 资源不会覆盖镜像内已经安装的应用。 + +在 Linux 上，如果工作区所有者的 UID/GID 不是 1000，请在启动前设置运行用户： + +```bash +export REME_UID=$(id -u) +export REME_GID=$(id -g) +docker compose up --build -d +``` + +需要模型驱动的记忆演化时，如果尚无 `.env`，将 `deploy/docker/example.env` 复制为 `.env`，再填写模型凭证。 +Compose 在运行时注入这个可选文件；Docker 构建排除 `.env` 文件。Compose 的 `--env-file` 用于配置变量替换， +`REME_ENV_FILE` 则选择实际注入容器的环境文件。 + +## 使用发布镜像 + +Docker 工作流在 main 分支检查成功后发布 `ghcr.io/agentscope-ai/reme:main`。稳定 GitHub Release 发布对应的包版本和 +`latest`；预发布版本不会更新 `latest`。Linux amd64 和 arm64 镜像分别通过验证后才发布共同的标签。 +镜像发布从仓库启用该工作流后开始生效。 + +```bash +mkdir -p "$HOME/.reme" +docker run -d --name reme \ + --user "$(id -u):$(id -g)" \ + -p 127.0.0.1:2333:2333 \ + --mount "type=bind,source=$HOME/.reme,target=/data" \ + --restart unless-stopped \ + ghcr.io/agentscope-ai/reme:main +``` + +需要模型凭证时，在镜像名之前添加 `--env-file /path/to/model.env`。需要固定部署版本时，将 `main` 替换为发布版本或 +镜像 digest。使用 Compose 拉取镜像时，设置 `REME_IMAGE`，再运行 `docker compose pull` 和 +`docker compose up -d --no-build`。 + +## 路径、配置和端口 + +镜像默认以 UID/GID 1000 运行。`/data` 保存完整工作区，包括源对话、资料、daily、digest 和可重建的 metadata。 +需要自己创建宿主机目录，并确保运行用户具有写权限。仅挂载 `metadata/` 不能保存源记忆。 +自定义配置中的路径指向容器文件系统；额外路径需要额外挂载。 + +| 配置项 | 含义 | +|---|---| +| `REME_WORKSPACE_DIR` | 容器工作区；镜像默认 `/data`，Compose 固定为 `/data` | +| `REME_CONFIG` | 已有配置名称或挂载的 YAML/JSON 路径；未设置时使用内置默认配置 | +| `REME_HOST` | HTTP 监听地址；镜像和 Compose 使用 `0.0.0.0` | +| `REME_PORT` | 容器端口覆盖；Compose 默认 `2333` | +| `REME_TIMEZONE` | 可选的应用时区覆盖；未设置时沿用应用默认值 | +| `REME_DATA_DIR` | Compose 的宿主机工作区目录；默认 `./.reme` | +| `REME_PUBLISHED_PORT` | Compose 的宿主机端口；默认 `2333`，与容器端口独立 | +| `REME_BIND_ADDRESS` | Compose 的宿主机监听地址；默认 `127.0.0.1` | +| `REME_UID`、`REME_GID` | Compose 的运行用户；均默认 `1000` | +| `REME_ENV_FILE` | 可选的 Compose 运行时环境文件；默认 `.env` | + +显式的 `start key=value` 参数优先于容器环境配置；容器环境配置再覆盖文件中的对应键。其他键继续采用 ReMe 的正常 +深合并规则。镜像默认关闭文件日志，使用容器日志查看运行情况。显式传入 `log_to_file=true` 可启用 `/app/logs` 下的文件 +日志，需要另外挂载才能持久保存。临时 home `/tmp/reme-home` 和探针地址文件均为可丢弃数据，不承担工作区存储职责。 + +需要自定义完整的 Job/Component 配置时，将 `reme/config/default.yaml` 复制为 `reme.yaml` 并修改，在 `.env` 中设置 +`REME_CONFIG=/etc/reme/config.yaml`，再添加 `compose.override.yaml`： + +```yaml +services: + reme: + volumes: + - ./reme.yaml:/etc/reme/config.yaml:ro +``` + +保留启用的 `health_check` Job；如果配置 `service.jobs` 白名单，也要包含它。镜像探针使用 HTTP；将服务覆盖为 CLI 或 +MCP stdio 时，请通过 `--no-healthcheck` 关闭 Docker 探针，Compose 中则使用 `healthcheck: {disable: true}`。 + +无需修改镜像即可覆盖启动配置： + +```bash +docker run --rm -p 127.0.0.1:2444:2444 \ + --mount "type=bind,source=$HOME/.reme,target=/data" \ + reme:local start service.port=2444 timezone=UTC +docker compose exec reme reme health_check +docker compose exec reme reme status +``` + +其他命令直接执行，例如 `reme start job=version` 可运行一次性 Job，`python` 可用于诊断。 +宿主机 CLI 应明确指定发布地址，例如 `reme health_check host=127.0.0.1 port=2444`；宿主机的进程发现无法还原容器内 +启动参数。Agent 集成应连接发布的 HTTP 或 MCP 地址，避免对同一工作区再启动第二个本机 ReMe。 + +## 网络和可选工具 + +容器内的 `127.0.0.1` 指向容器自身。宿主机上的模型服务需要使用可访问的宿主机地址，例如 Docker Desktop 的 +`host.docker.internal`。在 Linux 上，可为服务添加 `extra_hosts: ["host.docker.internal:host-gateway"]`，再配置模型 URL。 +另一个 Compose 服务可以通过其服务名访问。 + +HTTP action API 没有内置认证，并包含写入和删除操作。保留默认的回环地址端口映射。需要跨主机访问时，在服务前配置 +带认证的 TLS 代理，并限制服务端口的直接访问；Studio、HTTP Job 和 MCP 都应受到同样的访问限制。 + +镜像包含配置使用的 Agent SDK 依赖，但不会自动获取宿主机 OAuth 文件、对话记录、插件、外部 MCP 可执行程序或宿主机 +工作区路径。需要的数据应显式挂载，额外插件和工具应安装在派生镜像中，使重建容器后仍能保留安装内容。 +不要把凭证写入 Docker 构建参数或镜像层。可选的 FAISS/zvec 后端还依赖目标机器的能力。 + +## 健康检查、升级和恢复 + +Docker 调用已有的 `POST /health_check`，同时要求 `success=true` 和 `metadata.health.healthy=true`。 +探针地址跟随实际配置和 CLI 端口覆盖，并绕过出站代理设置。初始化有 120 秒健康检查宽限期；大型工作区可在 Compose +中延长 `healthcheck.start_period`。该检查报告组件状态，不保证远端模型会接受后续请求。Docker 的 unhealthy 状态本身 +不会触发 `restart: unless-stopped`；该策略只重启已经退出的进程。 + +升级前停止写入，并备份**完整**宿主机工作区和部署配置，再执行： + +```bash +docker compose stop +# 在这里备份已配置的宿主机工作区。 +docker compose pull +docker compose up -d --no-build +docker compose exec reme reme health_check +``` + +本地构建部署时，将 `pull` 和 `up --no-build` 替换为 `docker compose up --build -d`。Compose 为正常关闭留出 60 秒。 +重建容器会保留绑定挂载的工作区；一个工作区应只运行一个 ReMe 写入进程。派生状态恢复流程见 +[备份和恢复](./operations.md)，不要为修复索引删除记忆文件。 + +本地构建后可以验证容器： + +```bash +docker build -t reme:local . +python scripts/test_docker_image.py --image reme:local +``` + +验证脚本使用一次性工作区，不传入模型凭证；检查 Studio、HTTP/MCP、路径隔离、非 root 执行、正常关闭，以及更换容器和 +端口之后的记忆检索。 diff --git a/github-pages/tests/generated-content.test.mjs b/github-pages/tests/generated-content.test.mjs index b84b50f3..29297c4c 100644 --- a/github-pages/tests/generated-content.test.mjs +++ b/github-pages/tests/generated-content.test.mjs @@ -15,6 +15,7 @@ test("generates every required bilingual guide", async () => { "overview.md", "configuration.md", "services.md", + "docker.md", "operations.md", "integrations.md", "plugin_development.md", diff --git a/reme/application.py b/reme/application.py index d8ff59cc..5bbe3d14 100644 --- a/reme/application.py +++ b/reme/application.py @@ -4,6 +4,7 @@ import asyncio import heapq from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor +from inspect import getattr_static from pathlib import Path from typing import Any, AsyncGenerator, TypeVar @@ -18,6 +19,7 @@ from .utils import execute_stream_task, print_logo, get_logger T = TypeVar("T", bound=BaseComponent) _NodeKey = tuple[str, str] +_UNSET = object() class Application(BaseComponent): @@ -244,8 +246,9 @@ class Application(BaseComponent): raise KeyError(f"Component '{name}' not found in {component_type}") component = group[name] + # Validate fields without evaluating lazy properties before injection. for key in kwargs: - if not hasattr(component, key): + if getattr_static(component, key, _UNSET) is _UNSET and not hasattr(component, key): raise AttributeError(f"Component {component_type}:{name} has no attribute '{key}'") for key, value in kwargs.items(): setattr(component, key, value) @@ -314,8 +317,9 @@ class Application(BaseComponent): expected_type=BaseComponent, name=name, ) + # Validate fields without evaluating lazy properties before injection. for key, value in (runtime_updates or {}).items(): - if not hasattr(replacement, key): + if getattr_static(replacement, key, _UNSET) is _UNSET and not hasattr(replacement, key): raise AttributeError( f"Replacement {component_type}:{name} has no attribute '{key}'", ) diff --git a/reme/components/as_llm/__init__.py b/reme/components/as_llm/__init__.py index d4c5007d..7ea4e1bf 100644 --- a/reme/components/as_llm/__init__.py +++ b/reme/components/as_llm/__init__.py @@ -21,7 +21,8 @@ from ...enumeration import ComponentEnum class BaseAsLLM(BaseComponent): """Base wrapper for AgentScope chat models. - Subclasses set ``credential_cls`` and inherit ``_start`` / ``_close``. + Subclasses set ``credential_cls``. Providers are constructed on first use, + allowing local file and search jobs to run without model credentials. """ component_type = ComponentEnum.AS_LLM @@ -29,17 +30,33 @@ class BaseAsLLM(BaseComponent): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - self.model: ChatModelBase | None = None + self._model: ChatModelBase | None = None async def _start(self) -> None: - if self.model is not None: + """Keep service startup independent of provider credentials.""" + + @property + def model(self) -> ChatModelBase: + """Initialize on first access, including direct Step dependency resolution.""" + self.initialize_model() + assert self._model is not None + return self._model + + @model.setter + def model(self, value: ChatModelBase | None) -> None: + """Preserve explicit model injection used by standalone consumers.""" + self._model = value + + def initialize_model(self) -> None: + """Construct the configured provider once, without making a remote request.""" + if self._model is not None: return kwargs = dict(self.kwargs) credential = self.credential_cls(**kwargs.pop("credential", {})) model_cls = credential.get_chat_model_class() params_dict = kwargs.pop("parameters", None) parameters = model_cls.Parameters(**params_dict) if params_dict else None - self.model = model_cls(credential=credential, parameters=parameters, **kwargs) + self._model = model_cls(credential=credential, parameters=parameters, **kwargs) @R.register("openai") diff --git a/reme/utils/service_utils.py b/reme/utils/service_utils.py index 457b276d..9e79187d 100644 --- a/reme/utils/service_utils.py +++ b/reme/utils/service_utils.py @@ -51,24 +51,61 @@ def _pid_on_port(port: int) -> int | None: return None +def _start_arguments(cmdline: list[str]) -> list[str] | None: + """Identify a ReMe CLI process, excluding supervisors containing child argv.""" + if not cmdline: + return None + + def command_name(value: str) -> str: + return value.replace("\\", "/").rsplit("/", 1)[-1].lower() + + executable = command_name(cmdline[0]) + if executable in {"reme", "reme.exe"}: + action_index = 1 + elif executable.startswith(("python", "pypy")): + action_index = next( + ( + index + for index in range(2, len(cmdline)) + if command_name(cmdline[index - 1]) in {"reme", "reme.exe", "reme.py", "reme.reme"} + ), + len(cmdline), + ) + if "-c" in cmdline[:action_index]: + return None + else: + return None + if action_index >= len(cmdline) or cmdline[action_index] not in {"start", "-start", "--start"}: + return None + return [argument for argument in cmdline[action_index + 1 :] if "=" in argument] + + def _scan_reme_procs() -> list[tuple[int, str, int]]: """List running 'reme ... start' processes as (pid, host, port).""" + from ..config import parse_kwargs, resolve_app_config + procs: list[tuple[int, str, int]] = [] for proc in psutil.process_iter(["pid", "cmdline"]): try: cmdline = proc.info["cmdline"] or [] except (psutil.NoSuchProcess, psutil.AccessDenied): continue - # Match a `reme ... start` invocation (mirrors the old `pgrep -af`). - if "start" not in cmdline or not any("reme" in tok for tok in cmdline): + arguments = _start_arguments(cmdline) + if arguments is None: continue - host, port = REME_DEFAULT_HOST, REME_DEFAULT_PORT - for raw_arg in cmdline: - t = raw_arg.lstrip("-") - if t.startswith("service.host="): - host = t.split("=", 1)[1] - elif t.startswith("service.port=") and t.split("=", 1)[1].isdigit(): - port = int(t.split("=", 1)[1]) + try: + overrides = parse_kwargs(*arguments) + except ValueError: + continue + try: + config = resolve_app_config(log_config=False, **overrides) + except (OSError, ValueError): + # A different process's relative config file may be unavailable. + config = overrides + service = config.get("service") or {} + host = service.get("host") or REME_DEFAULT_HOST + port_value = str(service.get("port", REME_DEFAULT_PORT)) + port = int(port_value) if port_value.isdigit() else REME_DEFAULT_PORT procs.append((proc.info["pid"], host, port)) return procs @@ -86,10 +123,9 @@ def _reme_start_argv() -> list[list[str]]: cmdline = proc.info["cmdline"] or [] except (psutil.NoSuchProcess, psutil.AccessDenied): continue - if "start" not in cmdline or not any("reme" in tok for tok in cmdline): - continue - start_idx = cmdline.index("start") - argvs.append([t for t in cmdline[start_idx + 1 :] if "=" in t]) + arguments = _start_arguments(cmdline) + if arguments is not None: + argvs.append(arguments) return argvs diff --git a/scripts/test_docker_image.py b/scripts/test_docker_image.py new file mode 100644 index 00000000..fe34fc25 --- /dev/null +++ b/scripts/test_docker_image.py @@ -0,0 +1,187 @@ +"""Exercise a built Docker image with isolated memory and no model credentials.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import subprocess +import tempfile +import time +from urllib.request import ProxyHandler, Request, build_opener +import uuid + +_OFFLINE_JOB_OVERRIDES = [ + f"jobs.{name}.{key}={value}" + for name in ("dream_cron", "proactive_refresh_cron", "optimize_index_cron") + for key, value in (("backend", "base"), ("enable_serve", "false")) +] + + +def docker(*arguments: str) -> str: + """Run Docker without interpolating shell text.""" + return subprocess.check_output(["docker", *arguments], text=True).strip() + + +def request(base_url: str, path: str, payload: dict | None = None, headers: dict | None = None) -> tuple[object, dict]: + """Read JSON, MCP's SSE response, or a static resource from the container.""" + request_headers = {"Content-Type": "application/json", **(headers or {})} + body = json.dumps(payload).encode() if payload is not None else None + req = Request(base_url + path, data=body, headers=request_headers) + with build_opener(ProxyHandler({})).open(req, timeout=10) as response: + text = response.read().decode() + response_headers = dict(response.headers) + if "text/event-stream" in response.headers.get("Content-Type", ""): + messages = [json.loads(line[5:].strip()) for line in text.splitlines() if line.startswith("data:")] + return messages[-1], response_headers + try: + return json.loads(text), response_headers + except ValueError: + return text, response_headers + + +def wait_ready(container: str, base_url: str, port: int) -> None: + """Wait for initialization, checking the same health command Docker runs.""" + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + try: + payload, _ = request(base_url, "/health_check", {}) + if payload["success"] and payload["metadata"]["health"]["healthy"]: + docker("exec", container, "python", "/usr/local/lib/reme_container.py", "--healthcheck") + assert " - healthy" in docker("exec", container, "reme", "health_check") + assert f"PORT={port} " in docker("exec", container, "reme", "find_reme") + return + except (OSError, ValueError, KeyError, TypeError): + pass + if docker("inspect", "--format", "{{.State.Running}}", container) != "true": + raise RuntimeError("ReMe exited before it became healthy") + time.sleep(1) + raise TimeoutError("ReMe did not become healthy within 180 seconds") + + +def start_container(image: str, container: str, data: Path, port: int = 2432, cli_override: bool = False) -> str: + """Publish a random loopback port and mount only the disposable workspace.""" + docker( + "run", + "--detach", + "--name", + container, + "--user", + f"{os.getuid()}:{os.getgid()}", + "--publish", + f"127.0.0.1::{port}", + "--mount", + f"type=bind,source={data},target=/data", + "--env", + "LLM_API_KEY=", + "--env", + f"REME_PORT={2333 if cli_override else port}", + image, + "start", + *_OFFLINE_JOB_OVERRIDES, + *([f"service.port={port}"] if cli_override else []), + ) + mappings = json.loads(docker("inspect", "--format", "{{json .NetworkSettings.Ports}}", container)) + host_port = mappings[f"{port}/tcp"][0]["HostPort"] + base_url = f"http://127.0.0.1:{host_port}" + wait_ready(container, base_url, port) + return base_url + + +def stop_container(container: str) -> None: + """Require SIGTERM to close the application instead of timing out into SIGKILL.""" + docker("stop", "--time", "60", container) + exit_code = docker("inspect", "--format", "{{.State.ExitCode}}", container) + # Uvicorn re-raises SIGTERM after completing its lifespan shutdown. + assert exit_code in {"0", "143"}, f"Container did not shut down cleanly: {exit_code}" + logs = subprocess.check_output(["docker", "logs", container], text=True, stderr=subprocess.STDOUT) + assert "Application shutdown complete" in logs, logs + docker("rm", container) + + +def check_mcp(base_url: str) -> None: + """Initialize streamable HTTP MCP and verify its tools remain available.""" + headers = {"Accept": "application/json, text/event-stream"} + result, response_headers = request( + base_url, + "/mcp", + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": {"name": "smoke", "version": "1"}, + }, + }, + headers, + ) + headers["MCP-Protocol-Version"] = result["result"]["protocolVersion"] + session_id = next((value for key, value in response_headers.items() if key.lower() == "mcp-session-id"), None) + if session_id: + headers["Mcp-Session-Id"] = session_id + request(base_url, "/mcp", {"jsonrpc": "2.0", "method": "notifications/initialized"}, headers) + tools, _ = request(base_url, "/mcp", {"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}, headers) + names = {tool["name"] for tool in tools["result"]["tools"]} + assert {"version", "search", "health_check", "read"} <= names + + +def run_smoke(image: str) -> None: + """Verify packaged assets, file safety, indexing, restart recovery, and signals.""" + docker("run", "--rm", image, "python", "-c", "import os, faiss, zvec; assert os.getuid() != 0") + # Keep scheduled model jobs inactive even if a check crosses a cron boundary. + version = docker("run", "--rm", image, "reme", "start", "job=version", *_OFFLINE_JOB_OVERRIDES) + with tempfile.TemporaryDirectory(prefix="reme-docker-smoke-") as directory: + data = Path(directory) / "data" + data.mkdir() + container = f"reme-smoke-{uuid.uuid4().hex[:12]}" + try: + base_url = start_container(image, container, data) + page, _ = request(base_url, "/") + assert isinstance(page, str) and '
' in page + result, _ = request(base_url, "/version", {}) + assert result["success"] and result["answer"] == version + check_mcp(base_url) + note = "# Docker persistence\n\nDurable quokka memory survives container replacement.\n" + result, _ = request(base_url, "/write", {"path": "daily/2026-09-30/smoke.md", "content": note}) + assert result["success"], result + result, _ = request(base_url, "/write", {"path": "../escape.md", "content": "blocked"}) + assert not result["success"], result + note_file = data / "daily/2026-09-30/smoke.md" + assert note_file.read_text() == note + assert note_file.stat().st_uid == os.getuid() + stop_container(container) + # Recreate with a different CLI port to catch probes using stale ENV defaults. + base_url = start_container(image, container, data, port=2433, cli_override=True) + result, _ = request(base_url, "/read", {"path": "daily/2026-09-30/smoke.md"}) + assert result["success"] and "Durable quokka" in result["answer"] + deadline = time.monotonic() + 30 + while True: + result, _ = request(base_url, "/search", {"query": "quokka"}) + assert result["success"], result + if "smoke.md" in json.dumps(result): + break + if time.monotonic() >= deadline: + raise AssertionError(f"Persisted memory was not indexed: {result}") + time.sleep(1) + stop_container(container) + except Exception: + subprocess.run(["docker", "logs", container], check=False) + raise + finally: + subprocess.run(["docker", "rm", "--force", container], check=False, capture_output=True) + print(f"Docker smoke checks passed: {image}") + + +def main() -> None: + """Run against an already-built image; never build or publish implicitly.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--image", default="reme:local") + args = parser.parse_args() + run_smoke(args.image) + + +if __name__ == "__main__": + main() diff --git a/tests/unit/test_as_llm_lazy.py b/tests/unit/test_as_llm_lazy.py new file mode 100644 index 00000000..09c8a646 --- /dev/null +++ b/tests/unit/test_as_llm_lazy.py @@ -0,0 +1,100 @@ +"""Local jobs must not initialize credentialed chat providers at startup.""" + +# pylint: disable=protected-access + +import asyncio +from unittest.mock import AsyncMock, Mock + +import pytest + +from agentscope.model import ChatResponse + +from reme.components.application_context import ApplicationContext +from reme.components.as_llm import BaseAsLLM +from reme.components.job import BaseJob +from reme.enumeration import ComponentEnum + + +class LazyAsLLM(BaseAsLLM): + """Use an isolated mock credential boundary.""" + + credential_cls = Mock() + + +def test_start_without_credentials_and_initialize_once(): + """Startup stays local; first model access preserves provider configuration.""" + + async def go(): + model_cls = Mock() + credential_cls = Mock() + credential_cls.return_value.get_chat_model_class.return_value = model_cls + llm = LazyAsLLM(backend="fake", model="test", parameters={"temperature": 0.5}) + llm.credential_cls = credential_cls + await llm.start() + assert llm._model is None + credential_cls.assert_not_called() + assert llm.model is model_cls.return_value + assert llm.model is model_cls.return_value + credential_cls.assert_called_once_with() + model_cls.Parameters.assert_called_once_with(temperature=0.5) + model_cls.assert_called_once_with( + credential=credential_cls.return_value, + parameters=model_cls.Parameters.return_value, + model="test", + ) + assert llm.model is model_cls.return_value + await llm.close() + + asyncio.run(go()) + + +def test_provider_error_is_reported_when_model_is_requested(): + """Missing credentials fail the model operation and allow a subsequent retry.""" + credential_cls = Mock(side_effect=ValueError("Missing credentials")) + llm = LazyAsLLM(backend="fake", model="test") + llm.credential_cls = credential_cls + with pytest.raises(ValueError, match="Missing credentials"): + _ = llm.model + assert llm._model is None + credential_cls.side_effect = None + assert llm.model is not None + + +def test_compressor_job_initializes_named_model_without_wrapper(tmp_path): + """A started job resolves the independent compressor provider on first use.""" + + async def go(): + context = ApplicationContext(workspace_dir=str(tmp_path)) + model = AsyncMock(return_value=ChatResponse(content=[{"type": "text", "text": "compressed"}], is_last=True)) + credential_cls = Mock() + model_cls = credential_cls.return_value.get_chat_model_class.return_value + model_cls.return_value = model + llm = LazyAsLLM(name="compressor", app_context=context, model="test") + llm.credential_cls = credential_cls + context.components[ComponentEnum.AS_LLM] = {"compressor": llm} + job = BaseJob(app_context=context, steps=[{"backend": "compressor_step", "as_llm": "compressor"}]) + await llm.start() + await job.start() + try: + credential_cls.assert_not_called() + for _ in range(2): + response = await job(text="long source text") + assert response.success, response.answer + assert response.answer == "compressed" + credential_cls.assert_called_once_with() + assert model.await_count == 2 + finally: + await job.close() + await llm.close() + + asyncio.run(go()) + + +def test_explicit_model_assignment_preserves_injection(): + """An injected model bypasses provider construction.""" + llm = LazyAsLLM(model="test") + llm.credential_cls = Mock() + model = Mock() + llm.model = model + assert llm.model is model + llm.credential_cls.assert_not_called() diff --git a/tests/unit/test_auto_dream.py b/tests/unit/test_auto_dream.py index b31ea9e0..1bafb889 100644 --- a/tests/unit/test_auto_dream.py +++ b/tests/unit/test_auto_dream.py @@ -6,13 +6,14 @@ import asyncio import json import tempfile from pathlib import Path -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, Mock, patch import frontmatter import pytest import yaml from reme.components.application_context import ApplicationContext +from reme.components.as_llm import BaseAsLLM from reme.components.agent_wrapper import BaseAgentWrapper from reme.components.file_catalog import BaseFileCatalog from reme.components.file_store import BaseFileStore @@ -20,6 +21,7 @@ from reme.components.job import BaseJob from reme.components.runtime_context import RuntimeContext from reme.components.tag_index import LocalTagIndex from reme.config import resolve_app_config +from reme.enumeration import ComponentEnum from reme.schema import DreamState, FileNode from reme.steps.evolve.auto_tag import AutoTagStep from reme.steps.evolve.dream.extract import DreamExtractStep @@ -791,3 +793,54 @@ def test_finish_keeps_skipped_agent_output_successful(tmp_path): assert response.metadata["dream"]["checkpoint_paths"] == [rel_path] asyncio.run(run()) + + +@pytest.mark.parametrize("stage", ["extract", "integrate"]) +def test_dream_initializes_started_provider_before_availability_check(tmp_path, stage): + """Dream's real availability check must initialize a started lazy provider.""" + + async def run(): + context = ApplicationContext(workspace_dir=str(tmp_path)) + credential_cls = Mock() + llm = BaseAsLLM(app_context=context, model="test") + llm.credential_cls = credential_cls + source = "daily/2026-05-28/session.md" + _touch(tmp_path / source, "memory source") + unit = {"name": "memory", "bucket": "wiki", "summary": "memory summary", "paths": [source]} + target = "digest/wiki/memory.md" + if stage == "extract": + agent = _ReplyAgent({"result": json.dumps({"units": [unit]})}) + step = DreamExtractStep(app_context=context, scan_days=1) + inputs = {"date": "2026-05-28"} + else: + agent = _ReplyAgent( + {"result": json.dumps({"action": "CREATE", "target_path": target, "note": "created"})}, + on_reply=lambda: _touch(tmp_path / target, "# Memory\n\nmemory summary\n"), + ) + step = DreamIntegrateStep(app_context=context) + inputs = {"dream": {"units": [unit], "workspace": str(tmp_path)}} + context.components[ComponentEnum.AS_LLM] = {"default": llm} + context.components[ComponentEnum.AGENT_WRAPPER] = {"default": agent} + await llm.start() + await agent.start() + try: + credential_cls.assert_not_called() + with patch("reme.steps.evolve.dream.extract.refresh_day_index", return_value={}): + response = await step( + RuntimeContext(file_catalog=_Catalog(), file_store=_FileStore(tmp_path), **inputs), + ) + assert response.success, response.answer + assert agent.calls == 1 + credential_cls.assert_called_once_with() + dream = response.metadata["dream"] + assert dream["errors"] == [] + assert dream["failed_paths"] == [] + if stage == "extract": + assert dream["units"][0]["name"] == "memory" + else: + assert dream["integrate_results"][0]["target_path"] == target + finally: + await agent.close() + await llm.close() + + asyncio.run(run()) diff --git a/tests/unit/test_docker_runtime.py b/tests/unit/test_docker_runtime.py new file mode 100644 index 00000000..6a8f44fa --- /dev/null +++ b/tests/unit/test_docker_runtime.py @@ -0,0 +1,146 @@ +"""Container configuration and health checks without Docker or model calls.""" + +import json +from io import BytesIO +from unittest.mock import Mock + +import pytest + +from deploy.docker import reme_container +from reme.config import parse_kwargs, resolve_app_config + + +def test_container_defaults_and_cli_override_precedence(tmp_path): + """CLI overrides win over container settings, including structured service values.""" + config_file = tmp_path / "config.json" + config_file.write_text(json.dumps({"service": {"backend": "http", "port": 8001}, "language": "zh"})) + command = reme_container.start_command( + ['service={"port":8003,"web_enabled":false}', "log_to_file=true"], + { + "REME_CONFIG": str(config_file), + "REME_WORKSPACE_DIR": str(tmp_path / "data"), + "REME_HOST": "0.0.0.0", + "REME_PORT": "8002", + "REME_TIMEZONE": "UTC", + }, + ) + config = resolve_app_config(log_config=False, **parse_kwargs(*command[2:])) + assert config["service"] == {"backend": "http", "host": "0.0.0.0", "port": 8003, "web_enabled": False} + assert config["workspace_dir"] == str(tmp_path / "data") + assert config["timezone"] == "UTC" + assert config["language"] == "zh" + assert config["log_to_file"] is True + + +def test_container_preserves_file_config_port_and_quoted_paths(tmp_path): + """Unset environment options preserve the file; paths survive CLI serialization.""" + config_file = tmp_path / 'configuration "with spaces".json' + config_file.write_text(json.dumps({"service": {"backend": "http", "host": "0.0.0.0", "port": 8234}})) + command = reme_container.start_command([], {"REME_CONFIG": str(config_file)}) + state_path = tmp_path / "health.json" + reme_container.write_health_state(command, state_path) + assert json.loads(state_path.read_text()) == {"url": "http://127.0.0.1:8234/health_check"} + + +def test_health_state_contains_no_config_credentials(tmp_path): + """The ephemeral probe address follows effective CLI settings and stores no secrets.""" + command = reme_container.start_command( + ["service.port=8901", "components.as_llm.default.credential.api_key=not-a-real-key"], + {"REME_PORT": "8900", "REME_HOST": "0.0.0.0"}, + ) + state_path = tmp_path / "health.json" + reme_container.write_health_state(command, state_path) + assert json.loads(state_path.read_text()) == {"url": "http://127.0.0.1:8901/health_check"} + assert state_path.stat().st_mode & 0o777 == 0o600 + + +def test_one_shot_job_does_not_register_http_health_state(tmp_path): + """One-shot jobs use the ordinary CLI lifecycle without pretending to serve HTTP.""" + state_path = tmp_path / "health.json" + command = reme_container.start_command(["job=version"], {}) + reme_container.write_health_state(command, state_path) + assert not state_path.exists() + + +@pytest.mark.parametrize("port", ["0", "65536", "invalid"]) +def test_container_rejects_invalid_port(port, tmp_path): + """Invalid addresses fail before the server starts.""" + with pytest.raises(ValueError): + command = reme_container.start_command([], {"REME_PORT": port}) + reme_container.write_health_state(command, tmp_path / "health.json") + + +@pytest.mark.parametrize( + "payload,expected", + [ + ({"success": True, "metadata": {"health": {"healthy": True}}}, 0), + ({"success": True, "metadata": {"health": {"healthy": False}}}, 1), + ({"success": False, "metadata": {"health": {"healthy": True}}}, 1), + ({"success": True, "metadata": {}}, 1), + ({"success": True, "metadata": None}, 1), + (["not a Job response"], 1), + ], +) +def test_healthcheck_validates_response_body(payload, expected, tmp_path, monkeypatch): + """HTTP 200 alone cannot turn an unhealthy or malformed response into success.""" + state_path = tmp_path / "health.json" + state_path.write_text(json.dumps({"url": "http://127.0.0.1:2333/health_check"})) + opener = Mock() + opener.open.return_value = BytesIO(json.dumps(payload).encode()) + monkeypatch.setattr(reme_container, "build_opener", Mock(return_value=opener)) + assert reme_container.healthcheck(state_path) == expected + request = opener.open.call_args.args[0] + assert request.get_method() == "POST" + assert request.data == b"{}" + + +def test_healthcheck_handles_missing_state_and_unreachable_server(tmp_path, monkeypatch): + """Missing initialization and connection errors fail the probe cleanly.""" + state_path = tmp_path / "health.json" + assert reme_container.healthcheck(state_path) == 1 + state_path.write_text(json.dumps({"url": "http://127.0.0.1:2333/health_check"})) + opener = Mock() + opener.open.side_effect = OSError("connection refused") + monkeypatch.setattr(reme_container, "build_opener", Mock(return_value=opener)) + assert reme_container.healthcheck(state_path) == 1 + + +@pytest.mark.parametrize("action", [["start"], ["--start"], ["reme", "-start"], ["reme", "start"]]) +def test_entrypoint_executes_start_and_preserves_cli_action_syntax(action, tmp_path, monkeypatch): + """All supported start spellings apply defaults and replace the supervisor child.""" + monkeypatch.setattr(reme_container.sys, "argv", ["entrypoint", *action, "service.port=8901"]) + monkeypatch.setattr(reme_container, "HEALTH_STATE_PATH", tmp_path / "health.json") + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.setenv("REME_WORKSPACE_DIR", str(tmp_path / "data")) + monkeypatch.setattr("reme.utils.load_env", Mock()) + state_writer = Mock() + monkeypatch.setattr(reme_container, "write_health_state", state_writer) + exec_command = Mock(side_effect=RuntimeError("exec called")) + monkeypatch.setattr(reme_container.os, "execvp", exec_command) + + with pytest.raises(RuntimeError, match="exec called"): + reme_container.main() + + executable, arguments = exec_command.call_args.args + assert executable == "reme" + assert arguments[:2] == ["reme", "start"] + assert parse_kwargs(*arguments[2:])["service"]["port"] == 8901 + state_writer.assert_called_once_with(arguments) + + +def test_entrypoint_passes_other_commands_without_shell_interpolation(tmp_path, monkeypatch): + """Diagnostic commands retain literal arguments and discard stale HTTP probe state.""" + command = ["python", "-c", "print('$HOME and `literal text`')"] + monkeypatch.setattr(reme_container.sys, "argv", ["entrypoint", *command]) + state_path = tmp_path / "health.json" + state_path.write_text("stale") + monkeypatch.setattr(reme_container, "HEALTH_STATE_PATH", state_path) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + exec_command = Mock(side_effect=RuntimeError("exec called")) + monkeypatch.setattr(reme_container.os, "execvp", exec_command) + + with pytest.raises(RuntimeError, match="exec called"): + reme_container.main() + + exec_command.assert_called_once_with("python", command) + assert not state_path.exists() diff --git a/tests/unit/test_embedded_consumer_compat.py b/tests/unit/test_embedded_consumer_compat.py index 40d12795..af42dbf7 100644 --- a/tests/unit/test_embedded_consumer_compat.py +++ b/tests/unit/test_embedded_consumer_compat.py @@ -3,6 +3,7 @@ # pylint: disable=protected-access import asyncio +from unittest.mock import Mock, patch import pytest @@ -98,22 +99,30 @@ def test_qwenpaw_style_config_keeps_in_process_application_api(tmp_path): asyncio.run(exercise_api()) -def test_update_component_validates_all_fields_before_mutation(tmp_path): - """A rejected field update does not leave earlier attributes changed.""" +@pytest.mark.parametrize("injected", [False, True], ids=["uninitialized", "injected"]) +def test_update_component_validates_all_fields_before_mutation(tmp_path, injected): + """Rejected updates preserve model state without constructing a provider.""" app = ReMe(**_qwenpaw_style_config(str(tmp_path))) component = app.context.components[ComponentEnum.AS_LLM]["default"] - original_model = component.model + original_model = object() if injected else None + if injected: + component.model = original_model + credential_cls = Mock(side_effect=AssertionError("provider must not be constructed")) async def exercise_api() -> None: - with pytest.raises(AttributeError, match="does_not_exist"): - await app.update_component( - "as_llm", - "default", - model=object(), - does_not_exist=True, - ) + with patch("reme.components.as_llm.OpenAIAsLLM.credential_cls", credential_cls): + with pytest.raises(AttributeError, match="does_not_exist"): + await app.update_component( + "as_llm", + "default", + model=object(), + does_not_exist=True, + ) - assert component.model is original_model + assert component._model is original_model + if injected: + assert component.model is original_model + credential_cls.assert_not_called() asyncio.run(exercise_api()) @@ -362,3 +371,31 @@ def test_replace_component_start_failure_keeps_old_generation(tmp_path): await app.close() asyncio.run(exercise_api()) + + +@pytest.mark.parametrize("operation", ["update", "replace"]) +def test_model_injection_does_not_initialize_provider(tmp_path, operation): + """Attribute validation must not construct a provider before model injection.""" + app = ReMe(**_qwenpaw_style_config(str(tmp_path))) + model = object() + credential_cls = Mock(side_effect=AssertionError("provider must not be constructed")) + + async def run(): + with patch("reme.components.as_llm.OpenAIAsLLM.credential_cls", credential_cls): + if operation == "update": + component = await app.update_component("as_llm", "default", model=model) + else: + component = await app.replace_component( + "as_llm", + "default", + config={"backend": "openai", "model": "injected"}, + runtime_updates={"model": model}, + ) + await app.start() + try: + assert component.model is model + credential_cls.assert_not_called() + finally: + await app.close() + + asyncio.run(run()) diff --git a/tests/unit/test_service_utils.py b/tests/unit/test_service_utils.py index 32f5dea2..fbed446a 100644 --- a/tests/unit/test_service_utils.py +++ b/tests/unit/test_service_utils.py @@ -13,10 +13,12 @@ shell-outs: # pylint: disable=protected-access,missing-function-docstring,unused-argument import os +import json import socket from types import SimpleNamespace import psutil +import pytest from reme.constants import normalize_connect_host from reme.utils import service_utils as su @@ -192,3 +194,54 @@ def test_running_app_config_preserves_plugins(monkeypatch): "service": {"backend": "plugin-client", "port": 9911}, } assert su.running_service_config() == {"backend": "plugin-client", "port": 9911} + + +def test_discovery_ignores_container_init_and_replays_real_cli(monkeypatch): + """Tini's original argv omits environment overrides applied by the entrypoint.""" + procs = [ + _FakeProc(1, cmdline=["/usr/bin/tini", "--", "python", "/usr/local/lib/reme_container.py", "start"]), + _FakeProc(2, cmdline=["/opt/venv/bin/python", "/opt/venv/bin/reme", "start", 'service={"port":8124}']), + ] + _patch_iter(monkeypatch, procs) + assert su._reme_start_argv() == [['service={"port":8124}']] + assert su.running_service_config()["port"] == 8124 + assert su._scan_reme_procs() == [(2, su.REME_DEFAULT_HOST, 8124)] + + +def test_discovery_resolves_mounted_config_port(tmp_path, monkeypatch): + """find_reme follows ports from JSON configs as well as dot-notation overrides.""" + config = tmp_path / "reme.json" + config.write_text(json.dumps({"service": {"backend": "http", "host": "0.0.0.0", "port": 8125}})) + _patch_iter(monkeypatch, [_FakeProc(2, cmdline=["reme", "start", f"config={config}"])]) + assert su._scan_reme_procs() == [(2, "0.0.0.0", 8125)] + + +@pytest.mark.parametrize( + "command", + [ + ["reme", "--start"], + ["python3.11", "-u", "-m", "reme.reme", "-start"], + ["/opt/venv/bin/python", "/opt/venv/bin/reme", "start"], + ["C:\\Python\\python.exe", "C:\\Python\\Scripts\\reme.exe", "start"], + ], +) +def test_discovery_accepts_cli_launchers_and_action_prefixes(command, monkeypatch): + _patch_iter(monkeypatch, [_FakeProc(7, cmdline=command)]) + assert su._reme_start_argv() == [[]] + assert su._scan_reme_procs() == [(7, su.REME_DEFAULT_HOST, su.REME_DEFAULT_PORT)] + + +@pytest.mark.parametrize( + "command", + [ + ["tini", "--", "reme", "start"], + ["dumb-init", "reme", "start"], + ["python", "/usr/local/lib/reme_container.py", "start"], + ["python", "-c", "reme.py", "start"], + ["other-reme-app", "start"], + ], +) +def test_discovery_filters_supervisors_and_unrelated_commands(command, monkeypatch): + _patch_iter(monkeypatch, [_FakeProc(7, cmdline=command)]) + assert not su._reme_start_argv() + assert not su._scan_reme_procs()