From 166c578f3f3cbe1124d9d69cf00427d66fe33e05 Mon Sep 17 00:00:00 2001 From: Docker_Admin_v1 Date: Tue, 7 Apr 2026 10:42:31 +0000 Subject: [PATCH] feat(docker): major improvements to docker deployment New files: - .env.example - Comprehensive environment template - docker-compose.prod.yml - Production overrides - docker-compose.override.yml - Development overrides Key improvements: - docker-compose.yml: Configurable port, healthcheck, resource limits, log rotation - Dockerfile: Non-root user, security hardening, better caching - DOCKER.md: Complete rewrite with troubleshooting, backup/restore, monitoring Co-authored-by: Docker_Admin_v1 --- .env.example | 50 +++++++ DOCKER.md | 283 +++++++++++++++++++++++++++++++----- Dockerfile | 43 +++++- docker-compose.override.yml | 22 +++ docker-compose.prod.yml | 22 +++ docker-compose.yml | 53 ++++++- 6 files changed, 421 insertions(+), 52 deletions(-) create mode 100644 .env.example create mode 100644 docker-compose.override.yml create mode 100644 docker-compose.prod.yml diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..87b03c3 --- /dev/null +++ b/.env.example @@ -0,0 +1,50 @@ +# OpenSpace Docker Environment Configuration +# Copy this file to .env and customize your settings + +# ==================== Required ==================== +# Generate a secure random key for your OpenSpace instance +# You can generate one with: openssl rand -base64 32 +OPENSPACE_API_KEY=your_openpspace_api_key_here + +# ==================== LLM Providers ==================== +# Choose ONE provider configuration below + +# Option 1: OpenAI (GPT-4, GPT-3.5, etc.) +# OPENAI_API_KEY=sk-... + +# Option 2: Anthropic (Claude) +# ANTHROPIC_API_KEY=sk-ant-... + +# Option 3: StepFun (Step models via OpenRouter) +# Note: Using OpenRouter as the API gateway +OPENSPACE_MODEL=stepfun/step-3.5-flash:free +OPENSPACE_LLM_API_KEY=sk-or-v1-... +OPENSPACE_LLM_API_BASE=https://openrouter.ai/api/v1 + +# Option 4: Custom provider (Local LLM, Together, etc.) +# OPENSPACE_LLM_API_KEY=your_api_key +# OPENSPACE_LLM_API_BASE=https://your-llm-provider.com/v1 +# OPENSPACE_MODEL=your-model-name + +# ==================== Optional ==================== +# Default model to use when not specified in queries +# OPENSPACE_MODEL=claude-3-7-sonnet-latest + +# Enable debug logging (set to 1) +# OPENSPACE_DEBUG=0 + +# ==================== Docker Compose Configuration ==================== +# Override default host port (default: 9001) +# HOST_PORT=9001 + +# Volume type: "named" (Docker-managed) or "bind" (host directory) +# VOLUME_TYPE=named + +# When VOLUME_TYPE=named, these named volumes are used: +# VOLUME_DATA=openspace-data +# VOLUME_SKILLS=openspace-skills + +# When VOLUME_TYPE=bind, create local directories: +# (not set via env, edit docker-compose.yml directly) +# - ./data:/app/.openspace +# - ./skills:/app/skills diff --git a/DOCKER.md b/DOCKER.md index 91d07c8..2cff714 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -4,75 +4,280 @@ This guide provides instructions on how to run OpenSpace in a Docker container. ## Prerequisites -- [Docker](https://docs.docker.com/get-docker/) -- [Docker Compose](https://docs.docker.com/compose/install/) +- [Docker](https://docs.docker.com/get-docker/) (20.10+) +- [Docker Compose](https://docs.docker.com/compose/install/) (v2.0+) ## Quick Start (Docker Compose) The easiest way to get OpenSpace running is using `docker-compose`. -1. **Clone the repository:** +### 1. Clone the Repository - ```bash - git clone https://github.com/HKUDS/OpenSpace.git - cd OpenSpace - ``` +```bash +git clone https://github.com/HKUDS/OpenSpace.git +cd OpenSpace +git checkout feat/docker-deployment +``` -2. **Configure Environment Variables:** +### 2. Configure Environment Variables - Create a `.env` file in the root directory and add your API keys: +```bash +# Copy the example environment file +cp .env.example .env - ```bash - cp .env.example .env - # Edit .env with your favorite editor and set your API keys: - # OPENSPACE_API_KEY=your_key - # OPENAI_API_KEY=your_key - # ANTHROPIC_API_KEY=your_key - - # Optionally, configure custom LLM models/providers: - # OPENSPACE_MODEL=deepseek/deepseek-chat - # OPENSPACE_LLM_API_KEY=sk-xxxx - # OPENSPACE_LLM_API_BASE=https://api.deepseek.com/v1 - ``` +# Edit .env with your configuration +# Required at minimum: +# OPENSPACE_API_KEY=$(openssl rand -base64 32) +# +# Choose and configure ONE LLM provider: +# - OpenAI + OPENAI_API_KEY +# - Anthropic + ANTHROPIC_API_KEY +# - OpenRouter + OPENSPACE_LLM_API_KEY + OPENSPACE_LLM_API_BASE + OPENSPACE_MODEL +``` -3. **Build and Run:** +**Environment Variables Reference:** - ```bash - docker-compose up -d --build - ``` +| Variable | Required | Description | +|----------|----------|-------------| +| `OPENSPACE_API_KEY` | **Yes** | Secret API key for this OpenSpace instance. Generate with `openssl rand -base64 32` | +| `OPENAI_API_KEY` | No | OpenAI API key (sk-...) | +| `ANTHROPIC_API_KEY` | No | Anthropic API key (sk-ant-...) | +| `OPENSPACE_MODEL` | No | Default model name (e.g., `claude-3-7-sonnet-latest`, `stepfun/step-3.5-flash:free`) | +| `OPENSPACE_LLM_API_KEY` | No | API key for custom LLM provider | +| `OPENSPACE_LLM_API_BASE` | No | Base URL for custom LLM provider (e.g., `https://openrouter.ai/api/v1`) | +| `OPENSPACE_DEBUG` | No | Set to `1` to enable debug logging | +| `HOST_PORT` | No | Host port to expose (default: `9001`) | +| `VOLUME_TYPE` | No | `named` (default) or `bind` for local directories | -4. **Access the Dashboard:** +### 3. Build and Run - Open your browser and navigate to [http://localhost:7788](http://localhost:7788). The frontend and backend are both served seamlessly from the same container. +```bash +# Standard docker-compose (uses docker-compose.yml) +docker compose up -d --build -## Interacting with the CLI +# For production with stricter resource limits: +# docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build + +# To stop: +docker compose down + +# To view logs: +docker compose logs -f openspace + +# To check health status: +docker compose ps +``` + +### 4. Access the Dashboard + +Open your browser and navigate to: + +- **Dashboard:** http://localhost:${HOST_PORT:-9001} +- Health endpoint: http://localhost:${HOST_PORT:-9001}/health + +The frontend and backend are served seamlessly from the same container. + +--- + +## CLI Usage You can use the container to execute OpenSpace CLI commands. -**To run a single task using the CLI inside the running container:** +### Run a Query ```bash -docker exec -it openspace openspace --model "anthropic/claude-sonnet-4-5" --query "Analyze the local skills" +docker exec -it openspace openspace --model "anthropic/claude-sonnet-4.5" --query "Analyze the local skills" ``` -**To download/upload skills:** +### Download/Upload Skills ```bash docker exec -it openspace openspace-download-skill docker exec -it openspace openspace-upload-skill /app/skills/my-skill ``` -## Volumes +### Enter Container Shell -The `docker-compose.yml` is configured with two persistent volumes: +```bash +docker exec -it openspace bash +``` -- `openspace-data`: Mounted to `/app/.openspace`, storing the SQLite database (`openspace.db`) containing the skill evolution history and metadata. -- `openspace-skills`: Mounted to `/app/skills`, allowing you to persist downloaded and custom skills across container restarts. +--- -If you prefer to mount a local directory for your skills so you can edit them directly from your host machine, you can update the `docker-compose.yml` file: +## Volume & Data Management + +### Volume Types + +1. **Named volumes** (default) - Managed by Docker, good for simple deployments + - `openspace-data`: Contains SQLite database (`openspace.db`) and skill history + - `openspace-skills`: Persists downloaded and custom skills + +2. **Bind mounts** - Direct host directory access, better for development + Edit `docker-compose.yml`: + ```yaml + volumes: + - ./data:/app/.openspace + - ./skills:/app/skills + ``` + +### Backup & Restore + +**Backup:** +```bash +# Named volumes +docker run --rm -v openspace-data:/data -v $(pwd):/backup alpine tar czf /backup/openspace-data-$(date +%Y%m%d).tar.gz -C /data . + +# Bind mounts (just copy the directories) +cp -r data skills backup/ +``` + +**Restore:** +```bash +# Named volumes +docker run --rm -v openspace-data:/data -v $(pwd):/backup alpine sh -c "rm -rf /data/* && tar xzf /backup/openspace-data-YYYYMMDD.tar.gz -C /data" + +# Bind mounts +cp -r backup/data backup/skills ./ +``` + +--- + +## Monitoring + +### Health Check + +OpenSpace container includes a health check that pings `/health` endpoint every 30 seconds. Check status: + +```bash +docker compose ps +# Look for "healthy" in the STATUS column +``` + +### Prometheus Metrics + +If you have a Prometheus instance, you can scrape metrics from OpenSpace. Add to your `docker-compose.yml`: ```yaml - volumes: - - openspace-data:/app/.openspace - - ./my-local-skills:/app/skills +services: + openspace: + # Add this label for service discovery + labels: + - "prometheus-job=openspace" ``` + +Then configure Prometheus to scrape `openspace:7788/metrics` (if endpoint is available). + +### Logs + +Logs are configured with rotation (10MB max, 3 files by default). View logs: + +```bash +docker compose logs -f openspace +# Or with timestamps +docker compose logs -f --timestamp openspace +``` + +For centralized logging, consider using Loki/Promtail stack (not included by default). + +--- + +## Troubleshooting + +### Container fails to start + +Check logs: +```bash +docker compose logs openspace +``` + +Common issues: +- **Missing OPENSPACE_API_KEY**: Set it in `.env` file +- **Port already in use**: Change `HOST_PORT` in `.env` or stop the conflicting service +- **Insufficient memory**: Increase Docker memory limit (Settings → Resources) + +### Health check failing + +The health check endpoint `/health` might not be available in older versions. If using a development build, you may need to disable the health check by removing it from `docker-compose.yml`. + +### Permission denied on volumes + +If using bind mounts, ensure the host directories are readable/writable by the container user (UID 1000). Fix with: +```bash +sudo chown -R 1000:1000 data skills +``` + +--- + +## Production Deployment Checklist + +- [ ] Generate a strong `OPENSPACE_API_KEY` and keep it secret +- [ ] Configure SSL/TLS termination (use reverse proxy like nginx or Traefik) +- [ ] Set appropriate resource limits (memory: 2-4G, CPU: 2-4 cores) +- [ ] Enable log rotation and set up log aggregation +- [ ] Configure regular backups of `openspace-data` volume +- [ ] Set up monitoring (Prometheus + Grafana) +- [ ] Use `docker-compose.prod.yml` for additional production settings +- [ ] Restrict access to the API and dashboard via firewall/VPC +- [ ] Keep Docker and base images updated regularly + +--- + +## Advanced Configuration + +### Custom Network + +To integrate with other services on a custom network: + +```yaml +networks: + app-network: + driver: bridge + +services: + openspace: + networks: + - app-network +``` + +### Multi-stage Deployment (with separate frontend/backend) + +For large-scale deployments, you might split frontend and backend services. See `docker-compose.multi.yml` (if available). + +### Environment-Specific Configs + +Use multiple compose files: + +```bash +# Development (with hot-reload, less resource limits) +docker compose -f docker-compose.yml -f docker-compose.override.yml up -d + +# Production (strict limits, optimized) +docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d +``` + +--- + +## Security Notes + +- The Dockerfile creates and uses a non-root user `appuser` (UID 1000) +- `OPENSPACE_API_KEY` should be treated as a secret; rotate periodically +- Network access is limited to what the container needs; avoid `--network host` +- Keep the host and Docker daemon updated to prevent vulnerabilities + +--- + +## Contributing + +Found an issue or want to improve the Docker deployment? PRs welcome! + +Please update: +- `docker-compose.yml` (core config) +- `Dockerfile` (build instructions) +- `DOCKER.md` (this documentation) +- Add/maintain `.env.example` + +--- + +## License + +MIT diff --git a/Dockerfile b/Dockerfile index efc952d..7866016 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,25 @@ # Stage 1: Build the frontend FROM node:20-slim AS frontend-builder WORKDIR /app/frontend + +# Copy package files and install dependencies (cached layer) COPY frontend/package.json frontend/package-lock.json* ./ -RUN npm install +RUN npm ci --only=production + +# Copy source and build COPY frontend/ ./ RUN npm run build -# Stage 2: Build the backend and serve +# Stage 2: Build the backend FROM python:3.12-slim + +# Create a non-root user for security +RUN groupadd -r appuser && useradd -r -g appuser appuser + WORKDIR /app -# Install system dependencies required for python packages and GUI features +# Install system dependencies required for python packages +# Separate apt commands to leverage cache and clean up in same layer RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ python3-dev \ @@ -18,13 +27,22 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # Copy backend source -COPY . /app/ +COPY --chown=appuser:appuser . /app/ -# Install the Python package with linux optional dependencies -RUN pip install --no-cache-dir -e .[linux] +# Switch to non-root user for pip install (when possible) +# Some packages may need system deps, but we installed them as root above +USER appuser -# Copy built frontend from Stage 1 -COPY --from=frontend-builder /app/frontend/dist /app/frontend/dist +# Install Python package with minimal dependencies (linux extras require additional system deps) +# Use --no-cache-dir to reduce image size +RUN pip install --no-cache-dir -e . + +# Copy built frontend from Stage 1 (needs root to change ownership) +USER root +COPY --from=frontend-builder --chown=appuser:appuser /app/frontend/dist /app/frontend/dist + +# Switch back to non-root user +USER appuser # Expose the dashboard port EXPOSE 7788 @@ -33,6 +51,15 @@ EXPOSE 7788 ENV HOST=0.0.0.0 ENV PORT=7788 ENV OPENSPACE_WORKSPACE=/app +ENV PYTHONUNBUFFERED=1 + +# Create necessary directories with proper permissions +RUN mkdir -p /app/.openspace /app/skills && \ + chmod 700 /app/.openspace /app/skills + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:7788/health', timeout=5)" || exit 1 # Run the dashboard server by default CMD ["openspace-dashboard", "--host", "0.0.0.0", "--port", "7788"] diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000..edbd069 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,22 @@ +# Development overrides for OpenSpace +# Automatically used by docker-compose up in development + +services: + openspace: + # Mount source code for live editing (requires rebuild on changes) + # volumes: + # - ./openspace:/app/openspace:ro + # - ./frontend:/app/frontend:ro + # Development-friendly resource limits + deploy: + resources: + limits: + memory: 4G # More memory for dev + reservations: + memory: 1G + logging: + options: + max-size: "20m" + max-file: "3" + labels: + - "com.openspace.environment=development" diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..130d9e7 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,22 @@ +# Production-specific overrides for OpenSpace +# Usage: docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d + +services: + openspace: + # Production resource limits + deploy: + resources: + limits: + memory: 4G + cpus: '4.0' + reservations: + memory: 1G + cpus: '1.0' + # More aggressive log rotation for production + logging: + options: + max-size: "50m" + max-file: "5" + labels: + - "com.openspace.environment=production" + - "com.openspace.log-level=info" diff --git a/docker-compose.yml b/docker-compose.yml index b3c8df4..babfcf3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: openspace: build: @@ -7,19 +5,64 @@ services: dockerfile: Dockerfile container_name: openspace ports: - - "7788:7788" + - "${HOST_PORT:-9001}:7788" volumes: - - openspace-data:/app/.openspace - - openspace-skills:/app/skills + # Use bind mounts for easier access, or named volumes for persistence + - ${VOLUME_TYPE:-named}:${VOLUME_DATA:-openspace-data}:/app/.openspace + - ${VOLUME_TYPE:-named}:${VOLUME_SKILLS:-openspace-skills}:/app/skills + # For bind mounts, you would use: + # - ./data:/app/.openspace + # - ./skills:/app/skills environment: + # Required: Generate a secure API key for this instance - OPENSPACE_API_KEY=${OPENSPACE_API_KEY:-} + # Optional: OpenAI API key (if using OpenAI models) - OPENAI_API_KEY=${OPENAI_API_KEY:-} + # Optional: Anthropic API key (if using Claude) - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} + # Optional: Default model to use (e.g., "claude-3-7-sonnet-latest") - OPENSPACE_MODEL=${OPENSPACE_MODEL:-} + # Optional: Custom LLM provider (like OpenRouter, local LLM) - OPENSPACE_LLM_API_KEY=${OPENSPACE_LLM_API_KEY:-} - OPENSPACE_LLM_API_BASE=${OPENSPACE_LLM_API_BASE:-} + # Optional: Set to "1" to enable debug logging + - OPENSPACE_DEBUG=${OPENSPACE_DEBUG:-0} + # Internal config + - HOST=0.0.0.0 + - PORT=7788 + - OPENSPACE_WORKSPACE=/app restart: unless-stopped + # Health check to ensure the service is ready + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:7788/health || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + # Resource limits (adjust based on your needs) + deploy: + resources: + limits: + memory: 2G + cpus: '2.0' + reservations: + memory: 512M + cpus: '0.5' + # Logging configuration to prevent disk fill + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + # Labels for monitoring and identification + labels: + - "com.openspace.service=true" + - "com.openspace.component=api" + - "com.openspace.version=${OPENSPACE_VERSION:-latest}" +# Named volumes for data persistence (used when VOLUME_TYPE=named) volumes: openspace-data: + driver: local openspace-skills: + driver: local