diff --git a/apps/docs/docs.json b/apps/docs/docs.json index 75d91754..c11017b1 100644 --- a/apps/docs/docs.json +++ b/apps/docs/docs.json @@ -185,6 +185,7 @@ "self-hosting/quickstart", "self-hosting/configuration", "self-hosting/embeddings", + "self-hosting/backup-and-export", "self-hosting/providers", "self-hosting/local-vs-enterprise" ] diff --git a/apps/docs/self-hosting/backup-and-export.mdx b/apps/docs/self-hosting/backup-and-export.mdx new file mode 100644 index 00000000..d217f3aa --- /dev/null +++ b/apps/docs/self-hosting/backup-and-export.mdx @@ -0,0 +1,213 @@ +--- +title: "Backup & Export" +sidebarTitle: "Backup & Export" +description: "How to export and back up distilled memories and documents from your self-hosted instance." +icon: "download" +--- + +Self-hosted Supermemory stores all state inside `$SUPERMEMORY_DATA_DIR` (default: `./.supermemory`). +This guide explains how to back up and export both your **distilled memory cards** (extracted facts) and **raw ingested documents**. + +## Overview: Memories vs Documents + +Supermemory distinguishes between two layers of context: + +1. **Documents (`/v3/documents`)**: The raw source material you ingested (chat logs, transcripts, Markdown files, PDFs, web pages). +2. **Extracted Memories (`/v4/memories`)**: The distilled, atomic facts extracted from documents, complete with version history, temporal status, and container tags. + +> **Why not copy `./.supermemory/data` directly?** +> The local store uses an encrypted PGlite container (`SMD1` format) keyed to the machine environment. To produce portable, human-readable backups or migrate between machines, use the HTTP export APIs described below. + +--- + +## Exporting Extracted Memories + +To export the distilled memory cards for any container tag, use the `POST /v4/memories/list` endpoint. Unlike search recall (`searchMode: "memories"`), which only returns top-K semantically matching items, `/v4/memories/list` provides **exhaustive, paginated access** to all extracted facts. + +### Endpoint + +```http +POST /v4/memories/list +Content-Type: application/json +Authorization: Bearer +``` + +#### Request Payload + +```json +{ + "containerTags": ["user_123"], + "page": 1, + "limit": 50, + "sort": "createdAt", + "order": "desc" +} +``` + +#### Response Structure + +```json +{ + "memoryEntries": [ + { + "id": "mem_01J6ABC...", + "memory": "User prefers TypeScript and strict mode enabled", + "version": 1, + "isLatest": true, + "isForgotten": false, + "isStatic": true, + "createdAt": "2026-08-20T10:00:00.000Z", + "updatedAt": "2026-08-20T10:00:00.000Z", + "documentIds": ["doc_01J6XYZ..."], + "history": [] + } + ], + "pagination": { + "currentPage": 1, + "limit": 50, + "totalItems": 142, + "totalPages": 3 + } +} +``` + +--- + +## Full Export Scripts + +### 1. Using `@supermemory/tools` (Node.js / Bun) + +If you use `@supermemory/tools`, use the built-in export helpers: + +```typescript +import { + exportMemoriesAsJson, + exportMemoriesAsMarkdown, + fetchAllMemories, +} from "@supermemory/tools"; + +const options = { + baseUrl: "http://localhost:6767", + apiKey: process.env.SUPERMEMORY_API_KEY || "sm_...", +}; + +// Export to JSON backup +const jsonBackup = await exportMemoriesAsJson("user_123", options); +await Bun.write("memories-backup.json", jsonBackup); + +// Export to human-readable Markdown +const markdownNotes = await exportMemoriesAsMarkdown("user_123", options); +await Bun.write("memories-export.md", markdownNotes); +``` + +### 2. Standalone Python Export Script + +Run this script against your local server to export all memories to JSON: + +```python +import os +import json +import requests + +BASE_URL = os.getenv("SUPERMEMORY_API_URL", "http://localhost:6767") +API_KEY = os.getenv("SUPERMEMORY_API_KEY", "") +CONTAINER_TAG = "user_123" + +def export_all_memories(container_tag): + headers = { + "Authorization": f"Bearer {API_KEY}", + "Content-Type": "application/json" + } + + page = 1 + limit = 50 + all_memories = [] + + while True: + payload = { + "containerTags": [container_tag], + "page": page, + "limit": limit, + "sort": "createdAt", + "order": "desc" + } + res = requests.post(f"{BASE_URL}/v4/memories/list", json=payload, headers=headers) + res.raise_for_status() + data = res.json() + + entries = data.get("memoryEntries", []) + all_memories.extend(entries) + + pagination = data.get("pagination", {}) + total_pages = pagination.get("totalPages", 1) + + print(f"Fetched page {page} / {total_pages} ({len(entries)} entries)") + if page >= total_pages or not entries: + break + page += 1 + + return all_memories + +if __name__ == "__main__": + memories = export_all_memories(CONTAINER_TAG) + output_file = f"memories_{CONTAINER_TAG}.json" + with open(output_file, "w", encoding="utf-8") as f: + json.dump({ + "containerTag": CONTAINER_TAG, + "totalCount": len(memories), + "memories": memories + }, f, indent=2) + print(f"Exported {len(memories)} memories to {output_file}") +``` + +### 3. Quick cURL Export + +```bash +curl -s -X POST "http://localhost:6767/v4/memories/list" \ + -H "Authorization: Bearer $SUPERMEMORY_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "containerTags": ["user_123"], + "page": 1, + "limit": 50 + }' | jq . +``` + +--- + +## Exporting Raw Documents + +To back up the underlying conversation sessions, files, and text documents: + +```bash +curl -s "http://localhost:6767/v3/documents?containerTag=user_123&page=1&limit=50" \ + -H "Authorization: Bearer $SUPERMEMORY_API_KEY" | jq . +``` + +To fetch the full text of a specific document: + +```bash +curl -s "http://localhost:6767/v3/documents/{documentId}" \ + -H "Authorization: Bearer $SUPERMEMORY_API_KEY" | jq . +``` + +--- + +## Migration to Hosted Platform + +To migrate your exported memories from a local server to the hosted platform: + +1. Export your memory cards using the scripts above. +2. Direct-write the memories into the hosted platform via `POST /v4/memories`: + +```bash +curl -X POST "https://api.supermemory.ai/v4/memories" \ + -H "Authorization: Bearer $SUPERMEMORY_CLOUD_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "memories": [ + { "content": "User prefers TypeScript and strict mode enabled", "isStatic": true } + ], + "containerTag": "user_123" + }' +``` diff --git a/apps/docs/self-hosting/overview.mdx b/apps/docs/self-hosting/overview.mdx index 8fc9fe23..11460950 100644 --- a/apps/docs/self-hosting/overview.mdx +++ b/apps/docs/self-hosting/overview.mdx @@ -75,7 +75,7 @@ If you outgrow a single machine — or want connectors, MCP, and the best-tuned ## Next steps - + Install, run, and store your first memory in under two minutes @@ -85,4 +85,7 @@ If you outgrow a single machine — or want connectors, MCP, and the best-tuned Local default, remote providers, multilingual, dimension lock + + Export and back up distilled memories and documents + diff --git a/packages/tools/src/export-memories.test.ts b/packages/tools/src/export-memories.test.ts new file mode 100644 index 00000000..3fd20aa1 --- /dev/null +++ b/packages/tools/src/export-memories.test.ts @@ -0,0 +1,250 @@ +import { describe, expect, it } from "bun:test" +import { + exportMemoriesAsJson, + exportMemoriesAsMarkdown, + fetchAllMemories, + listMemoriesRequest, + type MemoriesListResponse, +} from "./shared/export-memories" + +function createMockFetch(responses: MemoriesListResponse[]) { + let callCount = 0 + const fetchMock = async (url: string | URL | Request, init?: RequestInit) => { + const pageResponse = responses[callCount] || { + memoryEntries: [], + pagination: { currentPage: callCount + 1, limit: 10, totalItems: 0, totalPages: 1 }, + } + callCount++ + + return { + ok: true, + status: 200, + statusText: "OK", + json: async () => pageResponse, + text: async () => JSON.stringify(pageResponse), + } as Response + } + return { + fetchFn: fetchMock as unknown as typeof fetch, + getCallCount: () => callCount, + } +} + +describe("listMemoriesRequest", () => { + it("sends correct POST body and headers to /v4/memories/list", async () => { + let capturedUrl = "" + let capturedInit: RequestInit | undefined + + const customFetch = async (url: string | URL | Request, init?: RequestInit) => { + capturedUrl = String(url) + capturedInit = init + return { + ok: true, + status: 200, + json: async () => ({ + memoryEntries: [ + { + id: "mem_1", + memory: "User prefers TypeScript", + version: 1, + isLatest: true, + isForgotten: false, + createdAt: "2026-09-01T10:00:00.000Z", + updatedAt: "2026-09-01T10:00:00.000Z", + }, + ], + pagination: { currentPage: 1, limit: 10, totalItems: 1, totalPages: 1 }, + }), + } as Response + } + + const result = await listMemoriesRequest( + "sm_test_key", + { containerTags: ["user_123"], page: 1, limit: 10 }, + "http://localhost:6767", + { fetchFn: customFetch as unknown as typeof fetch }, + ) + + expect(capturedUrl).toBe("http://localhost:6767/v4/memories/list") + expect(capturedInit?.method).toBe("POST") + expect((capturedInit?.headers as Record)?.Authorization).toBe( + "Bearer sm_test_key", + ) + const parsedBody = JSON.parse(String(capturedInit?.body)) + expect(parsedBody.containerTags).toEqual(["user_123"]) + expect(parsedBody.page).toBe(1) + expect(result.memoryEntries).toHaveLength(1) + expect(result.memoryEntries[0].memory).toBe("User prefers TypeScript") + }) + + it("throws on non-200 responses with descriptive error", async () => { + const customFetch = async () => + ({ + ok: false, + status: 400, + statusText: "Bad Request", + text: async () => JSON.stringify({ error: "Container tag is required" }), + }) as unknown as Response + + expect( + listMemoriesRequest( + "sm_key", + { containerTags: [] }, + "http://localhost:6767", + { fetchFn: customFetch as unknown as typeof fetch }, + ), + ).rejects.toThrow("Supermemory list memories failed: 400 Bad Request") + }) +}) + +describe("fetchAllMemories", () => { + it("paginates through all pages until totalPages is reached", async () => { + const page1: MemoriesListResponse = { + memoryEntries: [ + { + id: "mem_1", + memory: "Fact 1", + version: 1, + isLatest: true, + isForgotten: false, + createdAt: "2026-09-01T10:00:00.000Z", + updatedAt: "2026-09-01T10:00:00.000Z", + }, + { + id: "mem_2", + memory: "Fact 2 (forgotten)", + version: 1, + isLatest: true, + isForgotten: true, + createdAt: "2026-09-01T11:00:00.000Z", + updatedAt: "2026-09-01T11:00:00.000Z", + }, + ], + pagination: { currentPage: 1, limit: 2, totalItems: 3, totalPages: 2 }, + } + + const page2: MemoriesListResponse = { + memoryEntries: [ + { + id: "mem_3", + memory: "Fact 3", + version: 1, + isLatest: true, + isForgotten: false, + createdAt: "2026-09-01T12:00:00.000Z", + updatedAt: "2026-09-01T12:00:00.000Z", + }, + ], + pagination: { currentPage: 2, limit: 2, totalItems: 3, totalPages: 2 }, + } + + const { fetchFn, getCallCount } = createMockFetch([page1, page2]) + const memories = await fetchAllMemories("user_123", { + baseUrl: "http://localhost:6767", + apiKey: "sm_key", + fetchFn, + }) + + expect(getCallCount()).toBe(2) + // By default forgotten memories are excluded + expect(memories).toHaveLength(2) + expect(memories[0].id).toBe("mem_1") + expect(memories[1].id).toBe("mem_3") + }) + + it("includes forgotten memories when includeForgotten is true", async () => { + const page1: MemoriesListResponse = { + memoryEntries: [ + { + id: "mem_1", + memory: "Fact 1", + version: 1, + isLatest: true, + isForgotten: false, + createdAt: "2026-09-01T10:00:00.000Z", + updatedAt: "2026-09-01T10:00:00.000Z", + }, + { + id: "mem_2", + memory: "Fact 2 (forgotten)", + version: 1, + isLatest: true, + isForgotten: true, + createdAt: "2026-09-01T11:00:00.000Z", + updatedAt: "2026-09-01T11:00:00.000Z", + }, + ], + pagination: { currentPage: 1, limit: 2, totalItems: 2, totalPages: 1 }, + } + + const { fetchFn } = createMockFetch([page1]) + const memories = await fetchAllMemories("user_123", { + includeForgotten: true, + fetchFn, + }) + + expect(memories).toHaveLength(2) + expect(memories[1].id).toBe("mem_2") + expect(memories[1].isForgotten).toBe(true) + }) +}) + +describe("exportMemoriesAsJson and exportMemoriesAsMarkdown", () => { + const sampleData: MemoriesListResponse = { + memoryEntries: [ + { + id: "mem_abc123", + memory: "User works at Acme Corp", + version: 2, + isLatest: true, + isForgotten: false, + isStatic: true, + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-10T00:00:00.000Z", + documentIds: ["doc_1", "doc_2"], + history: [ + { + id: "mem_abc122", + memory: "User works at Startup", + version: 1, + createdAt: "2026-07-01T00:00:00.000Z", + updatedAt: "2026-07-01T00:00:00.000Z", + }, + ], + }, + ], + pagination: { currentPage: 1, limit: 50, totalItems: 1, totalPages: 1 }, + } + + it("exports memories as valid JSON with backup metadata", async () => { + const { fetchFn } = createMockFetch([sampleData]) + const jsonString = await exportMemoriesAsJson("user_123", { + baseUrl: "http://localhost:6767", + fetchFn, + }) + + const parsed = JSON.parse(jsonString) + expect(parsed.containerTag).toBe("user_123") + expect(parsed.totalCount).toBe(1) + expect(parsed.baseUrl).toBe("http://localhost:6767") + expect(parsed.memories).toHaveLength(1) + expect(parsed.memories[0].id).toBe("mem_abc123") + expect(parsed.memories[0].memory).toBe("User works at Acme Corp") + }) + + it("exports memories as clean Markdown format", async () => { + const { fetchFn } = createMockFetch([sampleData]) + const markdown = await exportMemoriesAsMarkdown("user_123", { + baseUrl: "http://localhost:6767", + fetchFn, + }) + + expect(markdown).toContain("# Supermemory Backup: user_123") + expect(markdown).toContain("Total Memories:** 1") + expect(markdown).toContain("### 1. User works at Acme Corp") + expect(markdown).toContain("- **ID:** `mem_abc123`") + expect(markdown).toContain("- **Type:** Static") + expect(markdown).toContain("- **Source Documents:** `doc_1`, `doc_2`") + expect(markdown).toContain("- **Previous Revisions:** 1") + }) +}) diff --git a/packages/tools/src/index.ts b/packages/tools/src/index.ts index e7bef409..ba52db4c 100644 --- a/packages/tools/src/index.ts +++ b/packages/tools/src/index.ts @@ -10,3 +10,18 @@ export { DEFAULT_VALUES, getContainerTags, } from "./tools-shared" + +export { + listMemoriesRequest, + fetchAllMemories, + exportMemoriesAsJson, + exportMemoriesAsMarkdown, + type MemoryEntry, + type MemoryEntryHistory, + type MemoriesListResponse, + type ListMemoriesParams, + type ListMemoriesRequestOptions, + type ExportMemoriesOptions, + type MemoriesExportData, +} from "./shared" + diff --git a/packages/tools/src/shared/export-memories.ts b/packages/tools/src/shared/export-memories.ts new file mode 100644 index 00000000..4a806373 --- /dev/null +++ b/packages/tools/src/shared/export-memories.ts @@ -0,0 +1,230 @@ +const DEFAULT_BASE_URL = "https://api.supermemory.ai" +const FETCH_TIMEOUT_MS = 30_000 +const DEFAULT_PAGE_SIZE = 50 + +export interface MemoryEntryHistory { + id: string + memory: string + version: number + createdAt: string + updatedAt: string + parentMemoryId?: string | null + rootMemoryId?: string | null + isLatest?: boolean + isForgotten?: boolean +} + +export interface MemoryEntry { + id: string + memory: string + version: number + isLatest: boolean + isForgotten: boolean + isStatic?: boolean + isInference?: boolean + createdAt: string + updatedAt: string + sourceCount?: number + documentIds?: string[] + history?: MemoryEntryHistory[] +} + +export interface MemoriesListResponse { + memoryEntries: MemoryEntry[] + pagination: { + currentPage: number + limit: number + totalItems: number + totalPages: number + } +} + +export interface ListMemoriesParams { + containerTags: string[] + page?: number + limit?: number + sort?: "createdAt" | "updatedAt" + order?: "asc" | "desc" + filters?: unknown +} + +export interface ListMemoriesRequestOptions { + signal?: AbortSignal + fetchFn?: typeof fetch +} + +export interface ExportMemoriesOptions { + baseUrl?: string + apiKey?: string + pageSize?: number + maxPages?: number + includeForgotten?: boolean + signal?: AbortSignal + fetchFn?: typeof fetch +} + +export interface MemoriesExportData { + containerTag: string + exportedAt: string + totalCount: number + baseUrl: string + memories: MemoryEntry[] +} + +/** + * Fetch a page of memory entries directly from the `/v4/memories/list` endpoint. + * Works with both hosted platform (`https://api.supermemory.ai`) and + * self-hosted Supermemory server (`http://localhost:6767`). + */ +export async function listMemoriesRequest( + apiKey: string, + params: ListMemoriesParams, + baseUrl: string = DEFAULT_BASE_URL, + options?: ListMemoriesRequestOptions, +): Promise { + const customFetch = options?.fetchFn ?? fetch + const cleanBase = baseUrl.replace(/\/+$/, "") + const response = await customFetch(`${cleanBase}/v4/memories/list`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + "x-sm-source": "tools-export", + }, + body: JSON.stringify({ + containerTags: params.containerTags, + page: params.page ?? 1, + limit: params.limit ?? DEFAULT_PAGE_SIZE, + sort: params.sort ?? "createdAt", + order: params.order ?? "desc", + ...(params.filters ? { filters: params.filters } : {}), + }), + signal: options?.signal ?? AbortSignal.timeout(FETCH_TIMEOUT_MS), + }) + + if (!response.ok) { + const errorText = await response.text().catch(() => "Unknown error") + throw new Error( + `Supermemory list memories failed: ${response.status} ${response.statusText}. ${errorText}`, + ) + } + + return (await response.json()) as MemoriesListResponse +} + +/** + * Iteratively fetch all memory entries for a specific containerTag across all pages. + */ +export async function fetchAllMemories( + containerTag: string, + options?: ExportMemoriesOptions, +): Promise { + const baseUrl = options?.baseUrl ?? DEFAULT_BASE_URL + const apiKey = options?.apiKey ?? "" + const limit = options?.pageSize ?? DEFAULT_PAGE_SIZE + const maxPages = options?.maxPages ?? 1000 + const includeForgotten = options?.includeForgotten ?? false + + const allMemories: MemoryEntry[] = [] + let currentPage = 1 + let hasMore = true + + while (hasMore && currentPage <= maxPages) { + const response = await listMemoriesRequest( + apiKey, + { + containerTags: [containerTag], + page: currentPage, + limit, + sort: "createdAt", + order: "desc", + }, + baseUrl, + { + signal: options?.signal, + fetchFn: options?.fetchFn, + }, + ) + + const entries = response.memoryEntries ?? [] + for (const entry of entries) { + if (!includeForgotten && entry.isForgotten) { + continue + } + allMemories.push(entry) + } + + const totalPages = response.pagination?.totalPages ?? 1 + if (currentPage >= totalPages || entries.length === 0) { + hasMore = false + } else { + currentPage++ + } + } + + return allMemories +} + +/** + * Export all memories for a container tag formatted as JSON with backup metadata. + */ +export async function exportMemoriesAsJson( + containerTag: string, + options?: ExportMemoriesOptions, +): Promise { + const memories = await fetchAllMemories(containerTag, options) + const exportData: MemoriesExportData = { + containerTag, + exportedAt: new Date().toISOString(), + totalCount: memories.length, + baseUrl: options?.baseUrl ?? DEFAULT_BASE_URL, + memories, + } + return JSON.stringify(exportData, null, 2) +} + +/** + * Export all memories for a container tag formatted as human-readable Markdown. + */ +export async function exportMemoriesAsMarkdown( + containerTag: string, + options?: ExportMemoriesOptions, +): Promise { + const memories = await fetchAllMemories(containerTag, options) + const lines: string[] = [ + `# Supermemory Backup: ${containerTag}`, + "", + `- **Exported At:** ${new Date().toISOString()}`, + `- **Total Memories:** ${memories.length}`, + `- **Server:** ${options?.baseUrl ?? DEFAULT_BASE_URL}`, + "", + "---", + "", + ] + + if (memories.length === 0) { + lines.push("_No memories found for this container tag._") + return lines.join("\n") + } + + for (let i = 0; i < memories.length; i++) { + const m = memories[i] + const status = m.isForgotten ? " [FORGOTTEN]" : "" + const type = m.isStatic ? "Static" : "Dynamic" + lines.push(`### ${i + 1}. ${m.memory}${status}`) + lines.push(`- **ID:** \`${m.id}\``) + lines.push(`- **Type:** ${type}`) + lines.push(`- **Version:** ${m.version}`) + lines.push(`- **Created:** ${m.createdAt}`) + lines.push(`- **Updated:** ${m.updatedAt}`) + if (m.documentIds && m.documentIds.length > 0) { + lines.push(`- **Source Documents:** ${m.documentIds.map((d) => `\`${d}\``).join(", ")}`) + } + if (m.history && m.history.length > 0) { + lines.push(`- **Previous Revisions:** ${m.history.length}`) + } + lines.push("") + } + + return lines.join("\n") +} diff --git a/packages/tools/src/shared/index.ts b/packages/tools/src/shared/index.ts index 602d9fe3..943e0c18 100644 --- a/packages/tools/src/shared/index.ts +++ b/packages/tools/src/shared/index.ts @@ -51,3 +51,19 @@ export { wrapMemoryContext, replaceMemoryContext, } from "./memory-context" + +// Memory listing and export +export { + listMemoriesRequest, + fetchAllMemories, + exportMemoriesAsJson, + exportMemoriesAsMarkdown, + type MemoryEntry, + type MemoryEntryHistory, + type MemoriesListResponse, + type ListMemoriesParams, + type ListMemoriesRequestOptions, + type ExportMemoriesOptions, + type MemoriesExportData, +} from "./export-memories" +