From b392bc7d1b294a5b3dd2407d060ddcf229481d6f Mon Sep 17 00:00:00 2001 From: MaheshtheDev <38828053+MaheshtheDev@users.noreply.github.com> Date: Tue, 29 Sep 2026 06:08:45 +0000 Subject: [PATCH] Instrument MCP server with metadata-only PostHog analytics (#1712) ## Summary - Instrument the MCP v2 server with the pinned PostHog MCP Analytics SDK and replace the custom `mcp_tool_executed` wrapper with standard `$mcp_*` events. - Send only allowlisted metadata, disable schema injection and exception autocapture, and use personless user IDs with person-profile processing disabled. - Deliver events through immediate capture guarded by Cloudflare `waitUntil`. ## Verification - The initial implementation passed the MCP typecheck, existing unit suite, Biome, and Wrangler dry-run bundle. - A local `who_am_i` MCP call returned successfully and emitted a metadata-only `$mcp_tool_call` on the initial implementation. - All five checks passed on the final `54a2384` head, including the Cloudflare MCP build. Actual PostHog ingestion is not verified yet; this workspace still needs a PostHog project token and authenticated Supermemory MCP credential. --- apps/mcp/package.json | 1 + apps/mcp/src/server/analytics.test.ts | 169 ---------------- apps/mcp/src/server/analytics.ts | 274 ++++++++------------------ apps/mcp/src/server/server.ts | 16 +- bun.lock | 7 + 5 files changed, 94 insertions(+), 373 deletions(-) delete mode 100644 apps/mcp/src/server/analytics.test.ts diff --git a/apps/mcp/package.json b/apps/mcp/package.json index d33880d5..e219c276 100644 --- a/apps/mcp/package.json +++ b/apps/mcp/package.json @@ -26,6 +26,7 @@ "@modelcontextprotocol/sdk": "1.30.0", "@modelcontextprotocol/server": "2.0.0", "@phosphor-icons/react": "^2.1.10", + "@posthog/mcp": "0.21.0", "@radix-ui/react-popover": "^1.1.15", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tooltip": "^1.2.7", diff --git a/apps/mcp/src/server/analytics.test.ts b/apps/mcp/src/server/analytics.test.ts deleted file mode 100644 index ac016419..00000000 --- a/apps/mcp/src/server/analytics.test.ts +++ /dev/null @@ -1,169 +0,0 @@ -import type { McpServer, ServerContext } from "@modelcontextprotocol/server" -import { describe, expect, it, vi } from "vitest" -import { z } from "zod" -import { - createTrackedToolServer, - posthogEventForToolExecution, - type McpToolAnalytics, -} from "./analytics" - -function testServer() { - let callback: ((...args: unknown[]) => unknown) | undefined - const registerTool = vi.fn( - ( - _name: string, - _config: unknown, - handler: (...args: unknown[]) => unknown, - ) => { - callback = handler - return {} - }, - ) - - return { - server: { registerTool } as unknown as McpServer, - invoke(...args: unknown[]) { - if (!callback) throw new Error("Tool was not registered") - return callback(...args) - }, - } -} - -const context = { - mcpReq: { envelope: {} }, -} as unknown as ServerContext - -describe("MCP tool analytics", () => { - it("records sanitized completion metadata without tool content", async () => { - const harness = testServer() - const record = vi.fn() - const analytics: McpToolAnalytics = { record } - const server = createTrackedToolServer(harness.server, analytics, () => ({ - name: "claude", - version: "1.2.3", - })) - - server.registerTool( - "search_memory", - { - inputSchema: z.object({ query: z.string(), containerTag: z.string() }), - }, - async () => ({ - content: [{ type: "text" as const, text: "secret result" }], - }), - ) - - await harness.invoke( - { query: "private query", containerTag: "private-space" }, - context, - ) - - expect(record).toHaveBeenCalledOnce() - expect(record).toHaveBeenCalledWith( - expect.objectContaining({ - toolName: "search_memory", - surface: "model_tool", - outcome: "success", - spaceExplicit: true, - client: { name: "claude", version: "1.2.3" }, - }), - ) - expect(JSON.stringify(record.mock.calls[0])).not.toContain("private query") - expect(JSON.stringify(record.mock.calls[0])).not.toContain("private-space") - expect(JSON.stringify(record.mock.calls[0])).not.toContain("secret result") - }) - - it("treats returned MCP errors as failed executions", async () => { - const harness = testServer() - const record = vi.fn() - const server = createTrackedToolServer( - harness.server, - { record }, - () => null, - ) - - server.registerTool( - "save-memory", - { inputSchema: z.object({}) }, - async () => ({ - content: [{ type: "text" as const, text: "failed" }], - isError: true, - }), - ) - - await harness.invoke({}, context) - - expect(record).toHaveBeenCalledWith( - expect.objectContaining({ - surface: "app_action", - outcome: "error", - errorType: "tool_result", - }), - ) - }) - - it("records thrown error categories and preserves the rejection", async () => { - const harness = testServer() - const record = vi.fn() - const server = createTrackedToolServer( - harness.server, - { record }, - () => null, - ) - - server.registerTool( - "fetch-graph-data", - { inputSchema: z.object({}) }, - async () => { - throw new TypeError("sensitive failure") - }, - ) - - await expect(harness.invoke({}, context)).rejects.toThrow( - "sensitive failure", - ) - expect(record).toHaveBeenCalledWith( - expect.objectContaining({ - surface: "app_internal", - outcome: "error", - errorType: "TypeError", - }), - ) - expect(JSON.stringify(record.mock.calls[0])).not.toContain( - "sensitive failure", - ) - }) - - it("uses the existing user identity and company group", () => { - const event = posthogEventForToolExecution( - { - userId: "user_123", - organizationId: "org_123", - oauthClientId: "client_123", - }, - { - toolName: "guided-save", - surface: "app_launcher", - outcome: "success", - durationMs: 42, - spaceExplicit: false, - }, - ) - - expect(event).toEqual({ - distinctId: "user_123", - event: "mcp_tool_executed", - groups: { company: "org_123" }, - properties: { - app: "mcp", - tool_name: "guided-save", - outcome: "success", - duration_ms: 42, - mcp_runtime: "stateless", - mcp_surface: "app_launcher", - space_explicit: false, - oauth_client_id: "client_123", - }, - }) - }) -}) diff --git a/apps/mcp/src/server/analytics.ts b/apps/mcp/src/server/analytics.ts index e92cd82a..7ceaa937 100644 --- a/apps/mcp/src/server/analytics.ts +++ b/apps/mcp/src/server/analytics.ts @@ -1,217 +1,109 @@ -import type { McpServer, ServerContext } from "@modelcontextprotocol/server" +import type { McpServer } from "@modelcontextprotocol/server" +import { instrument, type MCPAnalyticsOptions } from "@posthog/mcp" import { PostHog } from "posthog-node" import type { ActorContext, ServerEnv } from "./types" const DEFAULT_POSTHOG_HOST = "https://us.i.posthog.com" - -export type McpToolSurface = - | "model_tool" - | "app_launcher" - | "app_action" - | "app_internal" - -export type McpToolOutcome = "success" | "error" - -export interface McpToolExecution { - toolName: string - surface: McpToolSurface - outcome: McpToolOutcome - durationMs: number - spaceExplicit: boolean - client?: { name: string; version?: string } - errorType?: string -} - -export interface McpToolAnalytics { - record(execution: McpToolExecution): void -} +// Must match the API's personless id hashing so MCP and API events share one id. +const PERSONLESS_DISTINCT_ID_NAMESPACE = "supermemory-posthog-personless-v1" +const MCP_EVENT_PROPERTIES = [ + "$mcp_source", + "$mcp_server_name", + "$mcp_server_version", + "$mcp_tool_name", + "$mcp_duration_ms", + "$mcp_is_error", + "$mcp_client_name", + "$mcp_client_version", + "$mcp_protocol_version", + "$mcp_listed_tool_names", + "$groups", +] as const export type WaitUntil = (promise: Promise) => void -type ClientInfoResolver = ( - context: ServerContext, -) => { name: string; version?: string } | null - -const TOOL_SURFACES: Record = { - search_memory: "model_tool", - get_profile: "model_tool", - list_documents: "model_tool", - get_document: "model_tool", - list_memories: "model_tool", - list_spaces: "model_tool", - who_am_i: "model_tool", - add_memory: "model_tool", - "select-space": "app_launcher", - "memory-graph": "app_launcher", - "guided-save": "app_launcher", - "upload-file": "app_launcher", - "set-active-tag": "app_action", - "save-memory": "app_action", - "prepare-file-upload": "app_action", - "fetch-graph-data": "app_internal", +async function personlessDistinctId(userId: string): Promise { + const input = new TextEncoder().encode( + `${PERSONLESS_DISTINCT_ID_NAMESPACE}:${userId}`, + ) + const digest = await crypto.subtle.digest("SHA-256", input) + const hash = Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, "0"), + ).join("") + return `personless_${hash}` } -let posthogConfig: - | { - apiKey: string - host: string - client: PostHog - } - | undefined - -function posthogClient(apiKey: string, host: string): PostHog { - if (posthogConfig?.apiKey === apiKey && posthogConfig.host === host) { - return posthogConfig.client +class ImmediateMcpPostHog extends PostHog { + constructor( + apiKey: string, + host: string, + private readonly waitUntil: WaitUntil, + ) { + super(apiKey, { host }) } - const client = new PostHog(apiKey, { - host, - flushAt: 1, - flushInterval: 0, - }) - posthogConfig = { apiKey, host, client } - return client + override capture(event: Parameters[0]): void { + try { + this.waitUntil( + this.captureImmediate(event).catch((error) => + console.error("PostHog MCP tracking error:", error), + ), + ) + } catch (error) { + console.error("PostHog MCP tracking error:", error) + } + } } -export function posthogEventForToolExecution( - actor: Pick, - execution: McpToolExecution, -) { +const metadataOnlyMcpEvent: NonNullable = ( + event, +) => { + if ( + !["$mcp_tool_call", "$mcp_tools_list", "$mcp_initialize"].includes( + event.event, + ) + ) { + return null + } + return { - distinctId: actor.userId, - event: "mcp_tool_executed", - groups: { company: actor.organizationId }, + ...event, properties: { - app: "mcp", - tool_name: execution.toolName, - outcome: execution.outcome, - duration_ms: execution.durationMs, - mcp_runtime: "stateless", - mcp_surface: execution.surface, - space_explicit: execution.spaceExplicit, - ...(execution.client - ? { - mcp_client_name: execution.client.name, - ...(execution.client.version - ? { mcp_client_version: execution.client.version } - : {}), - } - : {}), - ...(actor.oauthClientId ? { oauth_client_id: actor.oauthClientId } : {}), - ...(execution.errorType ? { error_type: execution.errorType } : {}), + ...Object.fromEntries( + MCP_EVENT_PROPERTIES.filter( + (key) => event.properties[key] !== undefined, + ).map((key) => [key, event.properties[key]]), + ), + $process_person_profile: false, }, } } -export function createPosthogAnalytics( +export function instrumentPosthogMcp( + server: McpServer, env: ServerEnv, actor: ActorContext, waitUntil: WaitUntil, -): McpToolAnalytics { - const apiKey = env.POSTHOG_API_KEY - if (!apiKey) return { record: () => undefined } +): void { + if (!env.POSTHOG_API_KEY) return - const client = posthogClient(apiKey, env.POSTHOG_HOST || DEFAULT_POSTHOG_HOST) - - return { - record(execution) { - try { - const capture = client - .captureImmediate(posthogEventForToolExecution(actor, execution)) - .catch((error) => console.error("PostHog MCP tracking error:", error)) - waitUntil(capture) - } catch (error) { - console.error("PostHog MCP tracking error:", error) - } + instrument( + server, + new ImmediateMcpPostHog( + env.POSTHOG_API_KEY, + env.POSTHOG_HOST || DEFAULT_POSTHOG_HOST, + waitUntil, + ), + { + identify: async () => ({ + distinctId: await personlessDistinctId(actor.userId), + groups: { company: actor.organizationId }, + }), + context: false, + captureModel: false, + enableConversationId: false, + enableExceptionAutocapture: false, + beforeSend: metadataOnlyMcpEvent, }, - } -} - -function spaceWasExplicit(value: unknown): boolean { - if (!value || typeof value !== "object") return false - const containerTag = Reflect.get(value, "containerTag") - return typeof containerTag === "string" && containerTag.trim().length > 0 -} - -function isErrorResult(value: unknown): boolean { - return ( - !!value && - typeof value === "object" && - Reflect.get(value, "isError") === true ) } - -function thrownErrorType(error: unknown): string { - if (error instanceof Error && error.name) return error.name - if (error && typeof error === "object") { - const status = Reflect.get(error, "status") - if (typeof status === "number") return `http_${status}` - } - return "unknown" -} - -function safeRecord(analytics: McpToolAnalytics, execution: McpToolExecution) { - try { - analytics.record(execution) - } catch (error) { - console.error("MCP analytics recording error:", error) - } -} - -export function createTrackedToolServer( - server: McpServer, - analytics: McpToolAnalytics, - getClientInfo: ClientInfoResolver, -): Pick { - const registerTool = (( - name: string, - config: unknown, - handler: (...args: unknown[]) => unknown, - ) => { - const trackedHandler = async (...callbackArgs: unknown[]) => { - const startedAt = performance.now() - const input = callbackArgs.length > 1 ? callbackArgs[0] : undefined - const context = callbackArgs.at(-1) as ServerContext - - const finish = (outcome: McpToolOutcome, errorType?: string) => { - let client: ReturnType = null - try { - client = getClientInfo(context) - } catch { - // Client metadata is optional and must never affect a tool call. - } - - safeRecord(analytics, { - toolName: name, - surface: TOOL_SURFACES[name] ?? "model_tool", - outcome, - durationMs: Math.max(0, Math.round(performance.now() - startedAt)), - spaceExplicit: spaceWasExplicit(input), - ...(client ? { client } : {}), - ...(errorType ? { errorType } : {}), - }) - } - - try { - const result = await handler(...callbackArgs) - if (isErrorResult(result)) { - finish("error", "tool_result") - } else { - finish("success") - } - return result - } catch (error) { - finish("error", thrownErrorType(error)) - throw error - } - } - - return Reflect.apply(server.registerTool, server, [ - name, - config, - trackedHandler, - ]) - }) as McpServer["registerTool"] - - return { registerTool } -} diff --git a/apps/mcp/src/server/server.ts b/apps/mcp/src/server/server.ts index 5498569c..8c0857ed 100644 --- a/apps/mcp/src/server/server.ts +++ b/apps/mcp/src/server/server.ts @@ -3,11 +3,7 @@ import { McpServer, type ServerContext, } from "@modelcontextprotocol/server" -import { - createPosthogAnalytics, - createTrackedToolServer, - type WaitUntil, -} from "./analytics" +import { instrumentPosthogMcp, type WaitUntil } from "./analytics" import { fetchSession } from "./auth" import { DEFAULT_PROJECT_ID, SupermemoryClient } from "./client" import { registerContextPrompt } from "./prompts/context" @@ -59,6 +55,7 @@ export function createSupermemoryServer( }, { instructions: SERVER_INSTRUCTIONS }, ) + instrumentPosthogMcp(server, env, actor, waitUntil) const apiUrl = env.API_URL || DEFAULT_API_URL const spaceState = env.SPACE_STATE.getByName(spaceStateName(actor)) @@ -88,15 +85,8 @@ export function createSupermemoryServer( expiresAt, } } - const analytics = createPosthogAnalytics(env, actor, waitUntil) - const toolServer = createTrackedToolServer( - server, - analytics, - clientInfoFromContext, - ) - registerAllTools({ - server: toolServer, + server, actor, getClient, getSession: () => fetchSession(actor.bearerToken, apiUrl), diff --git a/bun.lock b/bun.lock index 3ad86ef7..a5c9b1d8 100644 --- a/bun.lock +++ b/bun.lock @@ -71,6 +71,7 @@ "@modelcontextprotocol/sdk": "1.30.0", "@modelcontextprotocol/server": "2.0.0", "@phosphor-icons/react": "^2.1.10", + "@posthog/mcp": "0.21.0", "@radix-ui/react-popover": "^1.1.15", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tooltip": "^1.2.7", @@ -1038,6 +1039,8 @@ "@posthog/core": ["@posthog/core@1.23.2", "", { "dependencies": { "cross-spawn": "^7.0.6" } }, "sha512-zTDdda9NuSHrnwSOfFMxX/pyXiycF4jtU1kTr8DL61dHhV+7LF6XF1ndRZZTuaGGbfbb/GJYkEsjEX9SXfNZeQ=="], + "@posthog/mcp": ["@posthog/mcp@0.21.0", "", { "dependencies": { "@posthog/core": "^1.55.2" }, "peerDependencies": { "@modelcontextprotocol/sdk": ">=1.26.0", "@modelcontextprotocol/server": ">=2.0.0", "posthog-node": "^5.0.0" }, "optionalPeers": ["@modelcontextprotocol/sdk", "@modelcontextprotocol/server"] }, "sha512-LhoQm2nUsXzGWgvxL4LHq1p2qC1B/FGoWxcRtMqQmirGQTCCfBNcNhlurX0J92X1iR7ZZuM7cZfkobdnQ43hvg=="], + "@posthog/types": ["@posthog/types@1.359.0", "", {}, "sha512-IcRRmDqDwXnwcB1ISg2NofvDXUwJKUjmIWuRqaEzD1sB+VyY7YX9R1pctXoZ4181cdjYG8Ed/g7fnpJMloZY/A=="], "@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="], @@ -4222,6 +4225,8 @@ "@poppinss/dumper/supports-color": ["supports-color@10.2.2", "", {}, "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g=="], + "@posthog/mcp/@posthog/core": ["@posthog/core@1.55.2", "", { "dependencies": { "@posthog/types": "^1.412.4" } }, "sha512-2mYxGbDmTLUs096VbcZqSWo0lrI48/BrX65r54NynxxFq409BOx9w73HxOAtOKZ1vblkdV6/GxQTe7KDcIQidg=="], + "@puppeteer/browsers/yargs": ["yargs@17.7.2", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w=="], "@radix-ui/react-alert-dialog/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], @@ -5094,6 +5099,8 @@ "@opentelemetry/otlp-exporter-base/@opentelemetry/otlp-transformer/@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@2.2.0", "", { "dependencies": { "@opentelemetry/core": "2.2.0", "@opentelemetry/resources": "2.2.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-xWQgL0Bmctsalg6PaXExmzdedSp3gyKV8mQBwK/j9VGdCDu2fmXIb2gAehBKbkXCpJ4HPkgv3QfoJWRT4dHWbw=="], + "@posthog/mcp/@posthog/core/@posthog/types": ["@posthog/types@1.412.4", "", {}, "sha512-Q7lV9O9TbLngjOYw1ucm3bS3tn48xb1B4ZQdDy8gv7yfKe99hCTeWYG4xZ36nJM80VxpTF9TriiJt5hngDOkBg=="], + "@puppeteer/browsers/yargs/cliui": ["cliui@8.0.1", "", { "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" } }, "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ=="], "@puppeteer/browsers/yargs/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="],