Support for uploading cline_messages.json task files (#126)

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

.github/workflows/CI.yml

@@ -36,7 +36,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       postgres:
-        image: postgres:15.4
+        image: postgres:17.5
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: password
@@ -48,6 +48,20 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+      clickhouse:
+        image: clickhouse/clickhouse-server
+        env:
+          CLICKHOUSE_DB: default
+          CLICKHOUSE_USER: default
+          CLICKHOUSE_PASSWORD: password
+        ports:
+          - 8123:8123
+          - 9000:9000
+        options: >-
+          --health-cmd "wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -72,10 +86,23 @@ jobs:
         run: pnpm --filter @roo-code-cloud/db db:test:push --force
         env:
           DATABASE_URL: postgres://postgres:password@localhost:5432/test
+      - name: Setup ClickHouse schema
+        run: |
+          # Wait for ClickHouse to be ready
+          curl --retry 30 --retry-delay 1 --retry-connrefused http://localhost:8123/ping
+
+          # Install clickhouse-client
+          sudo apt-get update
+          sudo apt-get install -y clickhouse-client
+
+          # Execute the schema file directly using clickhouse-client
+          clickhouse-client --host localhost --port 9000 --password password --multiquery < .docker/scripts/clickhouse/001-create-tables.sql
       - name: Run unit tests
         run: pnpm test
         env:
           DATABASE_URL: postgres://postgres:password@localhost:5432/test
+          CLICKHOUSE_URL: http://localhost:8123/default
+          CLICKHOUSE_PASSWORD: password
 
   check:
     needs: [build, test]

apps/web/package.json

@@ -52,6 +52,7 @@
     "next-intl": "^4.1.0",
     "next-themes": "^0.4.6",
     "next-typesafe-url": "^5.1.7",
+    "p-map": "^7.0.3",
     "pino": "^9.7.0",
     "pino-pretty": "^13.0.0",
     "postgres": "^3.4.7",

apps/web/src/app/api/events/backfill/__tests__/route.test.ts

@@ -0,0 +1,529 @@
+// pnpm test src/app/api/events/backfill/__tests__/route.test.ts
+
+import fs from 'fs';
+import path from 'path';
+import { NextRequest } from 'next/server';
+
+import { authorizeApi } from '@/actions/auth';
+import { analytics } from '@/lib/server';
+import type { ApiAuthResult } from '@/types';
+
+import { POST } from '../route';
+
+vi.mock('@/actions/auth', () => ({
+  authorizeApi: vi.fn(),
+}));
+
+const mockAuthorizeApi = vi.mocked(authorizeApi);
+
+describe('/api/events/backfill', () => {
+  const testProperties = {
+    appName: 'test-app',
+    appVersion: '1.0.0',
+    vscodeVersion: '1.80.0',
+    platform: 'darwin',
+    editorName: 'vscode',
+    language: 'typescript',
+    mode: 'code',
+    provider: 'anthropic',
+    model: 'claude-3-sonnet',
+    sessionId: 'test-session-123',
+    isSubtask: false,
+  };
+
+  afterEach(async () => {
+    // Clean up any test data from the analytics database.
+    try {
+      await analytics.command({
+        query: `DELETE FROM messages WHERE taskId LIKE 'test-%'`,
+      });
+    } catch {
+      // Ignore cleanup errors.
+    }
+  });
+
+  it('should successfully process a backfill request with valid file upload', async () => {
+    mockAuthorizeApi.mockResolvedValue({
+      success: true,
+      userId: 'test-user-id',
+      orgId: 'test-org-id',
+      userType: 'user',
+      orgRole: 'admin',
+    } as unknown as ApiAuthResult);
+
+    const messages = [
+      {
+        ts: 1750702747687,
+        type: 'say' as const,
+        say: 'text' as const,
+        text: 'Test message',
+        images: [],
+      },
+      {
+        ts: 1750702747696,
+        type: 'say' as const,
+        say: 'api_req_started' as const,
+        text: 'API request started',
+        images: [],
+      },
+    ];
+
+    const fileContent = JSON.stringify(messages);
+    const file = new File([fileContent], 'test-messages.json', {
+      type: 'application/json',
+    });
+
+    const formData = new FormData();
+    formData.append('file', file);
+    formData.append('taskId', 'test-task-integration');
+    formData.append('properties', JSON.stringify(testProperties));
+
+    const request = new NextRequest(
+      'http://localhost:3000/api/events/backfill',
+      {
+        method: 'POST',
+        body: formData,
+      },
+    );
+
+    const response = await POST(request);
+    const responseData = await response.json();
+
+    expect(response.status).toBe(200);
+    expect(responseData.success).toBe(true);
+
+    const dbResults = await analytics.query({
+      query: `
+        SELECT
+          taskId,
+          mode,
+          text,
+          type as messageType,
+          say,
+          ts
+        FROM messages
+        WHERE taskId = 'test-task-integration'
+        ORDER BY ts ASC
+      `,
+      format: 'JSONEachRow',
+    });
+
+    const dbData = (await dbResults.json()) as Array<{
+      taskId: string;
+      mode: string;
+      text: string;
+      messageType: string;
+      say: string;
+      ts: number;
+    }>;
+
+    expect(dbData).toHaveLength(2);
+
+    expect(dbData[0]).toMatchObject({
+      taskId: 'test-task-integration',
+      mode: 'code',
+      text: 'Test message',
+      messageType: 'say',
+      say: 'text',
+    });
+
+    expect(dbData[1]).toMatchObject({
+      taskId: 'test-task-integration',
+      mode: 'code',
+      text: 'API request started',
+      messageType: 'say',
+      say: 'api_req_started',
+    });
+
+    expect(typeof dbData[0]?.ts).toBe('number');
+    expect(typeof dbData[1]?.ts).toBe('number');
+  });
+
+  it('should successfully process a backfill request using task.json file', async () => {
+    mockAuthorizeApi.mockResolvedValue({
+      success: true,
+      userId: 'test-user-id',
+      orgId: 'test-org-id',
+      userType: 'user',
+      orgRole: 'admin',
+    } as unknown as ApiAuthResult);
+
+    const taskJsonPath = path.join(__dirname, 'task.json');
+    const taskJsonContent = fs.readFileSync(taskJsonPath, 'utf-8');
+    const messages = JSON.parse(taskJsonContent);
+
+    const file = new File([taskJsonContent], 'task.json', {
+      type: 'application/json',
+    });
+
+    const formData = new FormData();
+    formData.append('file', file);
+    formData.append('taskId', 'test-task-from-file');
+    formData.append('properties', JSON.stringify(testProperties));
+
+    const request = new NextRequest(
+      'http://localhost:3000/api/events/backfill',
+      {
+        method: 'POST',
+        body: formData,
+      },
+    );
+
+    const response = await POST(request);
+    const responseData = await response.json();
+
+    expect(response.status).toBe(200);
+    expect(responseData.success).toBe(true);
+
+    const dbResults = await analytics.query({
+      query: `
+        SELECT
+          taskId,
+          mode,
+          text,
+          type as messageType,
+          say,
+          timestamp
+        FROM messages
+        WHERE taskId = 'test-task-from-file'
+        ORDER BY timestamp ASC
+      `,
+      format: 'JSONEachRow',
+    });
+
+    const dbData = (await dbResults.json()) as Array<{
+      taskId: string;
+      mode: string;
+      text: string;
+      messageType: string;
+      say: string;
+      timestamp: number;
+    }>;
+
+    expect(dbData).toHaveLength(messages.length);
+
+    const textMessage = dbData.find((msg) => msg.say === 'text');
+    expect(textMessage).toMatchObject({
+      taskId: 'test-task-from-file',
+      mode: 'code',
+      text: 'Usql psql, how can I get all the namespaces?',
+      messageType: 'say',
+      say: 'text',
+    });
+
+    const apiMessage = dbData.find((msg) => msg.say === 'api_req_started');
+    expect(apiMessage).toMatchObject({
+      taskId: 'test-task-from-file',
+      mode: 'code',
+      messageType: 'say',
+      say: 'api_req_started',
+    });
+
+    expect(textMessage?.timestamp).toBeTypeOf('number');
+    expect(apiMessage?.timestamp).toBeTypeOf('number');
+
+    const messageTypes = dbData.map((item) => item.say);
+    expect(messageTypes).toContain('text');
+    expect(messageTypes).toContain('api_req_started');
+    expect(messageTypes).toContain('checkpoint_saved');
+    expect(messageTypes).toContain('reasoning');
+    expect(messageTypes).toContain('completion_result');
+  });
+
+  it('should return 401 if authentication fails', async () => {
+    mockAuthorizeApi.mockResolvedValue({
+      success: false,
+      error: 'Authentication failed',
+    });
+
+    const formData = new FormData();
+    formData.append('file', new File(['[]'], 'test.json'));
+    formData.append('properties', JSON.stringify(testProperties));
+    formData.append('taskId', 'test');
+
+    const request = new NextRequest(
+      'http://localhost:3000/api/events/backfill',
+      {
+        method: 'POST',
+        body: formData,
+      },
+    );
+
+    const response = await POST(request);
+    const responseData = await response.json();
+
+    expect(response.status).toBe(401);
+    expect(responseData.success).toBe(false);
+    expect(responseData.error).toBe('Authentication required');
+
+    const dbResults = await analytics.query({
+      query: `SELECT COUNT() as count FROM messages WHERE taskId = 'test'`,
+      format: 'JSONEachRow',
+    });
+
+    const dbData = (await dbResults.json()) as Array<{ count: string }>;
+    expect(dbData[0]?.count).toBe('0');
+  });
+
+  it('should return 400 if no file is provided', async () => {
+    mockAuthorizeApi.mockResolvedValue({
+      success: true,
+      userId: 'test-user-id',
+      orgId: 'test-org-id',
+      userType: 'user',
+      orgRole: 'admin',
+    } as unknown as ApiAuthResult);
+
+    const formData = new FormData();
+    formData.append('taskId', 'test-task-id');
+    formData.append('properties', JSON.stringify(testProperties));
+
+    const request = new NextRequest(
+      'http://localhost:3000/api/events/backfill',
+      {
+        method: 'POST',
+        body: formData,
+      },
+    );
+
+    const response = await POST(request);
+    const responseData = await response.json();
+
+    expect(response.status).toBe(400);
+    expect(responseData.success).toBe(false);
+    expect(responseData.error).toBe('No file provided');
+  });
+
+  it('should return 400 if taskId is missing', async () => {
+    mockAuthorizeApi.mockResolvedValue({
+      success: true,
+      userId: 'test-user-id',
+      orgId: 'test-org-id',
+      userType: 'user',
+      orgRole: 'admin',
+    } as unknown as ApiAuthResult);
+
+    const formData = new FormData();
+    formData.append('file', new File(['[]'], 'test.json'));
+    formData.append('properties', JSON.stringify(testProperties));
+
+    const request = new NextRequest(
+      'http://localhost:3000/api/events/backfill',
+      {
+        method: 'POST',
+        body: formData,
+      },
+    );
+
+    const response = await POST(request);
+    const responseData = await response.json();
+
+    expect(response.status).toBe(400);
+    expect(responseData.success).toBe(false);
+    expect(responseData.error).toBe('taskId is required');
+  });
+
+  it('should return 400 for invalid JSON content', async () => {
+    mockAuthorizeApi.mockResolvedValue({
+      success: true,
+      userId: 'test-user-id',
+      orgId: 'test-org-id',
+      userType: 'user',
+      orgRole: 'admin',
+    } as unknown as ApiAuthResult);
+
+    const formData = new FormData();
+    formData.append('file', new File(['invalid json content'], 'test.json'));
+    formData.append('taskId', 'test-task-id');
+    formData.append('properties', JSON.stringify(testProperties));
+
+    const request = new NextRequest(
+      'http://localhost:3000/api/events/backfill',
+      {
+        method: 'POST',
+        body: formData,
+      },
+    );
+
+    const response = await POST(request);
+    const responseData = await response.json();
+
+    expect(response.status).toBe(400);
+    expect(responseData.success).toBe(false);
+    expect(responseData.error).toBe('Invalid JSON file format');
+  });
+
+  it('should return 400 for empty message array', async () => {
+    mockAuthorizeApi.mockResolvedValue({
+      success: true,
+      userId: 'test-user-id',
+      orgId: 'test-org-id',
+      userType: 'user',
+      orgRole: 'admin',
+    } as unknown as ApiAuthResult);
+
+    const formData = new FormData();
+    formData.append('file', new File(['[]'], 'test.json'));
+    formData.append('properties', JSON.stringify(testProperties));
+    formData.append('taskId', 'test-task-id');
+
+    const request = new NextRequest(
+      'http://localhost:3000/api/events/backfill',
+      {
+        method: 'POST',
+        body: formData,
+      },
+    );
+
+    const response = await POST(request);
+    const responseData = await response.json();
+
+    expect(response.status).toBe(400);
+    expect(responseData.success).toBe(false);
+    expect(responseData.error).toBe('File contains no messages');
+  });
+
+  it('should extract mode from individual messages when mode slug is present', async () => {
+    mockAuthorizeApi.mockResolvedValue({
+      success: true,
+      userId: 'test-user-id',
+      orgId: 'test-org-id',
+      userType: 'user',
+      orgRole: 'admin',
+    } as unknown as ApiAuthResult);
+
+    const messages = [
+      {
+        ts: 1750702747687,
+        type: 'say' as const,
+        say: 'text' as const,
+        text: 'Starting task in <slug>debug</slug> mode',
+        images: [],
+      },
+      {
+        ts: 1750702747696,
+        type: 'say' as const,
+        say: 'api_req_started' as const,
+        text: 'API request started',
+        images: [],
+      },
+      {
+        ts: 1750702747705,
+        type: 'say' as const,
+        say: 'text' as const,
+        text: 'Switching to <slug>architect</slug> mode',
+        images: [],
+      },
+    ];
+
+    const fileContent = JSON.stringify(messages);
+    const file = new File([fileContent], 'test-messages.json', {
+      type: 'application/json',
+    });
+
+    const formData = new FormData();
+    formData.append('file', file);
+    formData.append('taskId', 'test-task-mode-extraction');
+    formData.append('properties', JSON.stringify(testProperties));
+
+    const request = new NextRequest(
+      'http://localhost:3000/api/events/backfill',
+      {
+        method: 'POST',
+        body: formData,
+      },
+    );
+
+    const response = await POST(request);
+    const responseData = await response.json();
+
+    expect(response.status).toBe(200);
+    expect(responseData.success).toBe(true);
+
+    const dbResults = await analytics.query({
+      query: `
+        SELECT
+          taskId,
+          mode,
+          text,
+          type as messageType,
+          say
+        FROM messages
+        WHERE taskId = 'test-task-mode-extraction'
+        ORDER BY ts ASC
+      `,
+      format: 'JSONEachRow',
+    });
+
+    const dbData = (await dbResults.json()) as Array<{
+      taskId: string;
+      mode: string;
+      text: string;
+      messageType: string;
+      say: string;
+    }>;
+
+    expect(dbData).toHaveLength(3);
+
+    expect(dbData[0]).toMatchObject({
+      taskId: 'test-task-mode-extraction',
+      mode: 'debug',
+      text: 'Starting task in <slug>debug</slug> mode',
+      messageType: 'say',
+      say: 'text',
+    });
+
+    expect(dbData[1]).toMatchObject({
+      taskId: 'test-task-mode-extraction',
+      mode: 'debug',
+      text: 'API request started',
+      messageType: 'say',
+      say: 'api_req_started',
+    });
+
+    expect(dbData[2]).toMatchObject({
+      taskId: 'test-task-mode-extraction',
+      mode: 'architect',
+      text: 'Switching to <slug>architect</slug> mode',
+      messageType: 'say',
+      say: 'text',
+    });
+  });
+
+  it('should handle invalid ClineMessage schema', async () => {
+    mockAuthorizeApi.mockResolvedValue({
+      success: true,
+      userId: 'test-user-id',
+      orgId: 'test-org-id',
+      userType: 'user',
+      orgRole: 'admin',
+    } as unknown as ApiAuthResult);
+
+    const invalidMessages = [{ text: 'Invalid message' }];
+
+    const formData = new FormData();
+
+    formData.append(
+      'file',
+      new File([JSON.stringify(invalidMessages)], 'test.json'),
+    );
+
+    formData.append('taskId', 'test-task-invalid');
+    formData.append('properties', JSON.stringify(testProperties));
+
+    const request = new NextRequest(
+      'http://localhost:3000/api/events/backfill',
+      {
+        method: 'POST',
+        body: formData,
+      },
+    );
+
+    const response = await POST(request);
+    const responseData = await response.json();
+
+    expect(response.status).toBe(400);
+    expect(responseData.success).toBe(false);
+    expect(responseData.error).toContain('Invalid file content');
+  });
+});

apps/web/src/app/api/events/backfill/route.ts

@@ -0,0 +1,135 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { v4 as uuidv4 } from 'uuid';
+import { z } from 'zod';
+import pMap from 'p-map';
+
+import {
+  type ClineMessage,
+  TelemetryEventName,
+  clineMessageSchema,
+  telemetryPropertiesSchema,
+} from '@roo-code/types';
+
+import { authorizeApi } from '@/actions/auth';
+import { captureEvent } from '@/actions/analytics';
+
+export async function POST(request: NextRequest) {
+  const authResult = await authorizeApi(request);
+
+  if (!authResult.success) {
+    return NextResponse.json(
+      { success: false, error: 'Authentication required' },
+      { status: 401 },
+    );
+  }
+
+  const { userId, orgId } = authResult;
+
+  try {
+    const formData = await request.formData();
+    const taskId = formData.get('taskId') as string;
+
+    if (!taskId) {
+      return NextResponse.json(
+        { success: false, error: 'taskId is required' },
+        { status: 400 },
+      );
+    }
+
+    const properties = telemetryPropertiesSchema.parse(
+      JSON.parse(formData.get('properties') as string),
+    );
+
+    const file = formData.get('file') as File;
+
+    if (!file) {
+      return NextResponse.json(
+        { success: false, error: 'No file provided' },
+        { status: 400 },
+      );
+    }
+
+    const fileContent = await file.text();
+    let messages: ClineMessage[];
+
+    try {
+      const parsedContent = JSON.parse(fileContent);
+
+      const messagesResult = z
+        .array(clineMessageSchema)
+        .safeParse(parsedContent);
+
+      if (!messagesResult.success) {
+        return NextResponse.json(
+          {
+            success: false,
+            error: `Invalid file content: ${messagesResult.error.message}`,
+          },
+          { status: 400 },
+        );
+      }
+
+      messages = messagesResult.data;
+    } catch (_error) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'Invalid JSON file format',
+        },
+        { status: 400 },
+      );
+    }
+
+    if (messages.length === 0) {
+      return NextResponse.json(
+        { success: false, error: 'File contains no messages' },
+        { status: 400 },
+      );
+    }
+
+    const defaultMode = extractMode(messages[0]?.text) || properties.mode;
+
+    await pMap(
+      messages,
+      async (message) => {
+        const id = uuidv4();
+        const timestamp = Math.round(message.ts / 1000);
+        const mode = extractMode(message.text) || defaultMode;
+
+        const event = {
+          type: TelemetryEventName.TASK_MESSAGE as const,
+          properties: { taskId, message, ...properties, mode },
+        };
+
+        await captureEvent({ id, orgId, userId, timestamp, event });
+      },
+      { concurrency: 10 },
+    );
+
+    return NextResponse.json({ success: true });
+  } catch (error) {
+    console.error(error);
+
+    return NextResponse.json(
+      {
+        success: false,
+        error: error instanceof Error ? error.message : 'Unknown error',
+      },
+      { status: 500 },
+    );
+  }
+}
+
+/**
+ * Extracts the mode from a message text if it contains a <slug>mode</slug> pattern.
+ * @param text - The message text to parse
+ * @returns The extracted mode or null if no mode slug is found
+ */
+function extractMode(text: string | undefined): string | null {
+  if (!text) {
+    return null;
+  }
+
+  const modeMatch = text.match(/<slug>(.+?)<\/slug>/);
+  return modeMatch && modeMatch[1] ? modeMatch[1] : null;
+}

pnpm-lock.yaml

@@ -260,6 +260,9 @@ importers:
       next-typesafe-url:
         specifier: ^5.1.7
         version: 5.1.7(next@15.3.3(@babel/core@7.27.3)(@opentelemetry/api@1.9.0)(@playwright/test@1.52.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0)(zod@3.25.41)
+      p-map:
+        specifier: ^7.0.3
+        version: 7.0.3
       pino:
         specifier: ^9.7.0
         version: 9.7.0
@@ -5121,6 +5124,10 @@ packages:
     resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==}
     engines: {node: '>=10'}
 
+  p-map@7.0.3:
+    resolution: {integrity: sha512-VkndIv2fIB99swvQoA65bm+fsmt6UNdGeIB0oxBs+WhAhdh08QA04JXpI7rbB9r08/nkbysKoya9rtDERYOYMA==}
+    engines: {node: '>=18'}
+
   p-timeout@6.1.4:
     resolution: {integrity: sha512-MyIV3ZA/PmyBN/ud8vV9XzwTrNtR4jFrObymZYnZqMmW0zA8Z17vnT0rBgFE/TlohB+YCHqXMgZzb3Csp49vqg==}
     engines: {node: '>=14.16'}
@@ -11552,6 +11559,8 @@ snapshots:
     dependencies:
       p-limit: 3.1.0
 
+  p-map@7.0.3: {}
+
   p-timeout@6.1.4: {}
 
   p-try@2.2.0: {}