diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index dcc84dde52..cc401439b8 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-latest services: postgres: - image: postgres:15.4 + image: postgres:17.5 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: password @@ -48,6 +48,20 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + clickhouse: + image: clickhouse/clickhouse-server + env: + CLICKHOUSE_DB: default + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: password + ports: + - 8123:8123 + - 9000:9000 + options: >- + --health-cmd "wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - name: Checkout code uses: actions/checkout@v4 @@ -72,10 +86,23 @@ jobs: run: pnpm --filter @roo-code-cloud/db db:test:push --force env: DATABASE_URL: postgres://postgres:password@localhost:5432/test + - name: Setup ClickHouse schema + run: | + # Wait for ClickHouse to be ready + curl --retry 30 --retry-delay 1 --retry-connrefused http://localhost:8123/ping + + # Install clickhouse-client + sudo apt-get update + sudo apt-get install -y clickhouse-client + + # Execute the schema file directly using clickhouse-client + clickhouse-client --host localhost --port 9000 --password password --multiquery < .docker/scripts/clickhouse/001-create-tables.sql - name: Run unit tests run: pnpm test env: DATABASE_URL: postgres://postgres:password@localhost:5432/test + CLICKHOUSE_URL: http://localhost:8123/default + CLICKHOUSE_PASSWORD: password check: needs: [build, test] diff --git a/apps/web/package.json b/apps/web/package.json index 0b8c576aa3..cf38c7d64c 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -52,6 +52,7 @@ "next-intl": "^4.1.0", "next-themes": "^0.4.6", "next-typesafe-url": "^5.1.7", + "p-map": "^7.0.3", "pino": "^9.7.0", "pino-pretty": "^13.0.0", "postgres": "^3.4.7", diff --git a/apps/web/src/app/api/events/backfill/__tests__/route.test.ts b/apps/web/src/app/api/events/backfill/__tests__/route.test.ts new file mode 100644 index 0000000000..1246d5adc2 --- /dev/null +++ b/apps/web/src/app/api/events/backfill/__tests__/route.test.ts @@ -0,0 +1,529 @@ +// pnpm test src/app/api/events/backfill/__tests__/route.test.ts + +import fs from 'fs'; +import path from 'path'; +import { NextRequest } from 'next/server'; + +import { authorizeApi } from '@/actions/auth'; +import { analytics } from '@/lib/server'; +import type { ApiAuthResult } from '@/types'; + +import { POST } from '../route'; + +vi.mock('@/actions/auth', () => ({ + authorizeApi: vi.fn(), +})); + +const mockAuthorizeApi = vi.mocked(authorizeApi); + +describe('/api/events/backfill', () => { + const testProperties = { + appName: 'test-app', + appVersion: '1.0.0', + vscodeVersion: '1.80.0', + platform: 'darwin', + editorName: 'vscode', + language: 'typescript', + mode: 'code', + provider: 'anthropic', + model: 'claude-3-sonnet', + sessionId: 'test-session-123', + isSubtask: false, + }; + + afterEach(async () => { + // Clean up any test data from the analytics database. + try { + await analytics.command({ + query: `DELETE FROM messages WHERE taskId LIKE 'test-%'`, + }); + } catch { + // Ignore cleanup errors. + } + }); + + it('should successfully process a backfill request with valid file upload', async () => { + mockAuthorizeApi.mockResolvedValue({ + success: true, + userId: 'test-user-id', + orgId: 'test-org-id', + userType: 'user', + orgRole: 'admin', + } as unknown as ApiAuthResult); + + const messages = [ + { + ts: 1750702747687, + type: 'say' as const, + say: 'text' as const, + text: 'Test message', + images: [], + }, + { + ts: 1750702747696, + type: 'say' as const, + say: 'api_req_started' as const, + text: 'API request started', + images: [], + }, + ]; + + const fileContent = JSON.stringify(messages); + const file = new File([fileContent], 'test-messages.json', { + type: 'application/json', + }); + + const formData = new FormData(); + formData.append('file', file); + formData.append('taskId', 'test-task-integration'); + formData.append('properties', JSON.stringify(testProperties)); + + const request = new NextRequest( + 'http://localhost:3000/api/events/backfill', + { + method: 'POST', + body: formData, + }, + ); + + const response = await POST(request); + const responseData = await response.json(); + + expect(response.status).toBe(200); + expect(responseData.success).toBe(true); + + const dbResults = await analytics.query({ + query: ` + SELECT + taskId, + mode, + text, + type as messageType, + say, + ts + FROM messages + WHERE taskId = 'test-task-integration' + ORDER BY ts ASC + `, + format: 'JSONEachRow', + }); + + const dbData = (await dbResults.json()) as Array<{ + taskId: string; + mode: string; + text: string; + messageType: string; + say: string; + ts: number; + }>; + + expect(dbData).toHaveLength(2); + + expect(dbData[0]).toMatchObject({ + taskId: 'test-task-integration', + mode: 'code', + text: 'Test message', + messageType: 'say', + say: 'text', + }); + + expect(dbData[1]).toMatchObject({ + taskId: 'test-task-integration', + mode: 'code', + text: 'API request started', + messageType: 'say', + say: 'api_req_started', + }); + + expect(typeof dbData[0]?.ts).toBe('number'); + expect(typeof dbData[1]?.ts).toBe('number'); + }); + + it('should successfully process a backfill request using task.json file', async () => { + mockAuthorizeApi.mockResolvedValue({ + success: true, + userId: 'test-user-id', + orgId: 'test-org-id', + userType: 'user', + orgRole: 'admin', + } as unknown as ApiAuthResult); + + const taskJsonPath = path.join(__dirname, 'task.json'); + const taskJsonContent = fs.readFileSync(taskJsonPath, 'utf-8'); + const messages = JSON.parse(taskJsonContent); + + const file = new File([taskJsonContent], 'task.json', { + type: 'application/json', + }); + + const formData = new FormData(); + formData.append('file', file); + formData.append('taskId', 'test-task-from-file'); + formData.append('properties', JSON.stringify(testProperties)); + + const request = new NextRequest( + 'http://localhost:3000/api/events/backfill', + { + method: 'POST', + body: formData, + }, + ); + + const response = await POST(request); + const responseData = await response.json(); + + expect(response.status).toBe(200); + expect(responseData.success).toBe(true); + + const dbResults = await analytics.query({ + query: ` + SELECT + taskId, + mode, + text, + type as messageType, + say, + timestamp + FROM messages + WHERE taskId = 'test-task-from-file' + ORDER BY timestamp ASC + `, + format: 'JSONEachRow', + }); + + const dbData = (await dbResults.json()) as Array<{ + taskId: string; + mode: string; + text: string; + messageType: string; + say: string; + timestamp: number; + }>; + + expect(dbData).toHaveLength(messages.length); + + const textMessage = dbData.find((msg) => msg.say === 'text'); + expect(textMessage).toMatchObject({ + taskId: 'test-task-from-file', + mode: 'code', + text: 'Usql psql, how can I get all the namespaces?', + messageType: 'say', + say: 'text', + }); + + const apiMessage = dbData.find((msg) => msg.say === 'api_req_started'); + expect(apiMessage).toMatchObject({ + taskId: 'test-task-from-file', + mode: 'code', + messageType: 'say', + say: 'api_req_started', + }); + + expect(textMessage?.timestamp).toBeTypeOf('number'); + expect(apiMessage?.timestamp).toBeTypeOf('number'); + + const messageTypes = dbData.map((item) => item.say); + expect(messageTypes).toContain('text'); + expect(messageTypes).toContain('api_req_started'); + expect(messageTypes).toContain('checkpoint_saved'); + expect(messageTypes).toContain('reasoning'); + expect(messageTypes).toContain('completion_result'); + }); + + it('should return 401 if authentication fails', async () => { + mockAuthorizeApi.mockResolvedValue({ + success: false, + error: 'Authentication failed', + }); + + const formData = new FormData(); + formData.append('file', new File(['[]'], 'test.json')); + formData.append('properties', JSON.stringify(testProperties)); + formData.append('taskId', 'test'); + + const request = new NextRequest( + 'http://localhost:3000/api/events/backfill', + { + method: 'POST', + body: formData, + }, + ); + + const response = await POST(request); + const responseData = await response.json(); + + expect(response.status).toBe(401); + expect(responseData.success).toBe(false); + expect(responseData.error).toBe('Authentication required'); + + const dbResults = await analytics.query({ + query: `SELECT COUNT() as count FROM messages WHERE taskId = 'test'`, + format: 'JSONEachRow', + }); + + const dbData = (await dbResults.json()) as Array<{ count: string }>; + expect(dbData[0]?.count).toBe('0'); + }); + + it('should return 400 if no file is provided', async () => { + mockAuthorizeApi.mockResolvedValue({ + success: true, + userId: 'test-user-id', + orgId: 'test-org-id', + userType: 'user', + orgRole: 'admin', + } as unknown as ApiAuthResult); + + const formData = new FormData(); + formData.append('taskId', 'test-task-id'); + formData.append('properties', JSON.stringify(testProperties)); + + const request = new NextRequest( + 'http://localhost:3000/api/events/backfill', + { + method: 'POST', + body: formData, + }, + ); + + const response = await POST(request); + const responseData = await response.json(); + + expect(response.status).toBe(400); + expect(responseData.success).toBe(false); + expect(responseData.error).toBe('No file provided'); + }); + + it('should return 400 if taskId is missing', async () => { + mockAuthorizeApi.mockResolvedValue({ + success: true, + userId: 'test-user-id', + orgId: 'test-org-id', + userType: 'user', + orgRole: 'admin', + } as unknown as ApiAuthResult); + + const formData = new FormData(); + formData.append('file', new File(['[]'], 'test.json')); + formData.append('properties', JSON.stringify(testProperties)); + + const request = new NextRequest( + 'http://localhost:3000/api/events/backfill', + { + method: 'POST', + body: formData, + }, + ); + + const response = await POST(request); + const responseData = await response.json(); + + expect(response.status).toBe(400); + expect(responseData.success).toBe(false); + expect(responseData.error).toBe('taskId is required'); + }); + + it('should return 400 for invalid JSON content', async () => { + mockAuthorizeApi.mockResolvedValue({ + success: true, + userId: 'test-user-id', + orgId: 'test-org-id', + userType: 'user', + orgRole: 'admin', + } as unknown as ApiAuthResult); + + const formData = new FormData(); + formData.append('file', new File(['invalid json content'], 'test.json')); + formData.append('taskId', 'test-task-id'); + formData.append('properties', JSON.stringify(testProperties)); + + const request = new NextRequest( + 'http://localhost:3000/api/events/backfill', + { + method: 'POST', + body: formData, + }, + ); + + const response = await POST(request); + const responseData = await response.json(); + + expect(response.status).toBe(400); + expect(responseData.success).toBe(false); + expect(responseData.error).toBe('Invalid JSON file format'); + }); + + it('should return 400 for empty message array', async () => { + mockAuthorizeApi.mockResolvedValue({ + success: true, + userId: 'test-user-id', + orgId: 'test-org-id', + userType: 'user', + orgRole: 'admin', + } as unknown as ApiAuthResult); + + const formData = new FormData(); + formData.append('file', new File(['[]'], 'test.json')); + formData.append('properties', JSON.stringify(testProperties)); + formData.append('taskId', 'test-task-id'); + + const request = new NextRequest( + 'http://localhost:3000/api/events/backfill', + { + method: 'POST', + body: formData, + }, + ); + + const response = await POST(request); + const responseData = await response.json(); + + expect(response.status).toBe(400); + expect(responseData.success).toBe(false); + expect(responseData.error).toBe('File contains no messages'); + }); + + it('should extract mode from individual messages when mode slug is present', async () => { + mockAuthorizeApi.mockResolvedValue({ + success: true, + userId: 'test-user-id', + orgId: 'test-org-id', + userType: 'user', + orgRole: 'admin', + } as unknown as ApiAuthResult); + + const messages = [ + { + ts: 1750702747687, + type: 'say' as const, + say: 'text' as const, + text: 'Starting task in debug mode', + images: [], + }, + { + ts: 1750702747696, + type: 'say' as const, + say: 'api_req_started' as const, + text: 'API request started', + images: [], + }, + { + ts: 1750702747705, + type: 'say' as const, + say: 'text' as const, + text: 'Switching to architect mode', + images: [], + }, + ]; + + const fileContent = JSON.stringify(messages); + const file = new File([fileContent], 'test-messages.json', { + type: 'application/json', + }); + + const formData = new FormData(); + formData.append('file', file); + formData.append('taskId', 'test-task-mode-extraction'); + formData.append('properties', JSON.stringify(testProperties)); + + const request = new NextRequest( + 'http://localhost:3000/api/events/backfill', + { + method: 'POST', + body: formData, + }, + ); + + const response = await POST(request); + const responseData = await response.json(); + + expect(response.status).toBe(200); + expect(responseData.success).toBe(true); + + const dbResults = await analytics.query({ + query: ` + SELECT + taskId, + mode, + text, + type as messageType, + say + FROM messages + WHERE taskId = 'test-task-mode-extraction' + ORDER BY ts ASC + `, + format: 'JSONEachRow', + }); + + const dbData = (await dbResults.json()) as Array<{ + taskId: string; + mode: string; + text: string; + messageType: string; + say: string; + }>; + + expect(dbData).toHaveLength(3); + + expect(dbData[0]).toMatchObject({ + taskId: 'test-task-mode-extraction', + mode: 'debug', + text: 'Starting task in debug mode', + messageType: 'say', + say: 'text', + }); + + expect(dbData[1]).toMatchObject({ + taskId: 'test-task-mode-extraction', + mode: 'debug', + text: 'API request started', + messageType: 'say', + say: 'api_req_started', + }); + + expect(dbData[2]).toMatchObject({ + taskId: 'test-task-mode-extraction', + mode: 'architect', + text: 'Switching to architect mode', + messageType: 'say', + say: 'text', + }); + }); + + it('should handle invalid ClineMessage schema', async () => { + mockAuthorizeApi.mockResolvedValue({ + success: true, + userId: 'test-user-id', + orgId: 'test-org-id', + userType: 'user', + orgRole: 'admin', + } as unknown as ApiAuthResult); + + const invalidMessages = [{ text: 'Invalid message' }]; + + const formData = new FormData(); + + formData.append( + 'file', + new File([JSON.stringify(invalidMessages)], 'test.json'), + ); + + formData.append('taskId', 'test-task-invalid'); + formData.append('properties', JSON.stringify(testProperties)); + + const request = new NextRequest( + 'http://localhost:3000/api/events/backfill', + { + method: 'POST', + body: formData, + }, + ); + + const response = await POST(request); + const responseData = await response.json(); + + expect(response.status).toBe(400); + expect(responseData.success).toBe(false); + expect(responseData.error).toContain('Invalid file content'); + }); +}); diff --git a/apps/web/src/app/api/events/backfill/__tests__/task.json b/apps/web/src/app/api/events/backfill/__tests__/task.json new file mode 100644 index 0000000000..203cdf4d18 --- /dev/null +++ b/apps/web/src/app/api/events/backfill/__tests__/task.json @@ -0,0 +1,96 @@ +[ + { + "ts": 1750702747687, + "type": "say", + "say": "text", + "text": "Usql psql, how can I get all the namespaces?", + "images": [] + }, + { + "ts": 1750702747696, + "type": "say", + "say": "api_req_started", + "text": "{\"request\":\"\\nUsql psql, how can I get all the namespaces?\\n\\n\\n\\n# VSCode Visible Files\\napps/web/.env.development\\n\\n# VSCode Open Tabs\\napps/web/.env.local,apps/web/.env.development\\n\\n# Current Time\\n6/23/2025, 11:19:07 AM (America/Los_Angeles, UTC-7:00)\\n\\n# Current Context Size (Tokens)\\n(Not available)\\n\\n# Current Cost\\n$0.00\\n\\n# Current Mode\\ncode\\n💻 Code\\nanthropic/claude-sonnet-4\\n\\n\\n# Current Workspace Directory (/Users/cte/Documents/Roo-Code-Cloud) Files\\n.env\\n.gitignore\\n.prettierrc.json\\n.roomodes\\n.tool-versions\\napps/\\napps/roomote/\\napps/roomote/.env.example\\napps/roomote/.gitignore\\napps/roomote/eslint.config.mjs\\napps/roomote/next-env.d.ts\\napps/roomote/next.config.ts\\napps/roomote/package.json\\napps/roomote/tsconfig.json\\napps/roomote/vitest.config.ts\\napps/roomote/scripts/\\napps/roomote/scripts/build.sh\\napps/roomote/scripts/enqueue-github-issue-job.sh\\napps/roomote/src/\\napps/roomote/src/app/\\napps/roomote/src/app/layout.tsx\\napps/roomote/src/app/page.tsx\\napps/roomote/src/app/api/\\napps/roomote/src/app/api/health/\\napps/roomote/src/app/api/health/route.ts\\napps/roomote/src/app/api/jobs/\\napps/roomote/src/app/api/jobs/route.ts\\napps/roomote/src/app/api/jobs/[id]/\\napps/roomote/src/app/api/jobs/[id]/route.ts\\napps/roomote/src/app/api/webhooks/\\napps/roomote/src/app/api/webhooks/github/\\napps/roomote/src/app/api/webhooks/github/route.ts\\napps/roomote/src/app/api/webhooks/github/__tests__/\\napps/roomote/src/app/api/webhooks/github/__tests__/route.test.ts\\napps/roomote/src/app/api/webhooks/github/handlers/\\napps/roomote/src/app/api/webhooks/github/handlers/index.ts\\napps/roomote/src/app/api/webhooks/github/handlers/issueCommentHandler.ts\\napps/roomote/src/app/api/webhooks/github/handlers/issueHandler.ts\\napps/roomote/src/app/api/webhooks/github/handlers/pullRequestHandler.ts\\napps/roomote/src/app/api/webhooks/github/handlers/pullRequestReviewCommentHandler.ts\\napps/roomote/src/app/api/webhooks/github/handlers/utils.ts\\napps/roomote/src/app/api/webhooks/github/handlers/__tests__/\\napps/roomote/src/app/api/webhooks/github/handlers/__tests__/utils.test.ts\\napps/roomote/src/lib/\\napps/roomote/src/lib/controller.ts\\napps/roomote/src/lib/index.ts\\napps/roomote/src/lib/job.ts\\napps/roomote/src/lib/logger.ts\\napps/roomote/src/lib/queue.ts\\napps/roomote/src/lib/redis.ts\\napps/roomote/src/lib/runTask.ts\\napps/roomote/src/lib/slack.ts\\napps/roomote/src/lib/utils.ts\\napps/roomote/src/lib/worker.ts\\napps/roomote/src/lib/__tests__/\\napps/roomote/src/lib/__tests__/controller.test.ts\\napps/roomote/src/lib/jobs/\\napps/roomote/src/lib/jobs/fixGitHubIssue.ts\\napps/roomote/src/lib/jobs/processIssueComment.ts\\napps/roomote/src/lib/jobs/processPullRequestComment.ts\\napps/roomote/src/types/\\napps/roomote/src/types/index.ts\\napps/roomote-dashboard/\\napps/roomote-dashboard/eslint.config.mjs\\napps/roomote-dashboard/package.json\\napps/roomote-dashboard/tsconfig.json\\napps/roomote-dashboard/src/\\napps/roomote-dashboard/src/index.ts\\napps/web/\\napps/web/.env\\napps/web/.env.development\\napps/web/.env.test\\napps/web/.gitignore\\napps/web/components.json\\napps/web/eslint.config.mjs\\napps/web/next-env.d.ts\\napps/web/next.config.ts\\napps/web/package.json\\napps/web/postcss.config.mjs\\napps/web/README.md\\napps/web/public/\\napps/web/public/android-chrome-192x192.png\\napps/web/public/android-chrome-512x512.png\\napps/web/public/apple-touch-icon.png\\napps/web/public/favicon-16x16.png\\napps/web/public/favicon-32x32.png\\napps/web/public/favicon.ico\\napps/web/src/\\napps/web/src/actions/\\napps/web/src/actions/agents.ts\\napps/web/src/actions/auditLogs.ts\\napps/web/src/actions/auth.ts\\napps/web/src/actions/locale.ts\\napps/web/src/actions/sync.ts\\napps/web/src/actions/taskSharing.ts\\napps/web/src/actions/__tests__/\\napps/web/src/actions/__tests__/agents.test.ts\\napps/web/src/actions/__tests__/syncCurrentUser.test.ts\\napps/web/src/actions/__tests__/syncOrg.test.ts\\napps/web/src/actions/__tests__/taskSharing.test.ts\\napps/web/src/actions/analytics/\\napps/web/src/actions/analytics/events.ts\\napps/web/src/actions/analytics/index.ts\\napps/web/src/actions/analytics/messages.ts\\napps/web/src/app/\\napps/web/src/app/global-error.tsx\\napps/web/src/app/globals.css\\napps/web/src/app/page.tsx\\napps/web/src/app/(authenticated)/\\napps/web/src/app/(authenticated)/layout.tsx\\napps/web/src/app/(authenticated)/audit-logs/\\napps/web/src/app/(authenticated)/audit-logs/AuditLogs.tsx\\napps/web/src/app/(authenticated)/audit-logs/page.tsx\\napps/web/src/app/(authenticated)/org/\\napps/web/src/app/(authenticated)/org/[[...organization-profile]]/\\napps/web/src/app/(authenticated)/providers/\\napps/web/src/app/(authenticated)/providers/page.tsx\\napps/web/src/app/(authenticated)/providers/ProviderForm.tsx\\napps/web/src/app/(authenticated)/providers/ProviderSettings.tsx\\napps/web/src/app/(authenticated)/settings/\\napps/web/src/app/(authenticated)/settings/page.tsx\\napps/web/src/app/(authenticated)/settings/SettingsForm.tsx\\napps/web/src/app/(authenticated)/settings/SettingsPage.tsx\\napps/web/src/app/(authenticated)/usage/\\napps/web/src/app/(authenticated)/usage/Developers.tsx\\napps/web/src/app/(authenticated)/usage/Tasks.tsx\\napps/web/src/app/(centered)/\\napps/web/src/app/(centered)/layout.tsx\\napps/web/src/app/(centered)/authorized/\\napps/web/src/app/(centered)/authorized/page.tsx\\napps/web/src/app/(centered)/extension/\\napps/web/src/app/(centered)/extension/sign-in/\\napps/web/src/app/(centered)/select-org/\\napps/web/src/app/(centered)/select-org/SelectOrg.tsx\\napps/web/src/app/(centered)/sign-in/\\napps/web/src/app/(centered)/sign-in/[[...sign-in]]/\\napps/web/src/app/(centered)/sign-up/\\napps/web/src/app/(centered)/sign-up/[[...sign-up]]/\\napps/web/src/app/api/\\napps/web/src/app/api/events/\\napps/web/src/app/api/events/route.ts\\napps/web/src/app/api/events/__tests__/\\napps/web/src/app/api/extension/\\napps/web/src/app/api/extension/share/\\napps/web/src/app/api/marketplace/\\napps/web/src/app/api/marketplace/mcps/\\napps/web/src/app/api/marketplace/modes/\\napps/web/src/app/api/me/\\napps/web/src/app/api/organization-settings/\\napps/web/src/app/share/\\napps/web/src/app/share/[token]/\\napps/web/src/app/share/[token]/page.tsx\\napps/web/src/components/\\napps/web/src/components/__tests__/\\napps/web/src/components/__tests__/ProviderForm.test.tsx\\napps/web/src/components/audit-logs/\\napps/web/src/components/audit-logs/AuditLogCard.tsx\\napps/web/src/components/audit-logs/AuditLogDetails.tsx\\napps/web/src/components/audit-logs/AuditLogDrawer.tsx\\napps/web/src/components/audit-logs/AuditLogEntry.tsx\\napps/web/src/components/audit-logs/index.ts\\napps/web/src/components/layout/\\napps/web/src/components/layout/Connected.tsx\\napps/web/src/components/layout/DataTable.tsx\\napps/web/src/components/layout/LocaleSwitcher.tsx\\napps/web/src/components/layout/NavbarHeader.tsx\\napps/web/src/components/layout/NavbarMenu.tsx\\napps/web/src/components/layout/Section.tsx\\napps/web/src/components/layout/SentryUserContext.tsx\\napps/web/src/components/layout/TitleBar.tsx\\napps/web/src/components/task-sharing/\\napps/web/src/components/task-sharing/index.ts\\napps/web/src/components/task-sharing/ShareButton.tsx\\napps/web/src/components/ui/\\napps/web/src/components/ui/ecosystem/\\napps/web/src/components/usage/\\napps/web/src/data/\\napps/web/src/data/marketplace/\\napps/web/src/hooks/\\napps/web/src/hooks/__tests__/\\napps/web/src/i18n/\\napps/web/src/i18n/__tests__/\\napps/web/src/i18n/locales/\\napps/web/src/lib/\\napps/web/src/lib/__tests__/\\napps/web/src/lib/server/\\napps/web/src/lib/server/__tests__/\\napps/web/src/types/\\npackages/\\npackages/config-eslint/\\npackages/config-typescript/\\npackages/db/\\npackages/db/clickhouse/\\npackages/db/drizzle/\\npackages/db/drizzle/meta/\\npackages/db/src/\\npackages/db/src/queries/\\npackages/ipc/\\npackages/ipc/src/\\nscripts/\\n\\n(File list truncated. Use list_files on specific subdirectories if you need to explore further.)\\n\",\"tokensIn\":12990,\"tokensOut\":559,\"cacheWrites\":0,\"cacheReads\":0,\"cost\":0.02246925}" + }, + { + "ts": 1750702748307, + "type": "say", + "say": "checkpoint_saved", + "text": "b2081498c6cedce94caa342a87dd1f124188107b", + "checkpoint": { + "isFirst": true, + "from": "b2081498c6cedce94caa342a87dd1f124188107b", + "to": "b2081498c6cedce94caa342a87dd1f124188107b" + } + }, + { + "ts": 1750702750164, + "type": "say", + "say": "reasoning", + "text": "The user is asking about how to get all namespaces using psql. This is a PostgreSQL-related question about database namespaces/schemas. They want to know the SQL command to list all namespaces (schemas) in a PostgreSQL database.\n\nI should provide the SQL query they can run in psql to list all namespaces/schemas. In PostgreSQL, namespaces are called schemas, and there are a few ways to list them:\n\n1. Using `\\dn` command in psql (interactive command)\n2. Using SQL query to select from information_schema.schemata\n3. Using SQL query to select from pg_namespace\n\nSince they specifically mentioned psql, I should provide both the psql command and the SQL alternatives.", + "partial": true + }, + { + "ts": 1750702752692, + "type": "say", + "say": "text", + "text": "In PostgreSQL, namespaces are called **schemas**. Here are several ways to get all namespaces/schemas using psql:\n\n## 1. Using psql Meta-command (Easiest)\n```sql\n\\dn\n```\nThis shows all schemas with their names and owners.\n\nFor more detailed information:\n```sql\n\\dn+\n```\n\n## 2. Using SQL Query - Information Schema\n```sql\nSELECT schema_name \nFROM information_schema.schemata\nORDER BY schema_name;\n```\n\n## 3. Using SQL Query - System Catalog\n```sql\nSELECT nspname AS schema_name\nFROM pg_namespace\nWHERE nspname NOT LIKE 'pg_%' \n AND nspname != 'information_schema'\nORDER BY nspname;\n```\n\n## 4. Include System Schemas\nIf you want to see ALL schemas including system ones:\n```sql\nSELECT nspname AS schema_name\nFROM pg_namespace\nORDER BY nspname;\n```\n\n## 5. With Additional Details\n```sql\nSELECT \n n.nspname AS schema_name,\n r.rolname AS owner,\n n.nspacl AS permissions\nFROM pg_namespace n\nLEFT JOIN pg_roles r ON n.nspowner = r.oid\nWHERE n.nspname NOT LIKE 'pg_%' \n AND n.nspname != 'information_schema'\nORDER BY n.nspname;\n```\n\nThe **`\\dn`** command is usually the quickest way when working interactively in psql, while the SQL queries are useful when you need to incorporate the results into scripts or applications.", + "partial": false + }, + { + "ts": 1750702758355, + "type": "say", + "say": "api_req_started", + "text": "{\"request\":\"[ERROR] You did not use a tool in your previous response! Please retry with a tool use.\\n\\n# Reminder: Instructions for Tool Use\\n\\nTool uses are formatted using XML-style tags. The tool name itself becomes the XML tag name. Each parameter is enclosed within its own set of tags. Here's the structure:\\n\\n\\nvalue1\\nvalue2\\n...\\n\\n\\nFor example, to use the attempt_completion tool:\\n\\n\\n\\nI have completed the task...\\n\\n\\n\\nAlways use the actual tool name as the XML tag name for proper parsing and execution.\\n\\n# Next Steps\\n\\nIf you have completed the user's task, use the attempt_completion tool. \\nIf you require additional information from the user, use the ask_followup_question tool. \\nOtherwise, if you have not completed the task and do not need additional information, then proceed with the next step of the task. \\n(This is an automated message, so do not respond to it conversationally.)\\n\\n\\n# VSCode Visible Files\\napps/web/.env.development\\n\\n# VSCode Open Tabs\\napps/web/.env.local,apps/web/.env.development\\n\\n# Current Time\\n6/23/2025, 11:19:18 AM (America/Los_Angeles, UTC-7:00)\\n\\n# Current Context Size (Tokens)\\n13,549 (7%)\\n\\n# Current Cost\\n$0.02\\n\\n# Current Mode\\ncode\\n💻 Code\\nanthropic/claude-sonnet-4\\n\",\"tokensIn\":13788,\"tokensOut\":487,\"cacheWrites\":0,\"cacheReads\":0,\"cost\":0.0142215}" + }, + { + "ts": 1750702761909, + "type": "say", + "say": "reasoning", + "text": "The user asked about how to get all namespaces using psql. I provided the answer but didn't use a tool. Since I've provided the complete information needed to answer their question about PostgreSQL schemas/namespaces, I should use the attempt_completion tool to mark this task as complete.", + "partial": true + }, + { "ts": 1750702762580, "type": "say", "say": "text", "text": "" }, + { + "ts": 1750702762592, + "type": "say", + "say": "completion_result", + "text": "In PostgreSQL, namespaces are called **schemas**. Here are several ways to get all namespaces/schemas using psql:\n\n## 1. Using psql Meta-command (Easiest)\n```sql\n\\dn\n```\nThis shows all schemas with their names and owners.\n\nFor more detailed information:\n```sql\n\\dn+\n```\n\n## 2. Using SQL Query - Information Schema\n```sql\nSELECT schema_name \nFROM information_schema.schemata\nORDER BY schema_name;\n```\n\n## 3. Using SQL Query - System Catalog\n```sql\nSELECT nspname AS schema_name\nFROM pg_namespace\nWHERE nspname NOT LIKE 'pg_%' \n AND nspname != 'information_schema'\nORDER BY nspname;\n```\n\n## 4. Include System Schemas\nIf you want to see ALL schemas including system ones:\n```sql\nSELECT nspname AS schema_name\nFROM pg_namespace\nORDER BY nspname;\n```\n\n## 5. With Additional Details\n```sql\nSELECT \n n.nspname AS schema_name,\n r.rolname AS owner,\n n.nspacl AS permissions\nFROM pg_namespace n\nLEFT JOIN pg_roles r ON n.nspowner = r.oid\nWHERE n.nspname NOT LIKE 'pg_%' \n AND n.nspname != 'information_schema'\nORDER BY n.nspname;\n```\n\nThe **[`\\dn`](psql:meta-command)** command is usually the quickest way when working interactively in psql, while the SQL queries are useful when you need to incorporate the results into scripts or applications.", + "partial": false + }, + { + "ts": 1750702766387, + "type": "ask", + "ask": "completion_result", + "text": "" + }, + { + "ts": 1750702838217, + "type": "say", + "say": "user_feedback", + "text": "How do I enter a namespace?", + "images": [] + }, + { + "ts": 1750702838250, + "type": "say", + "say": "api_req_started", + "text": "{\"request\":\"[attempt_completion] Result:\\n\\nThe user has provided feedback on the results. Consider their input to continue the task, and then attempt completion again.\\n\\nHow do I enter a namespace?\\n\\n\\n\\n# VSCode Visible Files\\napps/web/.env.development\\n\\n# VSCode Open Tabs\\napps/web/.env.local,apps/web/.env.development\\n\\n# Current Time\\n6/23/2025, 11:20:38 AM (America/Los_Angeles, UTC-7:00)\\n\\n# Current Context Size (Tokens)\\n14,275 (7%)\\n\\n# Current Cost\\n$0.04\\n\\n# Current Mode\\ncode\\n💻 Code\\nanthropic/claude-sonnet-4\\n\",\"tokensIn\":14399,\"tokensOut\":466,\"cacheWrites\":0,\"cacheReads\":0,\"cost\":0.01344465}" + }, + { + "ts": 1750702841419, + "type": "say", + "say": "reasoning", + "text": "The user is asking how to enter a namespace in PostgreSQL. In PostgreSQL, \"entering\" a namespace (schema) means setting the search path to that schema or connecting to/using that schema. I should provide the different ways to do this.", + "partial": true + }, + { "ts": 1750702841779, "type": "say", "say": "text", "text": "" }, + { + "ts": 1750702841794, + "type": "say", + "say": "completion_result", + "text": "To enter or switch to a specific namespace (schema) in PostgreSQL, you have several options:\n\n## 1. Set the Search Path (Most Common)\n```sql\nSET search_path TO schema_name;\n```\n\nThis makes the specified schema the default for subsequent queries. For example:\n```sql\nSET search_path TO my_schema;\n```\n\n## 2. Set Multiple Schemas in Search Path\n```sql\nSET search_path TO schema1, schema2, public;\n```\n\nThis will search for objects in the order listed.\n\n## 3. Reset to Default Schema\n```sql\nSET search_path TO DEFAULT;\n```\nor\n```sql\nSET search_path TO public;\n```\n\n## 4. Check Current Search Path\n```sql\nSHOW search_path;\n```\n\n## 5. Temporary Session Setting\nThe [`SET search_path`](postgresql:command) command only affects your current session. To make it permanent for a user:\n\n```sql\nALTER USER username SET search_path = schema_name;\n```\n\n## 6. Qualify Table Names (Alternative)\nInstead of changing search path, you can always specify the schema explicitly:\n```sql\nSELECT * FROM schema_name.table_name;\n```\n\n## 7. Using psql Meta-command\nYou can also use the psql meta-command to show current schema:\n```sql\n\\c database_name\n```\n\nAfter setting the search path, all unqualified table/object references will look in that schema first. For example, after [`SET search_path TO my_schema`](postgresql:command), running [`SELECT * FROM users`](postgresql:query) will look for the table in `my_schema.users` first.", + "partial": false + }, + { "ts": 1750702847873, "type": "ask", "ask": "completion_result", "text": "" } +] diff --git a/apps/web/src/app/api/events/backfill/route.ts b/apps/web/src/app/api/events/backfill/route.ts new file mode 100644 index 0000000000..fe46c43853 --- /dev/null +++ b/apps/web/src/app/api/events/backfill/route.ts @@ -0,0 +1,135 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { v4 as uuidv4 } from 'uuid'; +import { z } from 'zod'; +import pMap from 'p-map'; + +import { + type ClineMessage, + TelemetryEventName, + clineMessageSchema, + telemetryPropertiesSchema, +} from '@roo-code/types'; + +import { authorizeApi } from '@/actions/auth'; +import { captureEvent } from '@/actions/analytics'; + +export async function POST(request: NextRequest) { + const authResult = await authorizeApi(request); + + if (!authResult.success) { + return NextResponse.json( + { success: false, error: 'Authentication required' }, + { status: 401 }, + ); + } + + const { userId, orgId } = authResult; + + try { + const formData = await request.formData(); + const taskId = formData.get('taskId') as string; + + if (!taskId) { + return NextResponse.json( + { success: false, error: 'taskId is required' }, + { status: 400 }, + ); + } + + const properties = telemetryPropertiesSchema.parse( + JSON.parse(formData.get('properties') as string), + ); + + const file = formData.get('file') as File; + + if (!file) { + return NextResponse.json( + { success: false, error: 'No file provided' }, + { status: 400 }, + ); + } + + const fileContent = await file.text(); + let messages: ClineMessage[]; + + try { + const parsedContent = JSON.parse(fileContent); + + const messagesResult = z + .array(clineMessageSchema) + .safeParse(parsedContent); + + if (!messagesResult.success) { + return NextResponse.json( + { + success: false, + error: `Invalid file content: ${messagesResult.error.message}`, + }, + { status: 400 }, + ); + } + + messages = messagesResult.data; + } catch (_error) { + return NextResponse.json( + { + success: false, + error: 'Invalid JSON file format', + }, + { status: 400 }, + ); + } + + if (messages.length === 0) { + return NextResponse.json( + { success: false, error: 'File contains no messages' }, + { status: 400 }, + ); + } + + const defaultMode = extractMode(messages[0]?.text) || properties.mode; + + await pMap( + messages, + async (message) => { + const id = uuidv4(); + const timestamp = Math.round(message.ts / 1000); + const mode = extractMode(message.text) || defaultMode; + + const event = { + type: TelemetryEventName.TASK_MESSAGE as const, + properties: { taskId, message, ...properties, mode }, + }; + + await captureEvent({ id, orgId, userId, timestamp, event }); + }, + { concurrency: 10 }, + ); + + return NextResponse.json({ success: true }); + } catch (error) { + console.error(error); + + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }, + { status: 500 }, + ); + } +} + +/** + * Extracts the mode from a message text if it contains a mode pattern. + * @param text - The message text to parse + * @returns The extracted mode or null if no mode slug is found + */ +function extractMode(text: string | undefined): string | null { + if (!text) { + return null; + } + + const modeMatch = text.match(/(.+?)<\/slug>/); + return modeMatch && modeMatch[1] ? modeMatch[1] : null; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fc56ffb2ca..3100c9839b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -260,6 +260,9 @@ importers: next-typesafe-url: specifier: ^5.1.7 version: 5.1.7(next@15.3.3(@babel/core@7.27.3)(@opentelemetry/api@1.9.0)(@playwright/test@1.52.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0)(zod@3.25.41) + p-map: + specifier: ^7.0.3 + version: 7.0.3 pino: specifier: ^9.7.0 version: 9.7.0 @@ -5121,6 +5124,10 @@ packages: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} + p-map@7.0.3: + resolution: {integrity: sha512-VkndIv2fIB99swvQoA65bm+fsmt6UNdGeIB0oxBs+WhAhdh08QA04JXpI7rbB9r08/nkbysKoya9rtDERYOYMA==} + engines: {node: '>=18'} + p-timeout@6.1.4: resolution: {integrity: sha512-MyIV3ZA/PmyBN/ud8vV9XzwTrNtR4jFrObymZYnZqMmW0zA8Z17vnT0rBgFE/TlohB+YCHqXMgZzb3Csp49vqg==} engines: {node: '>=14.16'} @@ -11552,6 +11559,8 @@ snapshots: dependencies: p-limit: 3.1.0 + p-map@7.0.3: {} + p-timeout@6.1.4: {} p-try@2.2.0: {}