Merge pull request #4 from RooCodeInc/cte/clickhouse

.env.local.sample

@@ -1 +1,5 @@
 CLERK_SECRET_KEY=your_clerk_secret_key
+
+CLICKHOUSE_URL=your_clickhouse_url
+CLICKHOUSE_USERNAME=your_clickhouse_username
+CLICKHOUSE_PASSWORD=fake-your_clickhouse_password

package.json

@@ -27,6 +27,7 @@
     "@clerk/localizations": "^3.14.2",
     "@clerk/nextjs": "^6.18.3",
     "@clerk/themes": "^2.1.36",
+    "@clickhouse/client": "^1.11.1",
     "@logtail/pino": "^0.5.2",
     "@radix-ui/react-dropdown-menu": "^2.1.14",
     "@radix-ui/react-icons": "^1.3.2",
@@ -52,6 +53,7 @@
     "react-hook-form": "^7.53.0",
     "stripe": "^18.1.0",
     "tailwind-merge": "^3.3.0",
+    "uuid": "^11.1.0",
     "zod": "^3.23.8"
   },
   "devDependencies": {

pnpm-lock.yaml

@@ -17,6 +17,9 @@ importers:
       '@clerk/themes':
         specifier: ^2.1.36
         version: 2.2.43
+      '@clickhouse/client':
+        specifier: ^1.11.1
+        version: 1.11.1
       '@logtail/pino':
         specifier: ^0.5.2
         version: 0.5.4(pino@9.6.0)
@@ -92,6 +95,9 @@ importers:
       tailwind-merge:
         specifier: ^3.3.0
         version: 3.3.0
+      uuid:
+        specifier: ^11.1.0
+        version: 11.1.0
       zod:
         specifier: ^3.23.8
         version: 3.24.4
@@ -989,6 +995,13 @@ packages:
     resolution: {integrity: sha512-P30Vnqaw2UzoSZivNG61NvF9WiAVbKItOvkVJzUbwqDkISdmNOio8mWCeFuKq+bxjPESvS+S615Uplm7dl6jpg==}
     engines: {node: '>=18.17.0'}
 
+  '@clickhouse/client-common@1.11.1':
+    resolution: {integrity: sha512-bme0le2yhDSAh13d2fxhSW5ZrNoVqZ3LTyac8jK6hNH0qkksXnjYkLS6KQalPU6NMpffxHmpI4+/Gi2MnX0NCA==}
+
+  '@clickhouse/client@1.11.1':
+    resolution: {integrity: sha512-u9h++h72SmWystijNqfNvMkfA+5+Y1LNfmLL/odCL3VgI3oyAPP9ubSw/Yrt2zRZkLKehMMD1kuOej0QHbSoBA==}
+    engines: {node: '>=16'}
+
   '@cspotcode/source-map-support@0.8.1':
     resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==}
     engines: {node: '>=12'}
@@ -8245,6 +8258,10 @@ packages:
   utila@0.4.0:
     resolution: {integrity: sha512-Z0DbgELS9/L/75wZbro8xAnT50pBVFQZ+hUEueGDU5FN51YSCYM+jdxsfCiHjwNP/4LCDD0i/graKpeBnOXKRA==}
 
+  uuid@11.1.0:
+    resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==}
+    hasBin: true
+
   uuid@8.3.2:
     resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==}
     hasBin: true
@@ -9521,6 +9538,12 @@ snapshots:
     dependencies:
       csstype: 3.1.3
 
+  '@clickhouse/client-common@1.11.1': {}
+
+  '@clickhouse/client@1.11.1':
+    dependencies:
+      '@clickhouse/client-common': 1.11.1
+
   '@cspotcode/source-map-support@0.8.1':
     dependencies:
       '@jridgewell/trace-mapping': 0.3.9
@@ -17751,6 +17774,8 @@ snapshots:
 
   utila@0.4.0: {}
 
+  uuid@11.1.0: {}
+
   uuid@8.3.2: {}
 
   uuid@9.0.1: {}

src/app/api/events/route.ts

@@ -1,9 +1,9 @@
 import { auth } from '@clerk/nextjs/server';
 import { NextRequest, NextResponse } from 'next/server';
+import { v4 as uuidv4 } from 'uuid';
 
 import { eventSchema } from '@/schemas';
-import { db } from '@/db';
-import { eventsTable } from '@/db/schema';
+import { captureEvent } from '@/lib/server/analytics';
 
 export async function POST(request: NextRequest) {
   const { userId } = await auth();
@@ -16,13 +16,22 @@ export async function POST(request: NextRequest) {
   }
 
   const payload = await request.json();
-  const result = eventSchema.safeParse(payload);
+  const id = uuidv4();
+  const timestamp = Date.now() / 1000;
+  const result = eventSchema.safeParse({ ...payload, id, userId, timestamp });
 
   if (!result.success) {
-    return NextResponse.json({ success: false });
+    return NextResponse.json({ success: false, error: result.error.message });
   }
 
-  const [record] = await db.insert(eventsTable).values(result.data).returning();
+  try {
+    await captureEvent(result.data);
+  } catch (error) {
+    return NextResponse.json({
+      success: false,
+      error: error instanceof Error ? error.message : 'Unknown error',
+    });
+  }
 
-  return NextResponse.json({ success: true, id: record?.id });
+  return NextResponse.json({ success: true, id });
 }

src/db/clickhouse.ts

@@ -0,0 +1,22 @@
+export const createEvents = `
+  CREATE TABLE default.events
+  (
+      \`id\` UUID,
+      \`userId\` String,
+      \`type\` String,
+      \`timestamp\` Int32,
+      \`taskId\` Nullable(String),
+      \`provider\` Nullable(String),
+      \`modelId\` Nullable(String),
+      \`prompt\` Nullable(String),
+      \`mode\` Nullable(String),
+      \`inputTokens\` Nullable(Int32),
+      \`outputTokens\` Nullable(Int32),
+      \`cacheReadTokens\` Nullable(Int32),
+      \`cacheWriteTokens\` Nullable(Int32),
+      \`cost\` Nullable(Float32)
+  )
+  ENGINE = SharedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
+  ORDER BY (id, type, timestamp)
+  SETTINGS index_granularity = 8192;
+`;

src/db/schema.ts

@@ -1,38 +1,8 @@
-import {
-  bigint,
-  pgTable,
-  text,
-  timestamp,
-  json,
-  integer,
-} from 'drizzle-orm/pg-core';
+import { pgTable, text, timestamp, integer } from 'drizzle-orm/pg-core';
 
-// export const organizationTable = pgTable(
-//   'organizations',
-//   {
-//     id: integer().primaryKey().generatedAlwaysAsIdentity(),
-//     stripeCustomerId: text('stripe_customer_id'),
-//     stripeSubscriptionId: text('stripe_subscription_id'),
-//     stripeSubscriptionPriceId: text('stripe_subscription_price_id'),
-//     stripeSubscriptionStatus: text('stripe_subscription_status'),
-//     stripeSubscriptionCurrentPeriodEnd: bigint(
-//       'stripe_subscription_current_period_end',
-//       { mode: 'number' },
-//     ),
-//     updatedAt: timestamp('updated_at', { mode: 'date' })
-//       .defaultNow()
-//       .$onUpdate(() => new Date())
-//       .notNull(),
-//     createdAt: timestamp('created_at', { mode: 'date' }).defaultNow().notNull(),
-//   },
-//   (table) => [uniqueIndex('stripe_customer_id_idx').on(table.stripeCustomerId)],
-// );
-
-export const eventsTable = pgTable('events', {
+export const usersTable = pgTable('users', {
   id: integer().primaryKey().generatedAlwaysAsIdentity(),
-  type: text('type').notNull(),
-  timestamp: bigint('timestamp', { mode: 'number' }).notNull(),
-  properties: json('properties').notNull(),
+  authenticationId: text('authentication_id').notNull(),
   createdAt: timestamp('created_at', { mode: 'date' }).defaultNow().notNull(),
   updatedAt: timestamp('updated_at', { mode: 'date' })
     .defaultNow()

src/lib/server/analytics.ts

@@ -0,0 +1,18 @@
+import { createClient } from '@clickhouse/client';
+
+import { Event } from '@/schemas';
+import { Env } from './env';
+
+const client = createClient({
+  url: Env.CLICKHOUSE_URL,
+  username: Env.CLICKHOUSE_USERNAME,
+  password: Env.CLICKHOUSE_PASSWORD,
+});
+
+export const captureEvent = async ({ properties, ...event }: Event) => {
+  await client.insert({
+    table: 'events',
+    values: [{ ...event, ...properties }],
+    format: 'JSONEachRow',
+  });
+};

src/lib/server/env.ts

@@ -11,6 +11,9 @@ export const Env = createEnv({
     BILLING_PLAN_ENV: z.enum(['dev', 'test', 'prod']),
     VSCODE_EXTENSION_BASE_URL: z.string().min(1),
     CURSOR_EXTENSION_BASE_URL: z.string().min(1),
+    CLICKHOUSE_URL: z.string().min(1),
+    CLICKHOUSE_USERNAME: z.string().min(1),
+    CLICKHOUSE_PASSWORD: z.string().min(1),
   },
   client: {
     NEXT_PUBLIC_APP_URL: z.string().optional(),
@@ -52,5 +55,8 @@ export const Env = createEnv({
     NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY:
       process.env.NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY,
     NODE_ENV: process.env.NODE_ENV,
+    CLICKHOUSE_URL: process.env.CLICKHOUSE_URL,
+    CLICKHOUSE_USERNAME: process.env.CLICKHOUSE_USERNAME,
+    CLICKHOUSE_PASSWORD: process.env.CLICKHOUSE_PASSWORD,
   },
 });

src/schemas/index.ts

@@ -24,10 +24,15 @@ export const providerNames = [
   'litellm',
 ] as const;
 
+const baseEventSchema = z.object({
+  id: z.string().uuid(),
+  userId: z.string(),
+  timestamp: z.number(),
+});
+
 export const eventSchema = z.discriminatedUnion('type', [
-  z.object({
+  baseEventSchema.extend({
     type: z.literal('task_created'),
-    timestamp: z.number(),
     properties: z.object({
       taskId: z.string(),
       provider: z.enum(providerNames),
@@ -36,16 +41,17 @@ export const eventSchema = z.discriminatedUnion('type', [
       mode: z.string(),
     }),
   }),
-  z.object({
+  baseEventSchema.extend({
     type: z.literal('completion'),
-    timestamp: z.number(),
     properties: z.object({
       taskId: z.string(),
       provider: z.enum(providerNames),
       modelId: z.string(),
       inputTokens: z.number(),
       outputTokens: z.number(),
-      cost: z.number(),
+      cacheReadTokens: z.number().optional(),
+      cacheWriteTokens: z.number().optional(),
+      cost: z.number().optional(),
     }),
   }),
 ]);