From a228c578ce66f7af1e3b5157d855c68b27c72056 Mon Sep 17 00:00:00 2001 From: cte Date: Wed, 14 May 2025 14:14:38 -0700 Subject: [PATCH] Use Clickhouse --- .env.local.sample | 4 ++++ package.json | 2 ++ pnpm-lock.yaml | 25 +++++++++++++++++++++++++ src/app/api/events/route.ts | 21 +++++++++++++++------ src/db/clickhouse.ts | 22 ++++++++++++++++++++++ src/db/schema.ts | 36 +++--------------------------------- src/lib/server/analytics.ts | 18 ++++++++++++++++++ src/lib/server/env.ts | 6 ++++++ src/schemas/index.ts | 16 +++++++++++----- 9 files changed, 106 insertions(+), 44 deletions(-) create mode 100644 src/db/clickhouse.ts create mode 100644 src/lib/server/analytics.ts diff --git a/.env.local.sample b/.env.local.sample index 5ce4c7aad0..14fbf342b2 100644 --- a/.env.local.sample +++ b/.env.local.sample @@ -1 +1,5 @@ CLERK_SECRET_KEY=your_clerk_secret_key + +CLICKHOUSE_URL=your_clickhouse_url +CLICKHOUSE_USERNAME=your_clickhouse_username +CLICKHOUSE_PASSWORD=fake-your_clickhouse_password diff --git a/package.json b/package.json index 70bd51de77..125316b90a 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "@clerk/localizations": "^3.14.2", "@clerk/nextjs": "^6.18.3", "@clerk/themes": "^2.1.36", + "@clickhouse/client": "^1.11.1", "@logtail/pino": "^0.5.2", "@radix-ui/react-dropdown-menu": "^2.1.14", "@radix-ui/react-icons": "^1.3.2", @@ -52,6 +53,7 @@ "react-hook-form": "^7.53.0", "stripe": "^18.1.0", "tailwind-merge": "^3.3.0", + "uuid": "^11.1.0", "zod": "^3.23.8" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 92e5cb2825..b30839a314 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: '@clerk/themes': specifier: ^2.1.36 version: 2.2.43 + '@clickhouse/client': + specifier: ^1.11.1 + version: 1.11.1 '@logtail/pino': specifier: ^0.5.2 version: 0.5.4(pino@9.6.0) @@ -92,6 +95,9 @@ importers: tailwind-merge: specifier: ^3.3.0 version: 3.3.0 + uuid: + specifier: ^11.1.0 + version: 11.1.0 zod: specifier: ^3.23.8 version: 3.24.4 @@ -989,6 +995,13 @@ packages: resolution: {integrity: sha512-P30Vnqaw2UzoSZivNG61NvF9WiAVbKItOvkVJzUbwqDkISdmNOio8mWCeFuKq+bxjPESvS+S615Uplm7dl6jpg==} engines: {node: '>=18.17.0'} + '@clickhouse/client-common@1.11.1': + resolution: {integrity: sha512-bme0le2yhDSAh13d2fxhSW5ZrNoVqZ3LTyac8jK6hNH0qkksXnjYkLS6KQalPU6NMpffxHmpI4+/Gi2MnX0NCA==} + + '@clickhouse/client@1.11.1': + resolution: {integrity: sha512-u9h++h72SmWystijNqfNvMkfA+5+Y1LNfmLL/odCL3VgI3oyAPP9ubSw/Yrt2zRZkLKehMMD1kuOej0QHbSoBA==} + engines: {node: '>=16'} + '@cspotcode/source-map-support@0.8.1': resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} @@ -8245,6 +8258,10 @@ packages: utila@0.4.0: resolution: {integrity: sha512-Z0DbgELS9/L/75wZbro8xAnT50pBVFQZ+hUEueGDU5FN51YSCYM+jdxsfCiHjwNP/4LCDD0i/graKpeBnOXKRA==} + uuid@11.1.0: + resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} + hasBin: true + uuid@8.3.2: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} hasBin: true @@ -9521,6 +9538,12 @@ snapshots: dependencies: csstype: 3.1.3 + '@clickhouse/client-common@1.11.1': {} + + '@clickhouse/client@1.11.1': + dependencies: + '@clickhouse/client-common': 1.11.1 + '@cspotcode/source-map-support@0.8.1': dependencies: '@jridgewell/trace-mapping': 0.3.9 @@ -17751,6 +17774,8 @@ snapshots: utila@0.4.0: {} + uuid@11.1.0: {} + uuid@8.3.2: {} uuid@9.0.1: {} diff --git a/src/app/api/events/route.ts b/src/app/api/events/route.ts index e1d5964a8c..9dae1b2b3a 100644 --- a/src/app/api/events/route.ts +++ b/src/app/api/events/route.ts @@ -1,9 +1,9 @@ import { auth } from '@clerk/nextjs/server'; import { NextRequest, NextResponse } from 'next/server'; +import { v4 as uuidv4 } from 'uuid'; import { eventSchema } from '@/schemas'; -import { db } from '@/db'; -import { eventsTable } from '@/db/schema'; +import { captureEvent } from '@/lib/server/analytics'; export async function POST(request: NextRequest) { const { userId } = await auth(); @@ -16,13 +16,22 @@ export async function POST(request: NextRequest) { } const payload = await request.json(); - const result = eventSchema.safeParse(payload); + const id = uuidv4(); + const timestamp = Date.now() / 1000; + const result = eventSchema.safeParse({ ...payload, id, userId, timestamp }); if (!result.success) { - return NextResponse.json({ success: false }); + return NextResponse.json({ success: false, error: result.error.message }); } - const [record] = await db.insert(eventsTable).values(result.data).returning(); + try { + await captureEvent(result.data); + } catch (error) { + return NextResponse.json({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } - return NextResponse.json({ success: true, id: record?.id }); + return NextResponse.json({ success: true, id }); } diff --git a/src/db/clickhouse.ts b/src/db/clickhouse.ts new file mode 100644 index 0000000000..4ff4d0bae4 --- /dev/null +++ b/src/db/clickhouse.ts @@ -0,0 +1,22 @@ +export const createEvents = ` + CREATE TABLE default.events + ( + \`id\` UUID, + \`userId\` String, + \`type\` String, + \`timestamp\` Int32, + \`taskId\` Nullable(String), + \`provider\` Nullable(String), + \`modelId\` Nullable(String), + \`prompt\` Nullable(String), + \`mode\` Nullable(String), + \`inputTokens\` Nullable(Int32), + \`outputTokens\` Nullable(Int32), + \`cacheReadTokens\` Nullable(Int32), + \`cacheWriteTokens\` Nullable(Int32), + \`cost\` Nullable(Float32) + ) + ENGINE = SharedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}') + ORDER BY (id, type, timestamp) + SETTINGS index_granularity = 8192; +`; diff --git a/src/db/schema.ts b/src/db/schema.ts index d600638eed..49dec5aea2 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -1,38 +1,8 @@ -import { - bigint, - pgTable, - text, - timestamp, - json, - integer, -} from 'drizzle-orm/pg-core'; +import { pgTable, text, timestamp, integer } from 'drizzle-orm/pg-core'; -// export const organizationTable = pgTable( -// 'organizations', -// { -// id: integer().primaryKey().generatedAlwaysAsIdentity(), -// stripeCustomerId: text('stripe_customer_id'), -// stripeSubscriptionId: text('stripe_subscription_id'), -// stripeSubscriptionPriceId: text('stripe_subscription_price_id'), -// stripeSubscriptionStatus: text('stripe_subscription_status'), -// stripeSubscriptionCurrentPeriodEnd: bigint( -// 'stripe_subscription_current_period_end', -// { mode: 'number' }, -// ), -// updatedAt: timestamp('updated_at', { mode: 'date' }) -// .defaultNow() -// .$onUpdate(() => new Date()) -// .notNull(), -// createdAt: timestamp('created_at', { mode: 'date' }).defaultNow().notNull(), -// }, -// (table) => [uniqueIndex('stripe_customer_id_idx').on(table.stripeCustomerId)], -// ); - -export const eventsTable = pgTable('events', { +export const usersTable = pgTable('users', { id: integer().primaryKey().generatedAlwaysAsIdentity(), - type: text('type').notNull(), - timestamp: bigint('timestamp', { mode: 'number' }).notNull(), - properties: json('properties').notNull(), + authenticationId: text('authentication_id').notNull(), createdAt: timestamp('created_at', { mode: 'date' }).defaultNow().notNull(), updatedAt: timestamp('updated_at', { mode: 'date' }) .defaultNow() diff --git a/src/lib/server/analytics.ts b/src/lib/server/analytics.ts new file mode 100644 index 0000000000..8c7c4e9dc5 --- /dev/null +++ b/src/lib/server/analytics.ts @@ -0,0 +1,18 @@ +import { createClient } from '@clickhouse/client'; + +import { Event } from '@/schemas'; +import { Env } from './env'; + +const client = createClient({ + url: Env.CLICKHOUSE_URL, + username: Env.CLICKHOUSE_USERNAME, + password: Env.CLICKHOUSE_PASSWORD, +}); + +export const captureEvent = async ({ properties, ...event }: Event) => { + await client.insert({ + table: 'events', + values: [{ ...event, ...properties }], + format: 'JSONEachRow', + }); +}; diff --git a/src/lib/server/env.ts b/src/lib/server/env.ts index d27e9e92ee..82d02b9506 100644 --- a/src/lib/server/env.ts +++ b/src/lib/server/env.ts @@ -11,6 +11,9 @@ export const Env = createEnv({ BILLING_PLAN_ENV: z.enum(['dev', 'test', 'prod']), VSCODE_EXTENSION_BASE_URL: z.string().min(1), CURSOR_EXTENSION_BASE_URL: z.string().min(1), + CLICKHOUSE_URL: z.string().min(1), + CLICKHOUSE_USERNAME: z.string().min(1), + CLICKHOUSE_PASSWORD: z.string().min(1), }, client: { NEXT_PUBLIC_APP_URL: z.string().optional(), @@ -52,5 +55,8 @@ export const Env = createEnv({ NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY: process.env.NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY, NODE_ENV: process.env.NODE_ENV, + CLICKHOUSE_URL: process.env.CLICKHOUSE_URL, + CLICKHOUSE_USERNAME: process.env.CLICKHOUSE_USERNAME, + CLICKHOUSE_PASSWORD: process.env.CLICKHOUSE_PASSWORD, }, }); diff --git a/src/schemas/index.ts b/src/schemas/index.ts index 27671fd376..eff722889a 100644 --- a/src/schemas/index.ts +++ b/src/schemas/index.ts @@ -24,10 +24,15 @@ export const providerNames = [ 'litellm', ] as const; +const baseEventSchema = z.object({ + id: z.string().uuid(), + userId: z.string(), + timestamp: z.number(), +}); + export const eventSchema = z.discriminatedUnion('type', [ - z.object({ + baseEventSchema.extend({ type: z.literal('task_created'), - timestamp: z.number(), properties: z.object({ taskId: z.string(), provider: z.enum(providerNames), @@ -36,16 +41,17 @@ export const eventSchema = z.discriminatedUnion('type', [ mode: z.string(), }), }), - z.object({ + baseEventSchema.extend({ type: z.literal('completion'), - timestamp: z.number(), properties: z.object({ taskId: z.string(), provider: z.enum(providerNames), modelId: z.string(), inputTokens: z.number(), outputTokens: z.number(), - cost: z.number(), + cacheReadTokens: z.number().optional(), + cacheWriteTokens: z.number().optional(), + cost: z.number().optional(), }), }), ]);