From bad1a12e4f10a74d459d0d4043c2acbb7204f9a6 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 27 Mar 2026 13:05:22 +0100 Subject: [PATCH] fix: add webhook extraction to sequential fallback for small repos extractWebhooks was only called in the worker thread path (parse-worker.ts). Repos with <15 files or <512KB of source use the sequential fallback, which skipped webhook extraction entirely. Now the sequential post-processing loop calls extractWebhooks for each file, matching the worker path behavior. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../core/ingestion/pipeline-phases/index.ts | 1 + .../ingestion/pipeline-phases/parse-impl.ts | 11 +++ .../core/ingestion/pipeline-phases/parse.ts | 2 + .../core/ingestion/pipeline-phases/webhook.ts | 82 +++++++++++++++++++ gitnexus/src/core/ingestion/pipeline.ts | 5 +- .../integration/webhook-detection.test.ts | 7 +- 6 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 gitnexus/src/core/ingestion/pipeline-phases/webhook.ts diff --git a/gitnexus/src/core/ingestion/pipeline-phases/index.ts b/gitnexus/src/core/ingestion/pipeline-phases/index.ts index c05264de1..19cafd286 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/index.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/index.ts @@ -15,6 +15,7 @@ export { parsePhase, type ParseOutput } from './parse.js'; export { routesPhase, type RoutesOutput, type RouteEntry } from './routes.js'; export { toolsPhase, type ToolsOutput, type ToolDef } from './tools.js'; export { ormPhase, type ORMOutput } from './orm.js'; +export { webhookPhase, type WebhookOutput } from './webhook.js'; export { crossFilePhase, type CrossFileOutput } from './cross-file.js'; export { mroPhase, type MROOutput } from './mro.js'; export { communitiesPhase, type CommunitiesOutput } from './communities.js'; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index f8e0d3b53..a252f0a89 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -68,6 +68,10 @@ import { fileURLToPath, pathToFileURL } from 'node:url'; import { isDev } from '../utils/env.js'; import { synthesizeWildcardImportBindings, needsSynthesis } from './wildcard-synthesis.js'; import { extractORMQueriesInline } from './orm-extraction.js'; +import { + extractWebhooks, + type ExtractedWebhook, +} from '../workers/parse-worker.js'; // ── Constants ────────────────────────────────────────────────────────────── @@ -106,6 +110,7 @@ export async function runChunkedParseAndResolve( allDecoratorRoutes: ExtractedDecoratorRoute[]; allToolDefs: ExtractedToolDef[]; allORMQueries: ExtractedORMQuery[]; + allWebhooks: ExtractedWebhook[]; bindingAccumulator: BindingAccumulator; resolutionContext: ReturnType; usedWorkerPool: boolean; @@ -248,6 +253,7 @@ export async function runChunkedParseAndResolve( const allDecoratorRoutes: ExtractedDecoratorRoute[] = []; const allToolDefs: ExtractedToolDef[] = []; const allORMQueries: ExtractedORMQuery[] = []; + const allWebhooks: ExtractedWebhook[] = []; const deferredWorkerCalls: ExtractedCall[] = []; const deferredWorkerHeritage: ExtractedHeritage[] = []; const deferredConstructorBindings: FileConstructorBindings[] = []; @@ -393,6 +399,9 @@ export async function runChunkedParseAndResolve( if (chunkWorkerData.ormQueries?.length) { for (const item of chunkWorkerData.ormQueries) allORMQueries.push(item); } + if (chunkWorkerData.webhooks?.length) { + for (const item of chunkWorkerData.webhooks) allWebhooks.push(item); + } } else { await processImports(graph, chunkFiles, astCache, ctx, undefined, repoPath, allPaths); sequentialChunkPaths.push(chunkPaths); @@ -509,6 +518,7 @@ export async function runChunkedParseAndResolve( } for (const f of chunkFiles) { extractORMQueriesInline(f.path, f.content, allORMQueries); + extractWebhooks(f.path, f.content, allWebhooks); } astCache.clear(); cachedSequentialChunkFiles[chunkIdx] = []; @@ -589,6 +599,7 @@ export async function runChunkedParseAndResolve( allDecoratorRoutes, allToolDefs, allORMQueries, + allWebhooks, bindingAccumulator, resolutionContext: ctx, // Whether a worker pool was actually live for this run. False means the diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts index 6415cb6e5..881be3820 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts @@ -26,6 +26,7 @@ import type { ExtractedDecoratorRoute, ExtractedToolDef, ExtractedORMQuery, + ExtractedWebhook, } from '../workers/parse-worker.js'; import type { createResolutionContext } from '../model/resolution-context.js'; import { runChunkedParseAndResolve } from './parse-impl.js'; @@ -47,6 +48,7 @@ export interface ParseOutput { readonly allDecoratorRoutes: readonly ExtractedDecoratorRoute[]; readonly allToolDefs: readonly ExtractedToolDef[]; readonly allORMQueries: readonly ExtractedORMQuery[]; + readonly allWebhooks: readonly ExtractedWebhook[]; bindingAccumulator: BindingAccumulator; /** Resolution context from the parse phase — carries importMap, namedImportMap, etc. */ resolutionContext: ReturnType; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/webhook.ts b/gitnexus/src/core/ingestion/pipeline-phases/webhook.ts new file mode 100644 index 000000000..e0a0c7403 --- /dev/null +++ b/gitnexus/src/core/ingestion/pipeline-phases/webhook.ts @@ -0,0 +1,82 @@ +/** + * Phase: webhook + * + * Processes webhook/event handler definitions and creates Webhook nodes + * with TRIGGERS edges from the handler file. + * + * @deps parse + * @reads allWebhooks (from parse) + * @writes graph (Webhook nodes, TRIGGERS edges) + */ + +import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js'; +import { getPhaseOutput } from './types.js'; +import type { ParseOutput } from './parse.js'; +import { generateId } from '../../../lib/utils.js'; +import type { ExtractedWebhook } from '../workers/parse-worker.js'; +import type { KnowledgeGraph } from '../../graph/types.js'; +import { isDev } from '../utils/env.js'; + +export interface WebhookOutput { + webhookCount: number; +} + +export const webhookPhase: PipelinePhase = { + name: 'webhook', + deps: ['parse'], + + async execute( + ctx: PipelineContext, + deps: ReadonlyMap>, + ): Promise { + const { allWebhooks } = getPhaseOutput(deps, 'parse'); + + if (allWebhooks.length === 0) { + return { webhookCount: 0 }; + } + + return processWebhooks(ctx.graph, allWebhooks); + }, +}; + +function processWebhooks( + graph: KnowledgeGraph, + webhooks: readonly ExtractedWebhook[], +): WebhookOutput { + const seenWebhookNames = new Set(); + let webhookCount = 0; + + for (const wh of webhooks) { + if (seenWebhookNames.has(wh.name)) continue; + seenWebhookNames.add(wh.name); + + const webhookNodeId = generateId('Webhook', wh.name); + graph.addNode({ + id: webhookNodeId, + label: 'Webhook', + properties: { + name: wh.name, + filePath: wh.filePath, + kind: wh.kind, + eventTypes: wh.eventTypes, + }, + }); + + const handlerFileId = generateId('File', wh.filePath); + graph.addRelationship({ + id: generateId('TRIGGERS', `${handlerFileId}->${webhookNodeId}`), + sourceId: handlerFileId, + targetId: webhookNodeId, + type: 'TRIGGERS', + confidence: 1.0, + reason: `webhook-handler:${wh.kind}`, + }); + webhookCount++; + } + + if (isDev) { + console.log(`Webhook detection: ${webhookCount} webhooks detected`); + } + + return { webhookCount }; +} diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index fc04dc3c9..5e84eb70e 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -29,6 +29,7 @@ import { routesPhase, toolsPhase, ormPhase, + webhookPhase, crossFilePhase, mroPhase, communitiesPhase, @@ -56,6 +57,7 @@ export interface PipelineOptions { }; } + // ── Phase registry ───────────────────────────────────────────────────────── /** @@ -63,7 +65,7 @@ export interface PipelineOptions { * * Phase dependency graph: * - * scan → structure → [markdown, cobol] → parse → [routes, tools, orm] + * scan → structure → [markdown, cobol] → parse → [routes, tools, orm, webhook] * → crossFile → mro → communities → processes * * To add a new phase: create a file in pipeline-phases/, export the phase @@ -79,6 +81,7 @@ function buildPhaseList(options?: PipelineOptions): PipelinePhase[] { routesPhase, toolsPhase, ormPhase, + webhookPhase, crossFilePhase, ]; diff --git a/gitnexus/test/integration/webhook-detection.test.ts b/gitnexus/test/integration/webhook-detection.test.ts index 041d46178..05e18bdda 100644 --- a/gitnexus/test/integration/webhook-detection.test.ts +++ b/gitnexus/test/integration/webhook-detection.test.ts @@ -10,10 +10,9 @@ * Part A: Detection logic against test/fixtures/webhook-repo/ file contents * Part B: Seed-based LadybugDB persistence + webhook_map tool dispatch * - * NOTE: Part A tests extractWebhooks directly rather than via runPipelineFromRepo - * because the pipeline only spawns workers (which run extractWebhooks) for repos - * with >= 15 files or >= 512KB of source. The 3-file fixture falls below both - * thresholds, triggering the sequential fallback which doesn't extract webhooks. + * NOTE: Part A tests extractWebhooks directly against fixture files for fast, + * isolated validation. The pipeline calls extractWebhooks in both the worker + * path AND the sequential fallback path (for repos with <15 files / <512KB). */ import { describe, it, expect, beforeAll, vi } from 'vitest'; import fs from 'fs';