From c5b9a1c11a6e55223a57357e7f9d3176f6640549 Mon Sep 17 00:00:00 2001
From: Dinh Huy <huynd86@fpt.com>
Date: Tue, 26 May 2026 16:48:42 +0700
Subject: [PATCH] [+] Update ingestion

---
 .../src/core/ingestion/parsing-processor.ts   | 50 +++++++++++++++++++
 .../ingestion/pipeline-phases/parse-impl.ts   |  4 +-
 .../core/ingestion/route-extractors/django.ts |  1 +
 3 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts
index 6465a9782..dd714921d 100644
--- a/gitnexus/src/core/ingestion/parsing-processor.ts
+++ b/gitnexus/src/core/ingestion/parsing-processor.ts
@@ -59,6 +59,15 @@ import {
   getTreeSitterContentByteLength,
   TREE_SITTER_MAX_BUFFER,
 } from './constants.js';
+import {
+  extractDjangoRoutes,
+  setDjangoParser,
+  type DjangoFileReader,
+} from './route-extractors/django.js';
+import { discoverDjangoRootUrl } from './route-extractors/django-root-discovery.js';
+import { extractLaravelRoutes } from './route-extractors/laravel.js';
+import fs from 'node:fs';
+import path from 'node:path';
 
 export type FileProgressCallback = (current: number, total: number, filePath: string) => void;
 
@@ -363,12 +372,18 @@ const processParsingSequential = async (
   astCache: ASTCache,
   scopeTreeCache: ASTCache | undefined,
   onFileProgress?: FileProgressCallback,
+  outRoutes?: ExtractedRoute[],
 ) => {
   const parser = await loadParser();
   const total = files.length;
   const logSkipped = isVerboseIngestionEnabled();
   const skippedByLang = logSkipped ? new Map<string, number>() : null;
 
+  // Pre-compute file content map and discover Django root URL for route extraction
+  const fileContentMap = new Map<string, string>();
+  for (const f of files) fileContentMap.set(f.path, f.content);
+  const djangoRootUrlFile = discoverDjangoRootUrl(files, fileContentMap);
+
   for (let i = 0; i < files.length; i++) {
     const file = files[i];
 
@@ -825,6 +840,32 @@ const processParsingSequential = async (
         });
       }
     });
+
+    // ── Route extraction (Django / Laravel) ──
+    // Replicates the per-file route extraction from parse-worker.ts processFileGroup.
+    const isRouteFile = provider.isRouteFile?.(file.path) ?? false;
+    if (isRouteFile) {
+      if (language === SupportedLanguages.Python) {
+        const isDjangoRoot =
+          djangoRootUrlFile !== null ? file.path === djangoRootUrlFile : isRouteFile;
+        if (!isDjangoRoot) continue;
+        setDjangoParser(parser);
+        const djangoReader: DjangoFileReader = (relativePath: string) => {
+          const cached = fileContentMap.get(relativePath);
+          if (cached != null) return cached;
+          try {
+            return fs.readFileSync(path.join(process.cwd(), relativePath), 'utf-8');
+          } catch {
+            return null;
+          }
+        };
+        const extractedRoutes = extractDjangoRoutes(tree, file.path, djangoReader);
+        for (const r of extractedRoutes) outRoutes?.push(r);
+      } else {
+        const extractedRoutes = extractLaravelRoutes(tree, file.path);
+        for (const r of extractedRoutes) outRoutes?.push(r);
+      }
+    }
   }
 
   if (skippedByLang && skippedByLang.size > 0) {
@@ -871,6 +912,14 @@ export const processParsing = async (
    * artifact to cache there). See `gitnexus/src/storage/parse-cache.ts`.
    */
   outRawResults?: ParseWorkerResult[],
+  /**
+   * Optional out-parameter for extracted routes from the sequential
+   * fallback path. The worker path returns routes inside the
+   * `WorkerExtractedData` return value; the sequential path writes
+   * them here so the caller can feed them into the deferred route
+   * processing pipeline (`processRoutesFromExtracted`).
+   */
+  outRoutes?: ExtractedRoute[],
 ): Promise<WorkerExtractedData | null> => {
   let lastProgress = 0;
   const reportProgress: FileProgressCallback | undefined = onFileProgress
@@ -969,6 +1018,7 @@ export const processParsing = async (
     astCache,
     scopeTreeCache,
     reportProgress,
+    outRoutes,
   );
   return null;
 };
diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts
index 4d06b8d58..02bd9b2aa 100644
--- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts
+++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts
@@ -519,9 +519,8 @@ export async function runChunkedParseAndResolve(
             scopeTreeCache,
             progressForChunk,
             activeWorkerPool,
-            // Capture raw results only when we have a cache to write to —
-            // otherwise we'd retain extra arrays for nothing.
             parseCache && chunkHash && activeWorkerPool ? rawResults : undefined,
+            allExtractedRoutes,
           );
         } catch (err) {
           if (!(err instanceof WorkerPoolInitializationError)) throw err;
@@ -546,6 +545,7 @@ export async function runChunkedParseAndResolve(
             progressForChunk,
             undefined,
             undefined,
+            allExtractedRoutes,
           );
         }
         // Persist the raw results for this chunk hash. Sequential path
diff --git a/gitnexus/src/core/ingestion/route-extractors/django.ts b/gitnexus/src/core/ingestion/route-extractors/django.ts
index 63b093856..49c874ae9 100644
--- a/gitnexus/src/core/ingestion/route-extractors/django.ts
+++ b/gitnexus/src/core/ingestion/route-extractors/django.ts
@@ -1,4 +1,5 @@
 import type Parser from 'tree-sitter';
+import { parseSourceSafe } from '../../tree-sitter/safe-parse.js';
 import { extractStringContent, type SyntaxNode } from '../utils/ast-helpers.js';
 import type { ExtractedRoute } from './laravel.js';