From 943166580c8bf3ac263fc9bb08757d5ca39413ac Mon Sep 17 00:00:00 2001 From: Ash Date: Sat, 9 May 2026 19:52:12 -0700 Subject: [PATCH] fix(markdown): handle CRLF line endings in section heading parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit split('\n') on CRLF content leaves a trailing \r on each line, and the heading regex /^(#{1,6})\s+(.+)$/ (anchored with $) fails to match '## Heading\r' because $ matches before end-of-string, not before \r. Result: Windows-authored markdown silently produces zero Section nodes. Use split(/\r\n|\r|\n/) to normalize all line-ending conventions. Pure additive — LF-only files produce identical output. CR-only (Mac OS Classic) becomes tolerated as a side benefit at zero risk. Adds integration test markdown-processor-crlf.test.ts covering LF baseline, CRLF (the regression), CR-only, mixed, and startLine/endLine correctness. --- .../src/core/ingestion/markdown-processor.ts | 7 +- .../markdown-processor-crlf.test.ts | 88 +++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 gitnexus/test/integration/markdown-processor-crlf.test.ts diff --git a/gitnexus/src/core/ingestion/markdown-processor.ts b/gitnexus/src/core/ingestion/markdown-processor.ts index dc0e7fee4..f44fbba42 100644 --- a/gitnexus/src/core/ingestion/markdown-processor.ts +++ b/gitnexus/src/core/ingestion/markdown-processor.ts @@ -36,7 +36,12 @@ export const processMarkdown = ( // Skip if file node doesn't exist (shouldn't happen, structure-processor creates it) if (!graph.getNode(fileNodeId)) continue; - const lines = file.content.split('\n'); + // Normalize CRLF/CR to LF before splitting so that line-end agnostic + // markdown files (Windows-authored, mixed) yield correct headings. + // Without this, `## Heading\r\n` produces `## Heading\r` after split, + // and the HEADING_RE regex (anchored with `$`) fails to match because + // `$` matches before end-of-string, not before `\r`. + const lines = file.content.split(/\r\n|\r|\n/); // --- Extract headings and build hierarchy --- // First pass: collect all heading positions so we can compute endLine spans diff --git a/gitnexus/test/integration/markdown-processor-crlf.test.ts b/gitnexus/test/integration/markdown-processor-crlf.test.ts new file mode 100644 index 000000000..8d1a850e5 --- /dev/null +++ b/gitnexus/test/integration/markdown-processor-crlf.test.ts @@ -0,0 +1,88 @@ +/** + * Regression test for CRLF-encoded markdown heading extraction. + * + * Files with CRLF line endings (Windows-authored markdown) previously + * produced zero Section nodes because `split('\n')` left a trailing `\r` + * on each line, and the heading regex `/^(#{1,6})\s+(.+)$/` (anchored + * with `$`) failed to match `## Heading\r` (since `$` matches before + * end-of-string, not before `\r`). + * + * Fix: split on `/\r\n|\r|\n/` so all line-ending conventions are + * normalized at split time. See markdown-processor.ts line 39. + */ + +import { describe, it, expect } from 'vitest'; +import { processMarkdown } from '../../src/core/ingestion/markdown-processor.js'; +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { generateId } from '../../src/lib/utils.js'; +import type { GraphNode } from 'gitnexus-shared'; + +function setupGraphWithFile(filePath: string) { + const graph = createKnowledgeGraph(); + const fileNode: GraphNode = { + id: generateId('File', filePath), + label: 'File', + properties: { name: filePath, filePath }, + }; + graph.addNode(fileNode); + return graph; +} + +describe('markdown-processor CRLF tolerance', () => { + it('extracts headings from LF-encoded markdown (baseline)', () => { + const filePath = 'lf.md'; + const graph = setupGraphWithFile(filePath); + const content = '# Title\nbody line 1\n## Sub\nbody line 2\n### SubSub\nmore\n'; + + const stats = processMarkdown(graph, [{ path: filePath, content }], new Set([filePath])); + + expect(stats.sections).toBe(3); + }); + + it('extracts headings from CRLF-encoded markdown (the regression)', () => { + const filePath = 'crlf.md'; + const graph = setupGraphWithFile(filePath); + const content = '# Title\r\nbody line 1\r\n## Sub\r\nbody line 2\r\n### SubSub\r\nmore\r\n'; + + const stats = processMarkdown(graph, [{ path: filePath, content }], new Set([filePath])); + + // Pre-fix: this returned 0 because `## Sub\r` failed the heading regex. + expect(stats.sections).toBe(3); + }); + + it('extracts headings from CR-only-encoded markdown (old Mac OS Classic)', () => { + const filePath = 'cr.md'; + const graph = setupGraphWithFile(filePath); + const content = '# Title\rbody line 1\r## Sub\rbody line 2\r'; + + const stats = processMarkdown(graph, [{ path: filePath, content }], new Set([filePath])); + + expect(stats.sections).toBe(2); + }); + + it('extracts headings from mixed CRLF + LF markdown', () => { + const filePath = 'mixed.md'; + const graph = setupGraphWithFile(filePath); + const content = '# LF Title\nbody\r\n## CRLF Sub\r\nmore\n### Trailing LF\nend\n'; + + const stats = processMarkdown(graph, [{ path: filePath, content }], new Set([filePath])); + + expect(stats.sections).toBe(3); + }); + + it('reports correct startLine and endLine for CRLF content', () => { + const filePath = 'crlf-lines.md'; + const graph = setupGraphWithFile(filePath); + // Lines 1, 3, 5 are headings (1-indexed) + const content = '# T\r\nbody\r\n## Sub\r\nmore\r\n### SubSub\r\ntail\r\n'; + + processMarkdown(graph, [{ path: filePath, content }], new Set([filePath])); + + const sections = Array.from(graph.iterNodes()).filter((n) => n.label === 'Section'); + const titleSection = sections.find((s) => s.properties.name === 'T'); + const subSection = sections.find((s) => s.properties.name === 'Sub'); + + expect(titleSection?.properties.startLine).toBe(1); + expect(subSection?.properties.startLine).toBe(3); + }); +});