mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-09-24 00:51:53 +00:00
fix(markdown): handle CRLF line endings in section heading parser
split('\n') on CRLF content leaves a trailing \r on each line, and the
heading regex /^(#{1,6})\s+(.+)$/ (anchored with $) fails to match
'## Heading\r' because $ matches before end-of-string, not before \r.
Result: Windows-authored markdown silently produces zero Section nodes.
Use split(/\r\n|\r|\n/) to normalize all line-ending conventions.
Pure additive — LF-only files produce identical output. CR-only (Mac OS
Classic) becomes tolerated as a side benefit at zero risk.
Adds integration test markdown-processor-crlf.test.ts covering LF
baseline, CRLF (the regression), CR-only, mixed, and startLine/endLine
correctness.
This commit is contained in:
parent
a2517e050d
commit
943166580c
2 changed files with 94 additions and 1 deletions
|
|
@ -36,7 +36,12 @@ export const processMarkdown = (
|
|||
// Skip if file node doesn't exist (shouldn't happen, structure-processor creates it)
|
||||
if (!graph.getNode(fileNodeId)) continue;
|
||||
|
||||
const lines = file.content.split('\n');
|
||||
// Normalize CRLF/CR to LF before splitting so that line-end agnostic
|
||||
// markdown files (Windows-authored, mixed) yield correct headings.
|
||||
// Without this, `## Heading\r\n` produces `## Heading\r` after split,
|
||||
// and the HEADING_RE regex (anchored with `$`) fails to match because
|
||||
// `$` matches before end-of-string, not before `\r`.
|
||||
const lines = file.content.split(/\r\n|\r|\n/);
|
||||
|
||||
// --- Extract headings and build hierarchy ---
|
||||
// First pass: collect all heading positions so we can compute endLine spans
|
||||
|
|
|
|||
88
gitnexus/test/integration/markdown-processor-crlf.test.ts
Normal file
88
gitnexus/test/integration/markdown-processor-crlf.test.ts
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
/**
|
||||
* Regression test for CRLF-encoded markdown heading extraction.
|
||||
*
|
||||
* Files with CRLF line endings (Windows-authored markdown) previously
|
||||
* produced zero Section nodes because `split('\n')` left a trailing `\r`
|
||||
* on each line, and the heading regex `/^(#{1,6})\s+(.+)$/` (anchored
|
||||
* with `$`) failed to match `## Heading\r` (since `$` matches before
|
||||
* end-of-string, not before `\r`).
|
||||
*
|
||||
* Fix: split on `/\r\n|\r|\n/` so all line-ending conventions are
|
||||
* normalized at split time. See markdown-processor.ts line 39.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { processMarkdown } from '../../src/core/ingestion/markdown-processor.js';
|
||||
import { createKnowledgeGraph } from '../../src/core/graph/graph.js';
|
||||
import { generateId } from '../../src/lib/utils.js';
|
||||
import type { GraphNode } from 'gitnexus-shared';
|
||||
|
||||
function setupGraphWithFile(filePath: string) {
|
||||
const graph = createKnowledgeGraph();
|
||||
const fileNode: GraphNode = {
|
||||
id: generateId('File', filePath),
|
||||
label: 'File',
|
||||
properties: { name: filePath, filePath },
|
||||
};
|
||||
graph.addNode(fileNode);
|
||||
return graph;
|
||||
}
|
||||
|
||||
describe('markdown-processor CRLF tolerance', () => {
|
||||
it('extracts headings from LF-encoded markdown (baseline)', () => {
|
||||
const filePath = 'lf.md';
|
||||
const graph = setupGraphWithFile(filePath);
|
||||
const content = '# Title\nbody line 1\n## Sub\nbody line 2\n### SubSub\nmore\n';
|
||||
|
||||
const stats = processMarkdown(graph, [{ path: filePath, content }], new Set([filePath]));
|
||||
|
||||
expect(stats.sections).toBe(3);
|
||||
});
|
||||
|
||||
it('extracts headings from CRLF-encoded markdown (the regression)', () => {
|
||||
const filePath = 'crlf.md';
|
||||
const graph = setupGraphWithFile(filePath);
|
||||
const content = '# Title\r\nbody line 1\r\n## Sub\r\nbody line 2\r\n### SubSub\r\nmore\r\n';
|
||||
|
||||
const stats = processMarkdown(graph, [{ path: filePath, content }], new Set([filePath]));
|
||||
|
||||
// Pre-fix: this returned 0 because `## Sub\r` failed the heading regex.
|
||||
expect(stats.sections).toBe(3);
|
||||
});
|
||||
|
||||
it('extracts headings from CR-only-encoded markdown (old Mac OS Classic)', () => {
|
||||
const filePath = 'cr.md';
|
||||
const graph = setupGraphWithFile(filePath);
|
||||
const content = '# Title\rbody line 1\r## Sub\rbody line 2\r';
|
||||
|
||||
const stats = processMarkdown(graph, [{ path: filePath, content }], new Set([filePath]));
|
||||
|
||||
expect(stats.sections).toBe(2);
|
||||
});
|
||||
|
||||
it('extracts headings from mixed CRLF + LF markdown', () => {
|
||||
const filePath = 'mixed.md';
|
||||
const graph = setupGraphWithFile(filePath);
|
||||
const content = '# LF Title\nbody\r\n## CRLF Sub\r\nmore\n### Trailing LF\nend\n';
|
||||
|
||||
const stats = processMarkdown(graph, [{ path: filePath, content }], new Set([filePath]));
|
||||
|
||||
expect(stats.sections).toBe(3);
|
||||
});
|
||||
|
||||
it('reports correct startLine and endLine for CRLF content', () => {
|
||||
const filePath = 'crlf-lines.md';
|
||||
const graph = setupGraphWithFile(filePath);
|
||||
// Lines 1, 3, 5 are headings (1-indexed)
|
||||
const content = '# T\r\nbody\r\n## Sub\r\nmore\r\n### SubSub\r\ntail\r\n';
|
||||
|
||||
processMarkdown(graph, [{ path: filePath, content }], new Set([filePath]));
|
||||
|
||||
const sections = Array.from(graph.iterNodes()).filter((n) => n.label === 'Section');
|
||||
const titleSection = sections.find((s) => s.properties.name === 'T');
|
||||
const subSection = sections.find((s) => s.properties.name === 'Sub');
|
||||
|
||||
expect(titleSection?.properties.startLine).toBe(1);
|
||||
expect(subSection?.properties.startLine).toBe(3);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue