Merge branch 'RooCodeInc:main' into main

2026-09-06 08:18:39 +00:00 · 2025-07-04 15:04:35 -03:00 · 2025-07-04 15:04:35 -03:00 · f5a51c452d
commit f5a51c452d
parent ef61905497 a83e8c0553
6 changed files with 1023 additions and 53 deletions
--- a/src/services/code-index/processors/tests/parser.spec.ts
+++ b/src/services/code-index/processors/tests/parser.spec.ts
@ -2,6 +2,7 @@

 import { CodeParser, codeParser } from "../parser"
 import { loadRequiredLanguageParsers } from "../../../tree-sitter/languageParser"
+import { parseMarkdown } from "../../../tree-sitter/markdownParser"
 import { readFile } from "fs/promises"
 import { Node } from "web-tree-sitter"

@ -23,6 +24,7 @@ vi.mock("fs/promises", () => ({
 }))

 vi.mock("../../../tree-sitter/languageParser")
+vi.mock("../../../tree-sitter/markdownParser")

 const mockLanguageParser = {
 	js: {
@ -242,4 +244,715 @@ describe("CodeParser", () => {
 			expect(result2).toBeDefined()
 		})
 	})
+
+	describe("Markdown Support", () => {
+		beforeEach(() => {
+			vi.clearAllMocks()
+		})
+
+		it("should generate unique segment hashes for each markdown block", async () => {
+			const markdownContent = `# Section One
+This is a section with substantial content that meets the minimum character requirements.
+It contains detailed information and multiple paragraphs to ensure proper indexing.
+The content is comprehensive and provides valuable information for search functionality.
+
+## Section Two
+Another section with different content but also meeting the minimum requirements.
+This ensures we can test that different sections get different segment hashes.
+Each section should have its own unique hash based on its content.`
+
+			vi.mocked(parseMarkdown).mockReturnValue([
+				{
+					node: { startPosition: { row: 0 }, endPosition: { row: 4 }, text: "Section One" },
+					name: "name.definition.header.h1",
+					patternIndex: 0,
+				},
+				{
+					node: { startPosition: { row: 0 }, endPosition: { row: 4 }, text: "Section One" },
+					name: "definition.header.h1",
+					patternIndex: 0,
+				},
+				{
+					node: { startPosition: { row: 5 }, endPosition: { row: 8 }, text: "Section Two" },
+					name: "name.definition.header.h2",
+					patternIndex: 0,
+				},
+				{
+					node: { startPosition: { row: 5 }, endPosition: { row: 8 }, text: "Section Two" },
+					name: "definition.header.h2",
+					patternIndex: 0,
+				},
+			] as any)
+
+			const result = await parser.parseFile("test.md", { content: markdownContent })
+
+			expect(result).toHaveLength(2)
+
+			// Verify each block has unique segment hash
+			expect(result[0].segmentHash).toMatch(/^[a-f0-9]{64}$/)
+			expect(result[1].segmentHash).toMatch(/^[a-f0-9]{64}$/)
+			expect(result[0].segmentHash).not.toBe(result[1].segmentHash)
+
+			// Verify file hash is consistent
+			expect(result[0].fileHash).toBe(result[1].fileHash)
+			expect(result[0].fileHash).toMatch(/^[a-f0-9]{64}$/)
+		})
+
+		it("should use fallback chunking for markdown files without headers", async () => {
+			const markdownContent = `This is a markdown file without any headers but with substantial content.
+It contains multiple paragraphs and detailed information that should be indexed.
+The content is long enough to meet the minimum character requirements for fallback chunking.
+This ensures that even headerless markdown files can be properly indexed and searched.
+Additional content to ensure we exceed the minimum block size requirements for proper indexing.`
+
+			vi.mocked(parseMarkdown).mockReturnValue([])
+
+			const result = await parser.parseFile("test.md", { content: markdownContent })
+
+			expect(parseMarkdown).toHaveBeenCalledWith(markdownContent)
+			expect(result).toHaveLength(1)
+			expect(result[0].type).toBe("markdown_content")
+			expect(result[0].content).toBe(markdownContent)
+			expect(result[0].start_line).toBe(1)
+
+			// Verify hash generation for fallback chunks
+			expect(result[0].segmentHash).toMatch(/^[a-f0-9]{64}$/)
+			expect(result[0].fileHash).toMatch(/^[a-f0-9]{64}$/)
+		})
+
+		it("should chunk large markdown files with no headers", async () => {
+			// Create a large markdown file without headers (2000+ chars)
+			const lines = []
+			for (let i = 0; i < 80; i++) {
+				lines.push(`This is line ${i} with substantial content to ensure proper chunking behavior.`)
+			}
+			const largeMarkdownContent = lines.join("\n") // ~80 lines * ~78 chars = ~6240 chars
+
+			vi.mocked(parseMarkdown).mockReturnValue([])
+
+			const result = await parser.parseFile("test.md", { content: largeMarkdownContent })
+
+			expect(parseMarkdown).toHaveBeenCalledWith(largeMarkdownContent)
+			// Should have multiple chunks due to size
+			expect(result.length).toBeGreaterThan(1)
+			// All chunks should be of type markdown_content
+			result.forEach((block) => {
+				expect(block.type).toBe("markdown_content")
+				expect(block.identifier).toBeNull()
+				// Each chunk should respect MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR
+				expect(block.content.length).toBeLessThanOrEqual(1150)
+			})
+			// Verify chunks cover the entire content
+			const totalLines = result.reduce((acc, block) => {
+				return acc + (block.end_line - block.start_line + 1)
+			}, 0)
+			expect(totalLines).toBe(80)
+		})
+
+		it("should enforce MIN_BLOCK_CHARS for all markdown sections", async () => {
+			const markdownContent = `# Short
+Small content.
+
+## Another Short
+Also small.
+
+### Long Section
+This section has substantial content that exceeds the minimum character requirements.
+It includes multiple lines with detailed information to ensure proper indexing.
+The content is comprehensive enough to be included in the search results.`
+
+			vi.mocked(parseMarkdown).mockReturnValue([
+				{
+					node: { startPosition: { row: 0 }, endPosition: { row: 1 }, text: "Short" },
+					name: "name.definition.header.h1",
+					patternIndex: 0,
+				},
+				{
+					node: { startPosition: { row: 0 }, endPosition: { row: 1 }, text: "Short" },
+					name: "definition.header.h1",
+					patternIndex: 0,
+				},
+				{
+					node: { startPosition: { row: 3 }, endPosition: { row: 4 }, text: "Another Short" },
+					name: "name.definition.header.h2",
+					patternIndex: 0,
+				},
+				{
+					node: { startPosition: { row: 3 }, endPosition: { row: 4 }, text: "Another Short" },
+					name: "definition.header.h2",
+					patternIndex: 0,
+				},
+				{
+					node: { startPosition: { row: 6 }, endPosition: { row: 9 }, text: "Long Section" },
+					name: "name.definition.header.h3",
+					patternIndex: 0,
+				},
+				{
+					node: { startPosition: { row: 6 }, endPosition: { row: 9 }, text: "Long Section" },
+					name: "definition.header.h3",
+					patternIndex: 0,
+				},
+			] as any)
+
+			const result = await parser.parseFile("test.md", { content: markdownContent })
+
+			// Only the long section should be included
+			expect(result).toHaveLength(1)
+			expect(result[0].identifier).toBe("Long Section")
+			expect(result[0].content.length).toBeGreaterThanOrEqual(100) // MIN_BLOCK_CHARS
+		})
+
+		it("should chunk large markdown sections and generate unique hashes for each chunk", async () => {
+			// Create content with multiple lines
+			const lines = []
+			// Add header
+			lines.push("# Large Section Header")
+			// Add 50 lines of content, each ~30 chars = ~1500 chars total
+			for (let i = 0; i < 50; i++) {
+				lines.push(`This is line ${i} with some content.`)
+			}
+
+			const markdownContent = lines.join("\n")
+
+			// The mock should return sections that span the actual content
+			vi.mocked(parseMarkdown).mockReturnValue([
+				{
+					node: {
+						startPosition: { row: 0 },
+						endPosition: { row: 50 }, // Header + 50 lines of content
+						text: "Large Section Header",
+					},
+					name: "name.definition.header.h1",
+					patternIndex: 0,
+				},
+				{
+					node: {
+						startPosition: { row: 0 },
+						endPosition: { row: 50 }, // Header + 50 lines of content
+						text: markdownContent, // Full section content
+					},
+					name: "definition.header.h1",
+					patternIndex: 0,
+				},
+			] as any)
+
+			const result = await parser.parseFile("test.md", { content: markdownContent })
+
+			// Large section should be chunked into multiple blocks
+			const h1Blocks = result.filter((r) => r.type === "markdown_header_h1")
+			expect(h1Blocks.length).toBeGreaterThan(1)
+
+			// Each chunk should have a unique segment hash
+			const segmentHashes = h1Blocks.map((block) => block.segmentHash)
+			const uniqueHashes = new Set(segmentHashes)
+			expect(uniqueHashes.size).toBe(h1Blocks.length)
+
+			// All chunks should preserve the header identifier
+			h1Blocks.forEach((block) => {
+				expect(block.identifier).toBe("Large Section Header")
+				// Each chunk should respect MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR
+				expect(block.content.length).toBeLessThanOrEqual(1150)
+				// Each chunk should have valid hashes
+				expect(block.segmentHash).toMatch(/^[a-f0-9]{64}$/)
+				expect(block.fileHash).toMatch(/^[a-f0-9]{64}$/)
+			})
+		})
+
+		it("should handle markdown with very long single lines with chunking", async () => {
+			const veryLongLine = "a".repeat(2000) // Single line exceeding max chars
+			const markdownContent = `# Section with Long Line
+Normal content here.
+${veryLongLine}
+More normal content.`
+
+			vi.mocked(parseMarkdown).mockReturnValue([
+				{
+					node: { startPosition: { row: 0 }, endPosition: { row: 3 }, text: "Section with Long Line" },
+					name: "name.definition.header.h1",
+					patternIndex: 0,
+				},
+				{
+					node: { startPosition: { row: 0 }, endPosition: { row: 3 }, text: markdownContent },
+					name: "definition.header.h1",
+					patternIndex: 0,
+				},
+			] as any)
+
+			const result = await parser.parseFile("test.md", { content: markdownContent })
+
+			// Should create multiple blocks due to chunking
+			expect(result.length).toBeGreaterThan(1)
+			// Should have segment blocks for the oversized line
+			const segmentBlocks = result.filter((r) => r.type === "markdown_header_h1_segment")
+			expect(segmentBlocks.length).toBeGreaterThan(0)
+			// All blocks should preserve the header identifier
+			result.forEach((block) => {
+				expect(block.identifier).toBe("Section with Long Line")
+			})
+		})
+
+		it("should preserve header information when chunking large sections", async () => {
+			const largeContent = Array(100).fill("Line with substantial content to ensure proper handling.").join("\n")
+			const markdownContent = `### Deep Header Level 3
+${largeContent}`
+
+			vi.mocked(parseMarkdown).mockReturnValue([
+				{
+					node: { startPosition: { row: 0 }, endPosition: { row: 100 }, text: "Deep Header Level 3" },
+					name: "name.definition.header.h3",
+					patternIndex: 0,
+				},
+				{
+					node: { startPosition: { row: 0 }, endPosition: { row: 100 }, text: markdownContent },
+					name: "definition.header.h3",
+					patternIndex: 0,
+				},
+			] as any)
+
+			const result = await parser.parseFile("test.md", { content: markdownContent })
+
+			// Should have multiple blocks due to chunking
+			expect(result.length).toBeGreaterThan(1)
+			// All blocks should have the same type and identifier
+			result.forEach((block) => {
+				expect(block.type).toBe("markdown_header_h3")
+				expect(block.identifier).toBe("Deep Header Level 3")
+			})
+		})
+
+		it("should apply chunking logic based on MAX_BLOCK_CHARS and re-balancing", async () => {
+			// Create content that will trigger re-balancing logic
+			// 60 lines * 30 chars = 1800 chars, which should trigger chunking
+			const lines = []
+			for (let i = 0; i < 60; i++) {
+				lines.push(`Line ${i}: Some content here to test.`) // ~30 chars per line
+			}
+			const markdownContent = lines.join("\n")
+
+			vi.mocked(parseMarkdown).mockReturnValue([])
+
+			const result = await parser.parseFile("test.md", { content: markdownContent })
+
+			// Should have multiple chunks due to size
+			expect(result.length).toBeGreaterThan(1)
+
+			// Verify re-balancing: chunks should be roughly equal in size
+			const chunkSizes = result.map((block) => block.content.length)
+			const avgSize = chunkSizes.reduce((a, b) => a + b, 0) / chunkSizes.length
+
+			chunkSizes.forEach((size) => {
+				// Each chunk should be within 30% of average size (re-balanced)
+				expect(Math.abs(size - avgSize) / avgSize).toBeLessThan(0.3)
+				// Each chunk should respect MIN_BLOCK_CHARS
+				expect(size).toBeGreaterThanOrEqual(100)
+			})
+
+			// Verify each chunk has unique segment hash
+			const segmentHashes = result.map((block) => block.segmentHash)
+			expect(new Set(segmentHashes).size).toBe(result.length)
+		})
+
+		it("should handle markdown content before the first header", async () => {
+			const preHeaderContent = `This is content before any headers that contains substantial information.
+It has multiple lines and should be indexed because it meets the minimum size requirements.
+This content contains important documentation that would be lost without proper handling.
+We need to ensure that all content is captured, not just content within header sections.
+This paragraph continues with more details to ensure we exceed the minimum block size.`
+
+			const headerContent = `# First Header
+
+Content under the first header with enough text to be indexed properly.
+This section contains multiple lines to ensure it meets the minimum character requirements.
+We need at least 100 characters for a section to be included in the index.
+This additional content ensures the header section will be processed correctly.`
+
+			const markdownContent = `${preHeaderContent}
+
+${headerContent}`
+
+			// Mock the parseMarkdown function to return headers
+			// The header section spans from line 6 to line 10 (5 lines total)
+			vi.mocked(parseMarkdown).mockReturnValue([
+				{
+					node: {
+						startPosition: { row: 6 },
+						endPosition: { row: 10 },
+						text: "First Header",
+					},
+					name: "name.definition.header.h1",
+					patternIndex: 0,
+				} as any,
+				{
+					node: {
+						startPosition: { row: 6 },
+						endPosition: { row: 10 },
+						text: "First Header",
+					},
+					name: "definition.header.h1",
+					patternIndex: 0,
+				} as any,
+			])
+
+			const result = await parser.parseFile("test.md", { content: markdownContent })
+
+			// Should have exactly 2 blocks: pre-header content and header section
+			expect(result.length).toBe(2)
+
+			// First block should be the content before the header
+			expect(result[0]).toMatchObject({
+				file_path: "test.md",
+				type: "markdown_content",
+				start_line: 1,
+				end_line: 6, // Up to the header line
+			})
+			expect(result[0].content).toContain("This is content before any headers")
+
+			// Second block should be the header section
+			expect(result[1]).toMatchObject({
+				file_path: "test.md",
+				identifier: "First Header",
+				type: "markdown_header_h1",
+				start_line: 7,
+				end_line: 11,
+			})
+		})
+
+		it("should handle markdown content after the last header", async () => {
+			const markdownContent = `# Header
+
+Header content with enough text to meet the minimum requirements for proper indexing.
+This header section needs to have at least 100 characters to be included in the results.
+We're adding this extra line to ensure the header section meets the minimum size threshold.
+
+This is content after the last header that contains substantial documentation.
+It has multiple lines and should be indexed because it's important information.
+This content would be lost without proper handling of content outside header sections.
+We're adding more content here to ensure we meet the minimum block size requirements.
+This ensures that trailing content in markdown files is properly captured and indexed.`
+
+			// Mock the parseMarkdown function to return headers
+			// The header section spans from line 0 to line 4 (5 lines)
+			vi.mocked(parseMarkdown).mockReturnValue([
+				{
+					node: {
+						startPosition: { row: 0 },
+						endPosition: { row: 4 },
+						text: "Header",
+					},
+					name: "name.definition.header.h1",
+					patternIndex: 0,
+				} as any,
+				{
+					node: {
+						startPosition: { row: 0 },
+						endPosition: { row: 4 },
+						text: "Header",
+					},
+					name: "definition.header.h1",
+					patternIndex: 0,
+				} as any,
+			])
+
+			const result = await parser.parseFile("test.md", { content: markdownContent })
+
+			// Should have exactly 2 blocks: header section and post-header content
+			expect(result.length).toBe(2)
+
+			// First block should be the header section
+			expect(result[0]).toMatchObject({
+				file_path: "test.md",
+				identifier: "Header",
+				type: "markdown_header_h1",
+				start_line: 1,
+				end_line: 5,
+			})
+
+			// Second block should be the content after the header
+			expect(result[1]).toMatchObject({
+				file_path: "test.md",
+				type: "markdown_content",
+				start_line: 6,
+			})
+			expect(result[1].content).toContain("This is content after the last header")
+		})
+
+		it("should handle very long paragraphs with chunking", async () => {
+			// Create a very long paragraph
+			const longParagraph = "This is a very long paragraph that contains substantial content. ".repeat(50)
+			const markdownContent = `# Introduction
+
+Some intro text.
+
+${longParagraph}
+
+## Conclusion
+
+Final thoughts that need to be long enough to meet the minimum character requirement.
+This conclusion section contains multiple lines to ensure it exceeds 100 characters.`
+
+			const lines = markdownContent.split("\n")
+
+			vi.mocked(parseMarkdown).mockReturnValue([
+				{
+					node: {
+						startPosition: { row: 0 },
+						endPosition: { row: 0 },
+						text: "Introduction",
+					},
+					name: "name.definition.header.h1",
+					patternIndex: 0,
+				} as any,
+				{
+					node: {
+						startPosition: { row: 0 },
+						endPosition: { row: 4 },
+						text: "Introduction",
+					},
+					name: "definition.header.h1",
+					patternIndex: 0,
+				} as any,
+				{
+					node: {
+						startPosition: { row: 6 },
+						endPosition: { row: 6 },
+						text: "Conclusion",
+					},
+					name: "name.definition.header.h2",
+					patternIndex: 0,
+				} as any,
+				{
+					node: {
+						startPosition: { row: 6 },
+						endPosition: { row: 9 },
+						text: "Conclusion",
+					},
+					name: "definition.header.h2",
+					patternIndex: 0,
+				} as any,
+			])
+
+			const result = await parser.parseFile("test.md", { content: markdownContent })
+
+			// The introduction section should be chunked
+			const h1Blocks = result.filter(
+				(r) => r.type === "markdown_header_h1" || r.type === "markdown_header_h1_segment",
+			)
+			expect(h1Blocks.length).toBeGreaterThan(1)
+			// All chunks should preserve the identifier
+			h1Blocks.forEach((block) => {
+				expect(block.identifier).toBe("Introduction")
+			})
+
+			// Conclusion should be a single block
+			const h2Blocks = result.filter((r) => r.type === "markdown_header_h2")
+			expect(h2Blocks.length).toBe(1)
+		})
+
+		it("should continue processing after encountering a very long line", async () => {
+			// Create a markdown file with a very long single line followed by more content
+			const veryLongLine = "a".repeat(5000) // 5000 characters - exceeds MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR
+
+			// Create content that will be chunked
+			const markdownContent = `This is content before the very long line that should be properly indexed.
+It contains multiple lines to ensure it meets the minimum character requirements.
+We need enough content here to trigger the chunking behavior.
+
+${veryLongLine}
+
+This is content after the very long line that must also be properly indexed.
+It's critical that this content is not ignored due to the oversized line bug.
+We need to ensure all content is processed, not just content before the long line.
+Adding more content to ensure we meet minimum block requirements.`
+
+			// Mock parseMarkdown to return no headers (testing fallback chunking)
+			vi.mocked(parseMarkdown).mockReturnValue([])
+
+			const result = await parser.parseFile("test.md", { content: markdownContent })
+
+			// The content should be chunked due to the oversized line
+			expect(result.length).toBeGreaterThan(1)
+
+			// Should have segment blocks for the oversized line
+			const segmentBlocks = result.filter((r) => r.type.includes("_segment"))
+			expect(segmentBlocks.length).toBeGreaterThan(0)
+
+			// Verify that content after the long line is included
+			const lastBlock = result[result.length - 1]
+			expect(lastBlock.content).toContain("content after the very long line")
+
+			// Verify all segments are from the oversized line
+			segmentBlocks.forEach((block) => {
+				expect(block.content).toMatch(/^a+$/)
+			})
+		})
+
+		it("should handle multiple oversized lines in sequence", async () => {
+			// Test with multiple consecutive oversized lines
+			const longLine1 = "x".repeat(3000)
+			const longLine2 = "y".repeat(3000)
+			const longLine3 = "z".repeat(3000)
+
+			const markdownContent = `# Test Multiple Long Lines
+Normal content before the long lines.
+${longLine1}
+${longLine2}
+${longLine3}
+Normal content after the long lines that must be indexed.
+This content verifies that processing continues after multiple oversized lines.`
+
+			vi.mocked(parseMarkdown).mockReturnValue([
+				{
+					node: {
+						startPosition: { row: 0 },
+						endPosition: { row: 6 },
+						text: "Test Multiple Long Lines",
+					},
+					name: "name.definition.header.h1",
+					patternIndex: 0,
+				} as any,
+				{
+					node: {
+						startPosition: { row: 0 },
+						endPosition: { row: 6 },
+						text: "Test Multiple Long Lines",
+					},
+					name: "definition.header.h1",
+					patternIndex: 0,
+				} as any,
+			])
+
+			const result = await parser.parseFile("test.md", { content: markdownContent })
+
+			// Should have multiple segment blocks
+			const segmentBlocks = result.filter((r) => r.type === "markdown_header_h1_segment")
+			expect(segmentBlocks.length).toBeGreaterThan(6) // At least 3 segments per long line
+
+			// Should also have regular blocks for the normal content
+			const regularBlocks = result.filter((r) => r.type === "markdown_header_h1" && !r.type.includes("_segment"))
+			expect(regularBlocks.length).toBeGreaterThan(0)
+
+			// Verify the last block includes content after the long lines
+			const lastRegularBlock = regularBlocks[regularBlocks.length - 1]
+			expect(lastRegularBlock.content).toContain("Normal content after the long lines")
+		})
+	})
+
+	describe("Edge case: Single oversized line in markdown", () => {
+		beforeEach(() => {
+			vi.clearAllMocks()
+		})
+
+		it("should properly chunk a markdown file with a single very long line", async () => {
+			const parser = new CodeParser()
+			const veryLongLine = "x".repeat(5000) // 5000 chars in a single line
+
+			// Mock parseMarkdown to return empty array (no headers)
+			vi.mocked(parseMarkdown).mockReturnValue([])
+
+			const results = await parser["parseContent"]("test.md", veryLongLine, "test-hash")
+
+			// Should create multiple segments
+			expect(results.length).toBeGreaterThan(1)
+			expect(results.length).toBe(5) // 5000 / 1000 = 5 segments
+
+			// All chunks should be segments
+			const segments = results.filter((r) => r.type === "markdown_content_segment")
+			expect(segments.length).toBe(5)
+
+			// Verify content is preserved
+			const reconstructed = results.map((r) => r.content).join("")
+			expect(reconstructed).toBe(veryLongLine)
+
+			// Each segment (except possibly the last) should be MAX_BLOCK_CHARS (1000)
+			for (let i = 0; i < segments.length - 1; i++) {
+				expect(segments[i].content.length).toBe(1000)
+			}
+
+			// Last segment should have the remainder
+			expect(segments[segments.length - 1].content.length).toBe(1000)
+		})
+
+		it("should handle markdown with headers followed by oversized lines", async () => {
+			const parser = new CodeParser()
+			const longLineA = "a".repeat(2000)
+			const longLineB = "b".repeat(3000)
+			const content = `# Header 1\n\n${longLineA}\n\n## Header 2\n\n${longLineB}`
+
+			// Mock parseMarkdown to return headers
+			vi.mocked(parseMarkdown).mockReturnValue([
+				{
+					node: { startPosition: { row: 0 }, endPosition: { row: 2 }, text: "Header 1" },
+					name: "name.definition.header.h1",
+					patternIndex: 0,
+				},
+				{
+					node: { startPosition: { row: 0 }, endPosition: { row: 2 }, text: "Header 1" },
+					name: "definition.header.h1",
+					patternIndex: 0,
+				},
+				{
+					node: { startPosition: { row: 4 }, endPosition: { row: 6 }, text: "Header 2" },
+					name: "name.definition.header.h2",
+					patternIndex: 0,
+				},
+				{
+					node: { startPosition: { row: 4 }, endPosition: { row: 6 }, text: "Header 2" },
+					name: "definition.header.h2",
+					patternIndex: 0,
+				},
+			] as any)
+
+			const results = await parser["parseContent"]("test.md", content, "test-hash")
+
+			// Should create multiple chunks
+			expect(results.length).toBeGreaterThan(2)
+
+			// Should have both header chunks and segments
+			const headers = results.filter((r) => r.type.startsWith("markdown_header"))
+			const segments = results.filter((r) => r.type.includes("_segment"))
+
+			expect(headers.length).toBeGreaterThan(0)
+			expect(segments.length).toBeGreaterThan(0)
+
+			// Verify segments were created for oversized lines
+			// 2000 chars = 2 segments, 3000 chars = 3 segments
+			expect(segments.length).toBeGreaterThanOrEqual(5)
+		})
+
+		it("should not chunk markdown files with lines under the threshold", async () => {
+			const parser = new CodeParser()
+			const normalContent = "This is a normal line.\n".repeat(50) // Multiple normal lines
+			const totalLength = normalContent.length
+
+			// Mock parseMarkdown to return empty array (no headers)
+			vi.mocked(parseMarkdown).mockReturnValue([])
+
+			const results = await parser["parseContent"]("test.md", normalContent, "test-hash")
+
+			// Since total content is 1150 chars (23 * 50), it's just over the threshold
+			// But no individual line is oversized, so it depends on total length
+			if (totalLength > 1150) {
+				// Content exceeds threshold, should be chunked
+				expect(results.length).toBeGreaterThan(1)
+			} else {
+				// Content is under threshold, should be single chunk
+				expect(results.length).toBe(1)
+				expect(results[0].type).toBe("markdown_content")
+			}
+		})
+
+		it("should return empty array for markdown content below MIN_BLOCK_CHARS threshold", async () => {
+			const parser = new CodeParser()
+			const smallContent = "This is a small markdown file.\nWith just a few lines.\nNothing special."
+
+			// Mock parseMarkdown to return empty array (no headers)
+			vi.mocked(parseMarkdown).mockReturnValue([])
+
+			const results = await parser["parseContent"]("test.md", smallContent, "test-hash")
+
+			// Should return empty array since content (71 chars) is below MIN_BLOCK_CHARS (100)
+			expect(results.length).toBe(0)
+			expect(smallContent.length).toBeLessThan(100) // Verify our test assumption
+		})
+	})
 })
--- a/src/services/code-index/processors/tests/scanner.spec.ts
+++ b/src/services/code-index/processors/tests/scanner.spec.ts
@ -241,5 +241,141 @@ describe("DirectoryScanner", () => {
 			// Verify the stats
 			expect(mockCodeParser.parseFile).toHaveBeenCalledTimes(2)
 		})
+
+		it("should process markdown files alongside code files", async () => {
+			const { listFiles } = await import("../../../glob/list-files")
+			vi.mocked(listFiles).mockResolvedValue([["test/README.md", "test/app.js", "docs/guide.markdown"], false])
+
+			const mockMarkdownBlocks: any[] = [
+				{
+					file_path: "test/README.md",
+					content: "# Introduction\nThis is a comprehensive guide...",
+					start_line: 1,
+					end_line: 10,
+					identifier: "Introduction",
+					type: "markdown_header_h1",
+					fileHash: "md-hash",
+					segmentHash: "md-segment-hash",
+				},
+			]
+
+			const mockJsBlocks: any[] = [
+				{
+					file_path: "test/app.js",
+					content: "function main() { return 'hello'; }",
+					start_line: 1,
+					end_line: 3,
+					identifier: "main",
+					type: "function",
+					fileHash: "js-hash",
+					segmentHash: "js-segment-hash",
+				},
+			]
+
+			const mockMarkdownBlocks2: any[] = [
+				{
+					file_path: "docs/guide.markdown",
+					content: "## Getting Started\nFollow these steps...",
+					start_line: 1,
+					end_line: 8,
+					identifier: "Getting Started",
+					type: "markdown_header_h2",
+					fileHash: "markdown-hash",
+					segmentHash: "markdown-segment-hash",
+				},
+			]
+
+			// Mock parseFile to return different blocks based on file extension
+			;(mockCodeParser.parseFile as any).mockImplementation((filePath: string) => {
+				if (filePath.endsWith(".md")) {
+					return mockMarkdownBlocks
+				} else if (filePath.endsWith(".markdown")) {
+					return mockMarkdownBlocks2
+				} else if (filePath.endsWith(".js")) {
+					return mockJsBlocks
+				}
+				return []
+			})
+
+			const result = await scanner.scanDirectory("/test")
+
+			// Verify all files were processed
+			expect(mockCodeParser.parseFile).toHaveBeenCalledTimes(3)
+			expect(mockCodeParser.parseFile).toHaveBeenCalledWith("test/README.md", expect.any(Object))
+			expect(mockCodeParser.parseFile).toHaveBeenCalledWith("test/app.js", expect.any(Object))
+			expect(mockCodeParser.parseFile).toHaveBeenCalledWith("docs/guide.markdown", expect.any(Object))
+
+			// Verify code blocks include both markdown and code content
+			expect(result.codeBlocks).toHaveLength(3)
+			expect(result.codeBlocks).toEqual(
+				expect.arrayContaining([
+					expect.objectContaining({ type: "markdown_header_h1" }),
+					expect.objectContaining({ type: "function" }),
+					expect.objectContaining({ type: "markdown_header_h2" }),
+				]),
+			)
+
+			expect(result.stats.processed).toBe(3)
+		})
+
+		it("should generate unique point IDs for each block from the same file", async () => {
+			const { listFiles } = await import("../../../glob/list-files")
+			vi.mocked(listFiles).mockResolvedValue([["test/large-doc.md"], false])
+
+			// Mock multiple blocks from the same file with different segmentHash values
+			const mockBlocks: any[] = [
+				{
+					file_path: "test/large-doc.md",
+					content: "# Introduction\nThis is the intro section...",
+					start_line: 1,
+					end_line: 10,
+					identifier: "Introduction",
+					type: "markdown_header_h1",
+					fileHash: "same-file-hash",
+					segmentHash: "unique-segment-hash-1",
+				},
+				{
+					file_path: "test/large-doc.md",
+					content: "## Getting Started\nHere's how to begin...",
+					start_line: 11,
+					end_line: 20,
+					identifier: "Getting Started",
+					type: "markdown_header_h2",
+					fileHash: "same-file-hash",
+					segmentHash: "unique-segment-hash-2",
+				},
+				{
+					file_path: "test/large-doc.md",
+					content: "## Advanced Topics\nFor advanced users...",
+					start_line: 21,
+					end_line: 30,
+					identifier: "Advanced Topics",
+					type: "markdown_header_h2",
+					fileHash: "same-file-hash",
+					segmentHash: "unique-segment-hash-3",
+				},
+			]
+
+			;(mockCodeParser.parseFile as any).mockResolvedValue(mockBlocks)
+
+			await scanner.scanDirectory("/test")
+
+			// Verify that upsertPoints was called with unique IDs for each block
+			expect(mockVectorStore.upsertPoints).toHaveBeenCalledTimes(1)
+			const upsertCall = mockVectorStore.upsertPoints.mock.calls[0]
+			const points = upsertCall[0]
+
+			// Extract the IDs from the points
+			const pointIds = points.map((point: any) => point.id)
+
+			// Verify all IDs are unique
+			expect(pointIds).toHaveLength(3)
+			expect(new Set(pointIds).size).toBe(3) // All IDs should be unique
+
+			// Verify that each point has the correct payload
+			expect(points[0].payload.segmentHash).toBe("unique-segment-hash-1")
+			expect(points[1].payload.segmentHash).toBe("unique-segment-hash-2")
+			expect(points[2].payload.segmentHash).toBe("unique-segment-hash-3")
+		})
 	})
 })
--- a/src/services/code-index/processors/parser.ts
+++ b/src/services/code-index/processors/parser.ts
@ -3,6 +3,7 @@ import { createHash } from "crypto"
 import * as path from "path"
 import { Node } from "web-tree-sitter"
 import { LanguageParser, loadRequiredLanguageParsers } from "../../tree-sitter/languageParser"
+import { parseMarkdown } from "../../tree-sitter/markdownParser"
 import { ICodeParser, CodeBlock } from "../interfaces"
 import { scannerExtensions } from "../shared/supported-extensions"
 import { MAX_BLOCK_CHARS, MIN_BLOCK_CHARS, MIN_CHUNK_REMAINDER_CHARS, MAX_CHARS_TOLERANCE_FACTOR } from "../constants"
@ -13,8 +14,8 @@ import { MAX_BLOCK_CHARS, MIN_BLOCK_CHARS, MIN_CHUNK_REMAINDER_CHARS, MAX_CHARS_
 export class CodeParser implements ICodeParser {
 	private loadedParsers: LanguageParser = {}
 	private pendingLoads: Map<string, Promise<LanguageParser>> = new Map()
-	// Markdown files are excluded because the current parser logic cannot effectively handle
-	// potentially large Markdown sections without a tree-sitter-like child node structure for chunking
+	// Markdown files are now supported using the custom markdown parser
+	// which extracts headers and sections for semantic indexing

 	/**
 	 * Parses a code file into code blocks
@ -87,6 +88,11 @@ export class CodeParser implements ICodeParser {
 		const ext = path.extname(filePath).slice(1).toLowerCase()
 		const seenSegmentHashes = new Set<string>()

+		// Handle markdown files specially
+		if (ext === "md" || ext === "markdown") {
+			return this.parseMarkdownContent(filePath, content, fileHash, seenSegmentHashes)
+		}
+
 		// Check if we already have the parser loaded
 		if (!this.loadedParsers[ext]) {
 			const pendingLoad = this.pendingLoads.get(ext)
@ -175,8 +181,9 @@ export class CodeParser implements ICodeParser {
 					const start_line = currentNode.startPosition.row + 1
 					const end_line = currentNode.endPosition.row + 1
 					const content = currentNode.text
+					const contentPreview = content.slice(0, 100)
 					const segmentHash = createHash("sha256")
-						.update(`${filePath}-${start_line}-${end_line}-${content}`)
+						.update(`${filePath}-${start_line}-${end_line}-${content.length}-${contentPreview}`)
 						.digest("hex")

 					if (!seenSegmentHashes.has(segmentHash)) {
@ -223,8 +230,9 @@ export class CodeParser implements ICodeParser {
 				const chunkContent = currentChunkLines.join("\n")
 				const startLine = baseStartLine + chunkStartLineIndex
 				const endLine = baseStartLine + endLineIndex
+				const contentPreview = chunkContent.slice(0, 100)
 				const segmentHash = createHash("sha256")
-					.update(`${filePath}-${startLine}-${endLine}-${chunkContent}`)
+					.update(`${filePath}-${startLine}-${endLine}-${chunkContent.length}-${contentPreview}`)
 					.digest("hex")

 				if (!seenSegmentHashes.has(segmentHash)) {
@ -247,8 +255,11 @@ export class CodeParser implements ICodeParser {
 		}

 		const createSegmentBlock = (segment: string, originalLineNumber: number, startCharIndex: number) => {
+			const segmentPreview = segment.slice(0, 100)
 			const segmentHash = createHash("sha256")
-				.update(`${filePath}-${originalLineNumber}-${originalLineNumber}-${startCharIndex}-${segment}`)
+				.update(
+					`${filePath}-${originalLineNumber}-${originalLineNumber}-${startCharIndex}-${segment.length}-${segmentPreview}`,
+				)
 				.digest("hex")

 			if (!seenSegmentHashes.has(segmentHash)) {
@ -287,6 +298,8 @@ export class CodeParser implements ICodeParser {
 					createSegmentBlock(segment, originalLineNumber, currentSegmentStartChar)
 					currentSegmentStartChar += MAX_BLOCK_CHARS
 				}
+				// Update chunkStartLineIndex to continue processing from the next line
+				chunkStartLineIndex = i + 1
 				continue
 			}

@ -370,6 +383,150 @@ export class CodeParser implements ICodeParser {
 			baseStartLine,
 		)
 	}
+
+	/**
+	 * Helper method to process markdown content sections with consistent chunking logic
+	 */
+	private processMarkdownSection(
+		lines: string[],
+		filePath: string,
+		fileHash: string,
+		type: string,
+		seenSegmentHashes: Set<string>,
+		startLine: number,
+		identifier: string | null = null,
+	): CodeBlock[] {
+		const content = lines.join("\n")
+
+		if (content.trim().length < MIN_BLOCK_CHARS) {
+			return []
+		}
+
+		// Check if content needs chunking (either total size or individual line size)
+		const needsChunking =
+			content.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR ||
+			lines.some((line) => line.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR)
+
+		if (needsChunking) {
+			// Apply chunking for large content or oversized lines
+			const chunks = this._chunkTextByLines(lines, filePath, fileHash, type, seenSegmentHashes, startLine)
+			// Preserve identifier in all chunks if provided
+			if (identifier) {
+				chunks.forEach((chunk) => {
+					chunk.identifier = identifier
+				})
+			}
+			return chunks
+		}
+
+		// Create a single block for normal-sized content with no oversized lines
+		const endLine = startLine + lines.length - 1
+		const contentPreview = content.slice(0, 100)
+		const segmentHash = createHash("sha256")
+			.update(`${filePath}-${startLine}-${endLine}-${content.length}-${contentPreview}`)
+			.digest("hex")
+
+		if (!seenSegmentHashes.has(segmentHash)) {
+			seenSegmentHashes.add(segmentHash)
+			return [
+				{
+					file_path: filePath,
+					identifier,
+					type,
+					start_line: startLine,
+					end_line: endLine,
+					content,
+					segmentHash,
+					fileHash,
+				},
+			]
+		}
+
+		return []
+	}
+
+	private parseMarkdownContent(
+		filePath: string,
+		content: string,
+		fileHash: string,
+		seenSegmentHashes: Set<string>,
+	): CodeBlock[] {
+		const lines = content.split("\n")
+		const markdownCaptures = parseMarkdown(content) || []
+
+		if (markdownCaptures.length === 0) {
+			// No headers found, process entire content
+			return this.processMarkdownSection(lines, filePath, fileHash, "markdown_content", seenSegmentHashes, 1)
+		}
+
+		const results: CodeBlock[] = []
+		let lastProcessedLine = 0
+
+		// Process content before the first header
+		if (markdownCaptures.length > 0) {
+			const firstHeaderLine = markdownCaptures[0].node.startPosition.row
+			if (firstHeaderLine > 0) {
+				const preHeaderLines = lines.slice(0, firstHeaderLine)
+				const preHeaderBlocks = this.processMarkdownSection(
+					preHeaderLines,
+					filePath,
+					fileHash,
+					"markdown_content",
+					seenSegmentHashes,
+					1,
+				)
+				results.push(...preHeaderBlocks)
+			}
+		}
+
+		// Process markdown captures (headers and sections)
+		for (let i = 0; i < markdownCaptures.length; i += 2) {
+			const nameCapture = markdownCaptures[i]
+			// Ensure we don't go out of bounds when accessing the next capture
+			if (i + 1 >= markdownCaptures.length) break
+			const definitionCapture = markdownCaptures[i + 1]
+
+			if (!definitionCapture) continue
+
+			const startLine = definitionCapture.node.startPosition.row + 1
+			const endLine = definitionCapture.node.endPosition.row + 1
+			const sectionLines = lines.slice(startLine - 1, endLine)
+
+			// Extract header level for type classification
+			const headerMatch = nameCapture.name.match(/\.h(\d)$/)
+			const headerLevel = headerMatch ? parseInt(headerMatch[1]) : 1
+			const headerText = nameCapture.node.text
+
+			const sectionBlocks = this.processMarkdownSection(
+				sectionLines,
+				filePath,
+				fileHash,
+				`markdown_header_h${headerLevel}`,
+				seenSegmentHashes,
+				startLine,
+				headerText,
+			)
+			results.push(...sectionBlocks)
+
+			lastProcessedLine = endLine
+		}
+
+		// Process any remaining content after the last header section
+		if (lastProcessedLine < lines.length) {
+			const remainingLines = lines.slice(lastProcessedLine)
+			const remainingBlocks = this.processMarkdownSection(
+				remainingLines,
+				filePath,
+				fileHash,
+				"markdown_content",
+				seenSegmentHashes,
+				lastProcessedLine + 1,
+			)
+			results.push(...remainingBlocks)
+		}
+
+		return results
+	}
 }

 // Export a singleton instance for convenience
--- a/src/services/code-index/processors/scanner.ts
+++ b/src/services/code-index/processors/scanner.ts
@ -308,8 +308,8 @@ export class DirectoryScanner implements IDirectoryScanner {
 				const points = batchBlocks.map((block, index) => {
 					const normalizedAbsolutePath = generateNormalizedAbsolutePath(block.file_path)

-					const stableName = `${normalizedAbsolutePath}:${block.start_line}`
-					const pointId = uuidv5(stableName, QDRANT_CODE_BLOCK_NAMESPACE)
+					// Use segmentHash for unique ID generation to handle multiple segments from same line
+					const pointId = uuidv5(block.segmentHash, QDRANT_CODE_BLOCK_NAMESPACE)

 					return {
 						id: pointId,
@ -319,6 +319,7 @@ export class DirectoryScanner implements IDirectoryScanner {
 							codeChunk: block.content,
 							startLine: block.start_line,
 							endLine: block.end_line,
+							segmentHash: block.segmentHash,
 						},
 					}
 				})
--- a/src/services/code-index/shared/supported-extensions.ts
+++ b/src/services/code-index/shared/supported-extensions.ts
@ -1,4 +1,4 @@
 import { extensions as allExtensions } from "../../tree-sitter"

-// Filter out markdown extensions for the scanner
-export const scannerExtensions = allExtensions.filter((ext) => ext !== ".md" && ext !== ".markdown")
+// Include all extensions including markdown for the scanner
+export const scannerExtensions = allExtensions
--- a/src/services/tree-sitter/tests/markdownIntegration.spec.ts
+++ b/src/services/tree-sitter/tests/markdownIntegration.spec.ts
@ -20,8 +20,9 @@ describe("Markdown Integration Tests", () => {
 		vi.clearAllMocks()
 	})

-	it("should parse markdown files and extract headers", async () => {
-		// Mock markdown content
+	it("should parse markdown files and extract headers for definition listing", async () => {
+		// This test verifies that the tree-sitter integration correctly
+		// formats markdown headers for the definition listing feature
 		const markdownContent =
 			"# Main Header\n\nThis is some content under the main header.\nIt spans multiple lines to meet the minimum section length.\n\n## Section 1\n\nThis is content for section 1.\nIt also spans multiple lines.\n\n### Subsection 1.1\n\nThis is a subsection with enough lines\nto meet the minimum section length requirement.\n\n## Section 2\n\nFinal section content.\nWith multiple lines.\n"

@ -34,7 +35,7 @@ describe("Markdown Integration Tests", () => {
 		// Verify fs.readFile was called with the correct path
 		expect(fs.readFile).toHaveBeenCalledWith("test.md", "utf8")

-		// Check the result
+		// Check the result formatting for definition listing
 		expect(result).toBeDefined()
 		expect(result).toContain("# test.md")
 		expect(result).toContain("1--5 | # Main Header")
@ -43,8 +44,8 @@ describe("Markdown Integration Tests", () => {
 		expect(result).toContain("16--20 | ## Section 2")
 	})

-	it("should handle markdown files with no headers", async () => {
-		// Mock markdown content with no headers
+	it("should return undefined for markdown files with no extractable definitions", async () => {
+		// This test verifies behavior when no headers meet the minimum requirements
 		const markdownContent = "This is just some text.\nNo headers here.\nJust plain text."

 		// Mock fs.readFile to return our markdown content
@ -56,45 +57,7 @@ describe("Markdown Integration Tests", () => {
 		// Verify fs.readFile was called with the correct path
 		expect(fs.readFile).toHaveBeenCalledWith("no-headers.md", "utf8")

-		// Check the result
+		// Check the result - should be undefined since no definitions found
 		expect(result).toBeUndefined()
 	})
-
-	it("should handle markdown files with headers that don't meet minimum section length", async () => {
-		// Mock markdown content with headers but short sections
-		const markdownContent = "# Header 1\nShort section\n\n# Header 2\nAnother short section"
-
-		// Mock fs.readFile to return our markdown content
-		;(fs.readFile as Mock).mockImplementation(() => Promise.resolve(markdownContent))
-
-		// Call the function with a markdown file path
-		const result = await parseSourceCodeDefinitionsForFile("short-sections.md")
-
-		// Verify fs.readFile was called with the correct path
-		expect(fs.readFile).toHaveBeenCalledWith("short-sections.md", "utf8")
-
-		// Check the result - should be undefined since no sections meet the minimum length
-		expect(result).toBeUndefined()
-	})
-
-	it("should handle markdown files with mixed header styles", async () => {
-		// Mock markdown content with mixed header styles
-		const markdownContent =
-			"# ATX Header\nThis is content under an ATX header.\nIt spans multiple lines to meet the minimum section length.\n\nSetext Header\n============\nThis is content under a setext header.\nIt also spans multiple lines to meet the minimum section length.\n"
-
-		// Mock fs.readFile to return our markdown content
-		;(fs.readFile as Mock).mockImplementation(() => Promise.resolve(markdownContent))
-
-		// Call the function with a markdown file path
-		const result = await parseSourceCodeDefinitionsForFile("mixed-headers.md")
-
-		// Verify fs.readFile was called with the correct path
-		expect(fs.readFile).toHaveBeenCalledWith("mixed-headers.md", "utf8")
-
-		// Check the result
-		expect(result).toBeDefined()
-		expect(result).toContain("# mixed-headers.md")
-		expect(result).toContain("1--4 | # ATX Header")
-		expect(result).toContain("5--9 | Setext Header")
-	})
 })