mirror of
https://github.com/open-webui/open-webui.git
synced 2026-09-12 23:02:35 +00:00
test: add test document generator for Bug 3 backward merge validation
Adds a Python script that generates a markdown document with precise section sizes designed to trigger orphan fragment creation in the forward-only merge strategy. Includes the generated test document. https://claude.ai/code/session_01HdoZtoLdB2J7p5qropVqCj
This commit is contained in:
Claude
DrMelone
parent
d05301d3fb
commit
03dc9857ce
2 changed files with 189 additions and 0 deletions
154
generate_test_doc.py
Normal file
154
generate_test_doc.py
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
"""
|
||||
Generate a test markdown document that triggers Bug 3 (tiny orphan fragments).
|
||||
|
||||
Settings to use:
|
||||
CHUNK_SIZE = 2048
|
||||
CHUNK_MIN_SIZE_TARGET = 1024
|
||||
ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER = ON
|
||||
TEXT_SPLITTER = "character" (default)
|
||||
|
||||
The MarkdownHeaderTextSplitter splits on headers, producing one chunk per
|
||||
section. The document is crafted so that several tiny heading-only sections
|
||||
sit between large sections, creating orphan fragments that the old
|
||||
forward-only merge cannot absorb.
|
||||
"""
|
||||
|
||||
import textwrap
|
||||
|
||||
|
||||
def pad_prose(target_len: int, seed_sentence: str) -> str:
|
||||
"""Repeat a seed sentence until we hit exactly target_len characters."""
|
||||
base = seed_sentence.strip() + " "
|
||||
repeated = base * ((target_len // len(base)) + 2)
|
||||
return repeated[:target_len]
|
||||
|
||||
|
||||
sections: list[tuple[str, str, int]] = [] # (header, body, body_target_len)
|
||||
|
||||
# ── Pattern 1: Large A, tiny B, large C ──────────────────────────────
|
||||
# A is above min_chunk_size_target (1024) → won't forward-merge
|
||||
# B is tiny (~35 chars body) → orphaned
|
||||
# C is large enough that B+C > 2048 → B can't merge forward into C
|
||||
|
||||
sections.append((
|
||||
"# Chapter 1: Introduction",
|
||||
pad_prose(1400, "This is introductory material covering the broad context of the research."),
|
||||
1400,
|
||||
))
|
||||
|
||||
sections.append((
|
||||
"## 1.1 Scope",
|
||||
"Brief scope statement.", # ~22 chars — tiny orphan
|
||||
None,
|
||||
))
|
||||
|
||||
sections.append((
|
||||
"## 1.2 Background",
|
||||
pad_prose(1950, "Background information provides the historical and technical foundation for this work."),
|
||||
1950,
|
||||
))
|
||||
|
||||
# ── Pattern 2: Large D, TWO consecutive tiny E & F, large G ─────────
|
||||
# Tests that consecutive tiny fragments are each handled.
|
||||
|
||||
sections.append((
|
||||
"# Chapter 2: Methods",
|
||||
pad_prose(1500, "The methodology section describes experimental design and data collection procedures."),
|
||||
1500,
|
||||
))
|
||||
|
||||
sections.append((
|
||||
"## 2.1 Overview",
|
||||
"Short overview.", # ~15 chars
|
||||
None,
|
||||
))
|
||||
|
||||
sections.append((
|
||||
"### 2.1.1 Substep",
|
||||
"Details.", # ~8 chars
|
||||
None,
|
||||
))
|
||||
|
||||
sections.append((
|
||||
"## 2.2 Data Collection",
|
||||
pad_prose(1900, "Data was gathered from multiple sources using standardized instruments and protocols."),
|
||||
1900,
|
||||
))
|
||||
|
||||
# ── Pattern 3: Large H, tiny I as LAST section ──────────────────────
|
||||
# Tests backward merge for the very last chunk in the document.
|
||||
|
||||
sections.append((
|
||||
"# Chapter 3: Results",
|
||||
pad_prose(1600, "The results demonstrate statistically significant improvements across all measured dimensions."),
|
||||
1600,
|
||||
))
|
||||
|
||||
sections.append((
|
||||
"## 3.1 Summary",
|
||||
"End.", # ~4 chars — tiny final orphan
|
||||
None,
|
||||
))
|
||||
|
||||
|
||||
# ── Build the document ───────────────────────────────────────────────
|
||||
doc_parts = []
|
||||
print("=" * 65)
|
||||
print("SECTION SIZES (header + '\\n\\n' + body)")
|
||||
print("=" * 65)
|
||||
|
||||
for header, body, _ in sections:
|
||||
section_text = f"{header}\n\n{body}"
|
||||
doc_parts.append(section_text)
|
||||
body_len = len(body)
|
||||
total_len = len(section_text)
|
||||
tag = " *** TINY ***" if body_len < 50 else ""
|
||||
print(f" {header:<30s} body={body_len:>5d} total={total_len:>5d}{tag}")
|
||||
|
||||
document = "\n\n".join(doc_parts) + "\n"
|
||||
print(f"\n Total document length: {len(document)} chars")
|
||||
print("=" * 65)
|
||||
|
||||
# ── Predict chunk behaviour ──────────────────────────────────────────
|
||||
print("\nEXPECTED CHUNKS AFTER MARKDOWN SPLIT (before merge):")
|
||||
print("-" * 65)
|
||||
for i, (header, body, _) in enumerate(sections):
|
||||
section_text = f"{header}\n\n{body}"
|
||||
# MarkdownHeaderTextSplitter keeps headers in content (strip_headers=False)
|
||||
# and splits between sections. Approximate each section as one chunk.
|
||||
print(f" Chunk {i}: {len(section_text):>5d} chars ← {header}")
|
||||
|
||||
print("\nOLD MERGE BEHAVIOUR (forward-only, CHUNK_SIZE=2048, MIN=1024):")
|
||||
print("-" * 65)
|
||||
print(" Chunk 0: ~1428 chars (Ch1 Introduction)")
|
||||
print(" Chunk 1: ~35 chars (1.1 Scope) *** ORPHAN ***")
|
||||
print(" Chunk 2: ~1970 chars (1.2 Background)")
|
||||
print(" Chunk 3: ~1522 chars (Ch2 Methods)")
|
||||
print(" Chunk 4: ~30 chars (2.1 Overview) *** ORPHAN ***")
|
||||
print(" Chunk 5: ~27 chars (2.1.1 Substep) *** ORPHAN ***")
|
||||
print(" Chunk 6: ~1924 chars (2.2 Data Collection)")
|
||||
print(" Chunk 7: ~1622 chars (Ch3 Results)")
|
||||
print(" Chunk 8: ~19 chars (3.1 Summary) *** ORPHAN ***")
|
||||
|
||||
print("\nNEW MERGE BEHAVIOUR (forward + backward):")
|
||||
print("-" * 65)
|
||||
print(" Chunk 0: ~1463 chars (Ch1 Introduction + 1.1 Scope) ← backward merge")
|
||||
print(" Chunk 1: ~1970 chars (1.2 Background)")
|
||||
print(" Chunk 2: ~1554 chars (Ch2 Methods + 2.1 Overview) ← backward merge")
|
||||
print(" Chunk 3: ~27 chars (2.1.1 Substep) ← merged into Ch2 above")
|
||||
print(" ...or Ch2+Overview+Substep = ~1583 chars ← both merged backward")
|
||||
print(" Chunk 4: ~1924 chars (2.2 Data Collection)")
|
||||
print(" Chunk 5: ~1643 chars (Ch3 Results + 3.1 Summary) ← backward merge")
|
||||
|
||||
print()
|
||||
|
||||
# Write the file
|
||||
output_path = "test_merge_bug3.md"
|
||||
with open(output_path, "w") as f:
|
||||
f.write(document)
|
||||
print(f"Document written to: {output_path}")
|
||||
print(f"\nSettings to use:")
|
||||
print(f" CHUNK_SIZE = 2048")
|
||||
print(f" CHUNK_MIN_SIZE_TARGET = 1024")
|
||||
print(f" ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER = ON")
|
||||
print(f" TEXT_SPLITTER = character (default)")
|
||||
35
test_merge_bug3.md
Normal file
35
test_merge_bug3.md
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
# Chapter 1: Introduction
|
||||
|
||||
This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the research. This is introductory material covering the broad context of the rese
|
||||
|
||||
## 1.1 Scope
|
||||
|
||||
Brief scope statement.
|
||||
|
||||
## 1.2 Background
|
||||
|
||||
Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the historical and technical foundation for this work. Background information provides the
|
||||
|
||||
# Chapter 2: Methods
|
||||
|
||||
The methodology section describes experimental design and data collection procedures. The methodology section describes experimental design and data collection procedures. The methodology section describes experimental design and data collection procedures. The methodology section describes experimental design and data collection procedures. The methodology section describes experimental design and data collection procedures. The methodology section describes experimental design and data collection procedures. The methodology section describes experimental design and data collection procedures. The methodology section describes experimental design and data collection procedures. The methodology section describes experimental design and data collection procedures. The methodology section describes experimental design and data collection procedures. The methodology section describes experimental design and data collection procedures. The methodology section describes experimental design and data collection procedures. The methodology section describes experimental design and data collection procedures. The methodology section describes experimental design and data collection procedures. The methodology section describes experimental design and data collection procedures. The methodology section describes experimental design and data collection procedures. The methodology section describes experimental design and data collection procedures. The methodology section describes expe
|
||||
|
||||
## 2.1 Overview
|
||||
|
||||
Short overview.
|
||||
|
||||
### 2.1.1 Substep
|
||||
|
||||
Details.
|
||||
|
||||
## 2.2 Data Collection
|
||||
|
||||
Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was gathered from multiple sources using standardized instruments and protocols. Data was
|
||||
|
||||
# Chapter 3: Results
|
||||
|
||||
The results demonstrate statistically significant improvements across all measured dimensions. The results demonstrate statistically significant improvements across all measured dimensions. The results demonstrate statistically significant improvements across all measured dimensions. The results demonstrate statistically significant improvements across all measured dimensions. The results demonstrate statistically significant improvements across all measured dimensions. The results demonstrate statistically significant improvements across all measured dimensions. The results demonstrate statistically significant improvements across all measured dimensions. The results demonstrate statistically significant improvements across all measured dimensions. The results demonstrate statistically significant improvements across all measured dimensions. The results demonstrate statistically significant improvements across all measured dimensions. The results demonstrate statistically significant improvements across all measured dimensions. The results demonstrate statistically significant improvements across all measured dimensions. The results demonstrate statistically significant improvements across all measured dimensions. The results demonstrate statistically significant improvements across all measured dimensions. The results demonstrate statistically significant improvements across all measured dimensions. The results demonstrate statistically significant improvements across all measured dimensions. The results demonstrate statistically significant improvements across all measur
|
||||
|
||||
## 3.1 Summary
|
||||
|
||||
End.
|
||||
Loading…
Add table
Reference in a new issue