From 01823d12a89935a9df3a6ff301aed2c448364b98 Mon Sep 17 00:00:00 2001 From: Liang Xu Date: Tue, 1 Sep 2026 15:39:41 +0800 Subject: [PATCH 1/2] fix(rag): validate chunk_overlap against chunk_size in RecursiveCharacterTextSplitter Fixes #38980 Signed-off-by: Liang Xu --- .../recursive_character_text_splitter.py | 14 +++++- .../test_recursive_character_text_splitter.py | 46 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 tests/test_litellm/rag/test_recursive_character_text_splitter.py diff --git a/litellm/rag/text_splitters/recursive_character_text_splitter.py b/litellm/rag/text_splitters/recursive_character_text_splitter.py index a8e86e74383..38e356f3a07 100644 --- a/litellm/rag/text_splitters/recursive_character_text_splitter.py +++ b/litellm/rag/text_splitters/recursive_character_text_splitter.py @@ -23,6 +23,15 @@ class RecursiveCharacterTextSplitter: chunk_overlap: int = DEFAULT_CHUNK_OVERLAP, separators: list[str] | None = None, ): + if chunk_size <= 0: + raise ValueError(f"chunk_size must be greater than 0, got {chunk_size}") + if chunk_overlap < 0: + raise ValueError(f"chunk_overlap must be non-negative, got {chunk_overlap}") + if chunk_overlap >= chunk_size: + raise ValueError( + f"Got a larger or equal chunk_overlap ({chunk_overlap}) than chunk_size ({chunk_size}), " + f"chunk_overlap must be smaller than chunk_size" + ) self.chunk_size = chunk_size self.chunk_overlap = chunk_overlap self.separators = separators or ["\n\n", "\n", " ", ""] @@ -123,12 +132,15 @@ class RecursiveCharacterTextSplitter: """Force split text by chunk_size when no separator works.""" chunks: Final[list[str]] = [] start = 0 + step = max(self.chunk_size - self.chunk_overlap, 1) while start < len(text): end = start + self.chunk_size chunk = text[start:end].strip() if chunk: chunks.append(chunk) - start = end - self.chunk_overlap if end < len(text) else len(text) + if end >= len(text): + break + start += step return chunks diff --git a/tests/test_litellm/rag/test_recursive_character_text_splitter.py b/tests/test_litellm/rag/test_recursive_character_text_splitter.py new file mode 100644 index 00000000000..f7b19a959da --- /dev/null +++ b/tests/test_litellm/rag/test_recursive_character_text_splitter.py @@ -0,0 +1,46 @@ +import pytest +from litellm.rag.text_splitters.recursive_character_text_splitter import ( + RecursiveCharacterTextSplitter, +) + + +def test_invalid_chunk_overlap_greater_than_or_equal_chunk_size(): + """chunk_overlap >= chunk_size must raise ValueError to prevent infinite loops.""" + with pytest.raises(ValueError, match="chunk_overlap"): + RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=500) + + with pytest.raises(ValueError, match="chunk_overlap"): + RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=1000) + + +def test_invalid_chunk_size_non_positive(): + """chunk_size <= 0 must raise ValueError.""" + with pytest.raises(ValueError, match="chunk_size"): + RecursiveCharacterTextSplitter(chunk_size=0, chunk_overlap=0) + + with pytest.raises(ValueError, match="chunk_size"): + RecursiveCharacterTextSplitter(chunk_size=-10, chunk_overlap=0) + + +def test_invalid_chunk_overlap_negative(): + """chunk_overlap < 0 must raise ValueError.""" + with pytest.raises(ValueError, match="chunk_overlap"): + RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=-1) + + +def test_split_text_basic(): + splitter = RecursiveCharacterTextSplitter(chunk_size=50, chunk_overlap=10) + text = "This is a test paragraph.\n\nThis is another paragraph that is longer and will need splitting." + chunks = splitter.split_text(text) + assert len(chunks) > 1 + for chunk in chunks: + assert len(chunk) <= 50 + + +def test_force_split_no_matching_separators(): + """Verify that text with no matching separators splits cleanly without hanging.""" + splitter = RecursiveCharacterTextSplitter(chunk_size=20, chunk_overlap=5, separators=["\n\n"]) + text = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + chunks = splitter.split_text(text) + assert len(chunks) > 1 + assert "".join(chunks) != "" From 177ad4ce2ac0931a0433450cb0fffea8927743de Mon Sep 17 00:00:00 2001 From: Liang Xu Date: Sun, 6 Sep 2026 04:21:53 +0800 Subject: [PATCH 2/2] fix(rag): annotate step with Final to pass type discipline gate --- litellm/rag/text_splitters/recursive_character_text_splitter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/rag/text_splitters/recursive_character_text_splitter.py b/litellm/rag/text_splitters/recursive_character_text_splitter.py index 38e356f3a07..9cf8019efd3 100644 --- a/litellm/rag/text_splitters/recursive_character_text_splitter.py +++ b/litellm/rag/text_splitters/recursive_character_text_splitter.py @@ -132,7 +132,7 @@ class RecursiveCharacterTextSplitter: """Force split text by chunk_size when no separator works.""" chunks: Final[list[str]] = [] start = 0 - step = max(self.chunk_size - self.chunk_overlap, 1) + step: Final = max(self.chunk_size - self.chunk_overlap, 1) while start < len(text): end = start + self.chunk_size