From 3eaab28be5a00536c990909db8cc59ea7afcbcf5 Mon Sep 17 00:00:00 2001 From: Sanjay Santhanam <51058514+Sanjays2402@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:58:05 -0700 Subject: [PATCH] fix(rag): reject invalid chunk_size/chunk_overlap in RecursiveCharacterTextSplitter Fixes #38980 --- .../recursive_character_text_splitter.py | 9 +++ tests/unit/rag/text_splitters/__init__.py | 0 .../test_recursive_character_text_splitter.py | 59 +++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 tests/unit/rag/text_splitters/__init__.py create mode 100644 tests/unit/rag/text_splitters/test_recursive_character_text_splitter.py diff --git a/litellm/rag/text_splitters/recursive_character_text_splitter.py b/litellm/rag/text_splitters/recursive_character_text_splitter.py index a8e86e74383..a4a8c2dae30 100644 --- a/litellm/rag/text_splitters/recursive_character_text_splitter.py +++ b/litellm/rag/text_splitters/recursive_character_text_splitter.py @@ -23,6 +23,15 @@ class RecursiveCharacterTextSplitter: chunk_overlap: int = DEFAULT_CHUNK_OVERLAP, separators: list[str] | None = None, ): + if chunk_size <= 0: + raise ValueError(f"chunk_size must be positive, got {chunk_size}") + if chunk_overlap < 0: + raise ValueError(f"chunk_overlap must be non-negative, got {chunk_overlap}") + if chunk_overlap >= chunk_size: + raise ValueError( + "chunk_overlap must be smaller than chunk_size, " + f"got chunk_overlap={chunk_overlap}, chunk_size={chunk_size}" + ) self.chunk_size = chunk_size self.chunk_overlap = chunk_overlap self.separators = separators or ["\n\n", "\n", " ", ""] diff --git a/tests/unit/rag/text_splitters/__init__.py b/tests/unit/rag/text_splitters/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/rag/text_splitters/test_recursive_character_text_splitter.py b/tests/unit/rag/text_splitters/test_recursive_character_text_splitter.py new file mode 100644 index 00000000000..5fc7a33ef66 --- /dev/null +++ b/tests/unit/rag/text_splitters/test_recursive_character_text_splitter.py @@ -0,0 +1,59 @@ +from typing import Final + +import pytest + +from litellm.constants import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE +from litellm.rag.text_splitters.recursive_character_text_splitter import ( + RecursiveCharacterTextSplitter, +) + + +@pytest.mark.parametrize( + "chunk_size,chunk_overlap", + [ + (100, 100), + (100, 150), + (1, 1), + ], +) +def test_chunk_overlap_at_least_chunk_size_is_rejected( + chunk_size: int, chunk_overlap: int +) -> None: + with pytest.raises( + ValueError, match="chunk_overlap must be smaller than chunk_size" + ): + RecursiveCharacterTextSplitter( + chunk_size=chunk_size, chunk_overlap=chunk_overlap + ) + + +@pytest.mark.parametrize("chunk_size", [0, -1, -100]) +def test_non_positive_chunk_size_is_rejected(chunk_size: int) -> None: + with pytest.raises(ValueError, match="chunk_size must be positive"): + RecursiveCharacterTextSplitter(chunk_size=chunk_size) + + +@pytest.mark.parametrize("chunk_overlap", [-1, -100]) +def test_negative_chunk_overlap_is_rejected(chunk_overlap: int) -> None: + with pytest.raises(ValueError, match="chunk_overlap must be non-negative"): + RecursiveCharacterTextSplitter(chunk_overlap=chunk_overlap) + + +def test_force_split_path_terminates_with_valid_overlap() -> None: + splitter: Final = RecursiveCharacterTextSplitter( + chunk_size=10, chunk_overlap=2, separators=["\n\n"] + ) + chunks: Final = splitter.split_text("abcdefghijklmnop") + assert chunks == ["abcdefghij", "ijklmnop"] + + +def test_valid_overlap_keeps_splitting() -> None: + splitter: Final = RecursiveCharacterTextSplitter(chunk_size=10, chunk_overlap=2) + chunks: Final = splitter.split_text("abcdefghijklmnop") + assert chunks == ["abcdefghij", "ijklmnop"] + + +def test_defaults_construct_without_error() -> None: + splitter: Final = RecursiveCharacterTextSplitter() + assert splitter.chunk_size == DEFAULT_CHUNK_SIZE + assert splitter.chunk_overlap == DEFAULT_CHUNK_OVERLAP