From 8c7428122bd83bc796741a076e2a369e3356e908 Mon Sep 17 00:00:00 2001
From: G30 <50341825+silentoplayz@users.noreply.github.com>
Date: Mon, 29 Jun 2026 11:43:24 -0400
Subject: [PATCH] fix(rag): allow special tokens in document text during chunk
 measurement (#26210)

Documents (especially AI/LLM documentation) legitimately contain special token strings like <|endoftext|> as literal text. The tiktoken encoder raises a ValueError when encountering these during chunk size measurement in merge_docs_to_target_size(), preventing the entire file from being indexed. Pass disallowed_special=() to encoding.encode() to treat all text as normal content.
---
 backend/open_webui/routers/retrieval.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py
index e5e3dc2d5c..bc20d92192 100644
--- a/backend/open_webui/routers/retrieval.py
+++ b/backend/open_webui/routers/retrieval.py
@@ -1541,7 +1541,7 @@ def merge_docs_to_target_size(
     measure: Callable[[str], int] = len
     if config.TEXT_SPLITTER == 'token':
         encoding = tiktoken.get_encoding(str(config.TIKTOKEN_ENCODING_NAME))
-        measure = lambda text: len(encoding.encode(text))
+        measure = lambda text: len(encoding.encode(text, disallowed_special=()))
 
     def _merge_backward(result: list[Document], content: str, chunk: Document) -> bool:
         """Try to append content into the last emitted chunk. Returns True on success."""