From 1bfa59acbda0b0a60b73c313f64bededa7e85252 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Sun, 6 Sep 2026 21:13:45 +0200 Subject: [PATCH] fix: keep HTML entities in extracted document text (#29736) Uploading a file whose text contains literal HTML entities stored a rewritten copy of it: ` ` became a non-breaking space, `>` became `>`, and `&nbsp;` was decoded twice down to a bare non-breaking space. That stored text is what gets indexed and what the model reads, so notes, specs and source files reached the model differing from the file that was uploaded. Every loaded document goes through `ftfy.fix_text`, which is there to repair mojibake left by the encoding-detection fallback. Its default configuration also decodes HTML entities, per line and sticky forward: entities are decoded on every line up to the first line holding a literal `<`, then left alone for the rest of the document. The same escape therefore survives or vanishes depending on where it sits in the file. This disables that one behaviour and leaves every other ftfy repair in place. Text from a third-party extraction engine that returns escaped output now keeps those escapes. Guessing whether an escape is markup or content is the bug being fixed. Fixes #29732 --- backend/open_webui/retrieval/loaders/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/open_webui/retrieval/loaders/main.py b/backend/open_webui/retrieval/loaders/main.py index 13570c37f0..bc50269f3d 100644 --- a/backend/open_webui/retrieval/loaders/main.py +++ b/backend/open_webui/retrieval/loaders/main.py @@ -314,7 +314,11 @@ class Loader: def load(self, filename: str, file_content_type: str, file_path: str) -> list[Document]: loader = self._get_loader(filename, file_content_type, file_path) docs = loader.load() - return [Document(page_content=ftfy.fix_text(doc.page_content), metadata=doc.metadata) for doc in docs] + # ftfy's auto mode unescapes entities on every line before the first literal '<', rewriting the document. + return [ + Document(page_content=ftfy.fix_text(doc.page_content, unescape_html=False), metadata=doc.metadata) + for doc in docs + ] async def aload(self, filename: str, file_content_type: str, file_path: str) -> list[Document]: """