From 021d9854990973fec8fc46bbdabef64c6d409509 Mon Sep 17 00:00:00 2001 From: lord_Tyrion Date: Sat, 23 May 2026 13:45:16 +0530 Subject: [PATCH] fix GB18030 text loader fallback --- backend/open_webui/retrieval/loaders/main.py | 32 +++++-- backend/open_webui/retrieval/loaders/text.py | 95 +++++++++++++++++++ .../test/retrieval/loaders/test_text.py | 31 ++++++ 3 files changed, 149 insertions(+), 9 deletions(-) create mode 100644 backend/open_webui/retrieval/loaders/text.py create mode 100644 backend/open_webui/test/retrieval/loaders/test_text.py diff --git a/backend/open_webui/retrieval/loaders/main.py b/backend/open_webui/retrieval/loaders/main.py index b322c84279..fcd072f971 100644 --- a/backend/open_webui/retrieval/loaders/main.py +++ b/backend/open_webui/retrieval/loaders/main.py @@ -13,8 +13,6 @@ from langchain_community.document_loaders import ( Docx2txtLoader, OutlookMessageLoader, PyPDFLoader, - TextLoader, - YoutubeLoader, ) from langchain_core.documents import Document from open_webui.env import AIOHTTP_CLIENT_SESSION_SSL, GLOBAL_LOG_LEVEL, REQUESTS_VERIFY @@ -23,6 +21,7 @@ from open_webui.retrieval.loaders.external_document import ExternalDocumentLoade from open_webui.retrieval.loaders.mineru import MinerULoader from open_webui.retrieval.loaders.mistral import MistralLoader from open_webui.retrieval.loaders.paddleocr_vl import PaddleOCRVLLoader +from open_webui.retrieval.loaders.text import read_text_file logging.basicConfig(stream=sys.stdout, level=GLOBAL_LOG_LEVEL) log = logging.getLogger(__name__) @@ -132,6 +131,21 @@ class PptxLoader: ] +class RobustTextLoader: + def __init__(self, file_path): + self.file_path = file_path + + def load(self) -> list[Document]: + text, encoding = read_text_file(self.file_path) + + return [ + Document( + page_content=text, + metadata={'source': self.file_path, 'encoding': encoding}, + ) + ] + + class TikaLoader: def __init__(self, url, file_path, mime_type=None, extract_images=None): self.url = url @@ -274,7 +288,7 @@ class Loader: ) elif self.engine == 'tika' and self.kwargs.get('TIKA_SERVER_URL'): if self._is_text_file(file_ext, file_content_type): - loader = TextLoader(file_path, autodetect_encoding=True) + loader = RobustTextLoader(file_path) else: loader = TikaLoader( url=self.kwargs.get('TIKA_SERVER_URL'), @@ -326,7 +340,7 @@ class Loader: ) elif self.engine == 'docling' and self.kwargs.get('DOCLING_SERVER_URL'): if self._is_text_file(file_ext, file_content_type): - loader = TextLoader(file_path, autodetect_encoding=True) + loader = RobustTextLoader(file_path) else: # Build params for DoclingLoader params = self.kwargs.get('DOCLING_PARAMS', {}) @@ -423,7 +437,7 @@ class Loader: 'Falling back to plain text loading for .rst file. ' 'Install it with: pip install unstructured' ) - loader = TextLoader(file_path, autodetect_encoding=True) + loader = RobustTextLoader(file_path) elif file_ext == 'xml': try: from langchain_community.document_loaders import UnstructuredXMLLoader @@ -435,11 +449,11 @@ class Loader: 'Falling back to plain text loading for .xml file. ' 'Install it with: pip install unstructured' ) - loader = TextLoader(file_path, autodetect_encoding=True) + loader = RobustTextLoader(file_path) elif file_ext in ['htm', 'html']: loader = BSHTMLLoader(file_path, open_encoding='unicode_escape') elif file_ext == 'md': - loader = TextLoader(file_path, autodetect_encoding=True) + loader = RobustTextLoader(file_path) elif file_content_type == 'application/epub+zip': try: from langchain_community.document_loaders import UnstructuredEPubLoader @@ -498,8 +512,8 @@ class Loader: 'Install it with: pip install unstructured' ) elif self._is_text_file(file_ext, file_content_type): - loader = TextLoader(file_path, autodetect_encoding=True) + loader = RobustTextLoader(file_path) else: - loader = TextLoader(file_path, autodetect_encoding=True) + loader = RobustTextLoader(file_path) return loader diff --git a/backend/open_webui/retrieval/loaders/text.py b/backend/open_webui/retrieval/loaders/text.py new file mode 100644 index 0000000000..17c6a12372 --- /dev/null +++ b/backend/open_webui/retrieval/loaders/text.py @@ -0,0 +1,95 @@ +import codecs +from collections.abc import Iterable +from pathlib import Path + +CJK_ENCODING_FALLBACKS = ( + 'gb18030', + 'gbk', + 'big5', + 'big5hkscs', + 'shift_jis', + 'euc_jp', + 'euc_kr', +) + +GB_FAMILY_ENCODINGS = { + 'gb2312', + 'gb_2312', + 'gbk', + 'gb18030', + 'hz', + 'hz-gb-2312', +} + + +def _normalize_encoding(encoding: str | None) -> str | None: + if not encoding: + return None + + try: + return codecs.lookup(encoding).name + except LookupError: + return encoding.lower().replace(' ', '-') + + +def _dedupe(items: Iterable[str]) -> list[str]: + seen = set() + deduped = [] + + for item in items: + normalized = _normalize_encoding(item) + if not normalized or normalized in seen: + continue + + seen.add(normalized) + deduped.append(item) + + return deduped + + +def _detect_encoding(data: bytes) -> str | None: + try: + import chardet + except ImportError: + return None + + result = chardet.detect(data) + encoding = result.get('encoding') + + if encoding: + return encoding + + return None + + +def get_text_decoding_candidates(data: bytes) -> list[str]: + detected_encoding = _detect_encoding(data) + candidates = ['utf-8-sig', 'utf-8'] + + if detected_encoding: + candidates.append(detected_encoding) + + normalized_detected = _normalize_encoding(detected_encoding) + if normalized_detected in GB_FAMILY_ENCODINGS: + candidates.extend(('gb18030', 'gbk')) + + candidates.extend(CJK_ENCODING_FALLBACKS) + candidates.append('latin-1') + + return _dedupe(candidates) + + +def read_text_file(file_path: str | Path) -> tuple[str, str]: + data = Path(file_path).read_bytes() + decode_errors: list[UnicodeDecodeError] = [] + + for encoding in get_text_decoding_candidates(data): + try: + return data.decode(encoding), encoding + except UnicodeDecodeError as e: + decode_errors.append(e) + + if decode_errors: + raise decode_errors[-1] + + return data.decode('utf-8'), 'utf-8' diff --git a/backend/open_webui/test/retrieval/loaders/test_text.py b/backend/open_webui/test/retrieval/loaders/test_text.py new file mode 100644 index 0000000000..7f9ee79f62 --- /dev/null +++ b/backend/open_webui/test/retrieval/loaders/test_text.py @@ -0,0 +1,31 @@ +import sys +from types import SimpleNamespace + +from open_webui.retrieval.loaders.text import ( + get_text_decoding_candidates, + read_text_file, +) + + +def test_read_text_file_falls_back_to_gb18030(tmp_path): + content = '# \u6807\u9898\n\n\U00020000 GB18030 only character\n' + file_path = tmp_path / 'gb18030.md' + file_path.write_bytes(content.encode('gb18030')) + + text, encoding = read_text_file(file_path) + + assert text == content + assert encoding.lower() == 'gb18030' + + +def test_gb2312_detection_adds_gb18030_fallback(monkeypatch): + monkeypatch.setitem( + sys.modules, + 'chardet', + SimpleNamespace(detect=lambda _: {'encoding': 'GB2312', 'confidence': 0.99}), + ) + + candidates = get_text_decoding_candidates(b'content') + + assert candidates.index('GB2312') < candidates.index('gb18030') + assert 'gbk' in candidates