fix GB18030 text loader fallback

backend/open_webui/retrieval/loaders/main.py

@@ -13,8 +13,6 @@ from langchain_community.document_loaders import (
     Docx2txtLoader,
     OutlookMessageLoader,
     PyPDFLoader,
-    TextLoader,
-    YoutubeLoader,
 )
 from langchain_core.documents import Document
 from open_webui.env import AIOHTTP_CLIENT_SESSION_SSL, GLOBAL_LOG_LEVEL, REQUESTS_VERIFY
@@ -23,6 +21,7 @@ from open_webui.retrieval.loaders.external_document import ExternalDocumentLoade
 from open_webui.retrieval.loaders.mineru import MinerULoader
 from open_webui.retrieval.loaders.mistral import MistralLoader
 from open_webui.retrieval.loaders.paddleocr_vl import PaddleOCRVLLoader
+from open_webui.retrieval.loaders.text import read_text_file
 
 logging.basicConfig(stream=sys.stdout, level=GLOBAL_LOG_LEVEL)
 log = logging.getLogger(__name__)
@@ -132,6 +131,21 @@ class PptxLoader:
         ]
 
 
+class RobustTextLoader:
+    def __init__(self, file_path):
+        self.file_path = file_path
+
+    def load(self) -> list[Document]:
+        text, encoding = read_text_file(self.file_path)
+
+        return [
+            Document(
+                page_content=text,
+                metadata={'source': self.file_path, 'encoding': encoding},
+            )
+        ]
+
+
 class TikaLoader:
     def __init__(self, url, file_path, mime_type=None, extract_images=None):
         self.url = url
@@ -274,7 +288,7 @@ class Loader:
             )
         elif self.engine == 'tika' and self.kwargs.get('TIKA_SERVER_URL'):
             if self._is_text_file(file_ext, file_content_type):
-                loader = TextLoader(file_path, autodetect_encoding=True)
+                loader = RobustTextLoader(file_path)
             else:
                 loader = TikaLoader(
                     url=self.kwargs.get('TIKA_SERVER_URL'),
@@ -326,7 +340,7 @@ class Loader:
             )
         elif self.engine == 'docling' and self.kwargs.get('DOCLING_SERVER_URL'):
             if self._is_text_file(file_ext, file_content_type):
-                loader = TextLoader(file_path, autodetect_encoding=True)
+                loader = RobustTextLoader(file_path)
             else:
                 # Build params for DoclingLoader
                 params = self.kwargs.get('DOCLING_PARAMS', {})
@@ -423,7 +437,7 @@ class Loader:
                         'Falling back to plain text loading for .rst file. '
                         'Install it with: pip install unstructured'
                     )
-                    loader = TextLoader(file_path, autodetect_encoding=True)
+                    loader = RobustTextLoader(file_path)
             elif file_ext == 'xml':
                 try:
                     from langchain_community.document_loaders import UnstructuredXMLLoader
@@ -435,11 +449,11 @@ class Loader:
                         'Falling back to plain text loading for .xml file. '
                         'Install it with: pip install unstructured'
                     )
-                    loader = TextLoader(file_path, autodetect_encoding=True)
+                    loader = RobustTextLoader(file_path)
             elif file_ext in ['htm', 'html']:
                 loader = BSHTMLLoader(file_path, open_encoding='unicode_escape')
             elif file_ext == 'md':
-                loader = TextLoader(file_path, autodetect_encoding=True)
+                loader = RobustTextLoader(file_path)
             elif file_content_type == 'application/epub+zip':
                 try:
                     from langchain_community.document_loaders import UnstructuredEPubLoader
@@ -498,8 +512,8 @@ class Loader:
                         'Install it with: pip install unstructured'
                     )
             elif self._is_text_file(file_ext, file_content_type):
-                loader = TextLoader(file_path, autodetect_encoding=True)
+                loader = RobustTextLoader(file_path)
             else:
-                loader = TextLoader(file_path, autodetect_encoding=True)
+                loader = RobustTextLoader(file_path)
 
         return loader

backend/open_webui/retrieval/loaders/text.py

@@ -0,0 +1,95 @@
+import codecs
+from collections.abc import Iterable
+from pathlib import Path
+
+CJK_ENCODING_FALLBACKS = (
+    'gb18030',
+    'gbk',
+    'big5',
+    'big5hkscs',
+    'shift_jis',
+    'euc_jp',
+    'euc_kr',
+)
+
+GB_FAMILY_ENCODINGS = {
+    'gb2312',
+    'gb_2312',
+    'gbk',
+    'gb18030',
+    'hz',
+    'hz-gb-2312',
+}
+
+
+def _normalize_encoding(encoding: str | None) -> str | None:
+    if not encoding:
+        return None
+
+    try:
+        return codecs.lookup(encoding).name
+    except LookupError:
+        return encoding.lower().replace(' ', '-')
+
+
+def _dedupe(items: Iterable[str]) -> list[str]:
+    seen = set()
+    deduped = []
+
+    for item in items:
+        normalized = _normalize_encoding(item)
+        if not normalized or normalized in seen:
+            continue
+
+        seen.add(normalized)
+        deduped.append(item)
+
+    return deduped
+
+
+def _detect_encoding(data: bytes) -> str | None:
+    try:
+        import chardet
+    except ImportError:
+        return None
+
+    result = chardet.detect(data)
+    encoding = result.get('encoding')
+
+    if encoding:
+        return encoding
+
+    return None
+
+
+def get_text_decoding_candidates(data: bytes) -> list[str]:
+    detected_encoding = _detect_encoding(data)
+    candidates = ['utf-8-sig', 'utf-8']
+
+    if detected_encoding:
+        candidates.append(detected_encoding)
+
+        normalized_detected = _normalize_encoding(detected_encoding)
+        if normalized_detected in GB_FAMILY_ENCODINGS:
+            candidates.extend(('gb18030', 'gbk'))
+
+    candidates.extend(CJK_ENCODING_FALLBACKS)
+    candidates.append('latin-1')
+
+    return _dedupe(candidates)
+
+
+def read_text_file(file_path: str | Path) -> tuple[str, str]:
+    data = Path(file_path).read_bytes()
+    decode_errors: list[UnicodeDecodeError] = []
+
+    for encoding in get_text_decoding_candidates(data):
+        try:
+            return data.decode(encoding), encoding
+        except UnicodeDecodeError as e:
+            decode_errors.append(e)
+
+    if decode_errors:
+        raise decode_errors[-1]
+
+    return data.decode('utf-8'), 'utf-8'

backend/open_webui/test/retrieval/loaders/test_text.py

@@ -0,0 +1,31 @@
+import sys
+from types import SimpleNamespace
+
+from open_webui.retrieval.loaders.text import (
+    get_text_decoding_candidates,
+    read_text_file,
+)
+
+
+def test_read_text_file_falls_back_to_gb18030(tmp_path):
+    content = '# \u6807\u9898\n\n\U00020000 GB18030 only character\n'
+    file_path = tmp_path / 'gb18030.md'
+    file_path.write_bytes(content.encode('gb18030'))
+
+    text, encoding = read_text_file(file_path)
+
+    assert text == content
+    assert encoding.lower() == 'gb18030'
+
+
+def test_gb2312_detection_adds_gb18030_fallback(monkeypatch):
+    monkeypatch.setitem(
+        sys.modules,
+        'chardet',
+        SimpleNamespace(detect=lambda _: {'encoding': 'GB2312', 'confidence': 0.99}),
+    )
+
+    candidates = get_text_decoding_candidates(b'content')
+
+    assert candidates.index('GB2312') < candidates.index('gb18030')
+    assert 'gbk' in candidates