mirror of
https://github.com/open-webui/open-webui.git
synced 2026-10-07 02:58:21 +00:00
fix: uploaded files lose Chinese punctuation and quotes (#31655)
Text extracted from any uploaded file had full-width punctuation like :(),!? turned into ASCII :(),!? and curly quotes like “ ” turned into straight quotes, so both the file preview and the model saw altered text. Extracted text now keeps these characters as written, while garbled text from wrong encodings (like café becoming café) is still repaired. Files uploaded before this change keep the altered text until they are uploaded again. Fixes #17087
This commit is contained in:
parent
3ef0d15433
commit
c3fbf36384
1 changed files with 6 additions and 1 deletions
|
|
@ -342,7 +342,12 @@ class Loader:
|
|||
docs = loader.load()
|
||||
# ftfy's auto mode unescapes entities on every line before the first literal '<', rewriting the document.
|
||||
return [
|
||||
Document(page_content=ftfy.fix_text(doc.page_content, unescape_html=False), metadata=doc.metadata)
|
||||
Document(
|
||||
page_content=ftfy.fix_text(
|
||||
doc.page_content, unescape_html=False, fix_character_width=False, uncurl_quotes=False
|
||||
),
|
||||
metadata=doc.metadata,
|
||||
)
|
||||
for doc in docs
|
||||
]
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue