scriptoriumadmin/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-08-28 05:27:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

refac

Browse source
This commit is contained in:
Timothy Jaeryang Baek 2026-07-24 01:13:04 -04:00
parent ff11ff5a3e
commit 33cf3fbb7f
1 changed files with 4 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
backend/open_webui/routers/retrieval.py
View file

@ -127,6 +127,8 @@ from sqlalchemy.ext.asyncio import AsyncSession
log = logging.getLogger(__name__)
TIKTOKEN_DISALLOWED_SPECIAL = ()
##########################################
#
# Utility functions
@ -1582,7 +1584,7 @@ def get_splitter_length_function(
) -> Callable[[str], int]:
if config.TEXT_SPLITTER == 'token':
encoding = tiktoken.get_encoding(str(config.TIKTOKEN_ENCODING_NAME))
return lambda text: len(encoding.encode(text, disallowed_special=()))
return lambda text: len(encoding.encode(text, disallowed_special=TIKTOKEN_DISALLOWED_SPECIAL))
if config.TEXT_SPLITTER == 'token_transformers':
tokenizer = get_transformers_tokenizer(request, config)
@ -1689,6 +1691,7 @@ def save_docs_to_vector_db(
chunk_size=config.CHUNK_SIZE,
chunk_overlap=config.CHUNK_OVERLAP,
add_start_index=True,
disallowed_special=TIKTOKEN_DISALLOWED_SPECIAL,
)
docs = text_splitter.split_documents(docs)
elif config.TEXT_SPLITTER == 'token_transformers':