scriptoriumadmin/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-09-16 23:43:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

chore: format

Browse source
This commit is contained in:
kela4 2026-04-29 21:58:53 +02:00
parent 25a0475598
commit 2b5d6a7a15
3 changed files with 15 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
backend/open_webui/main.py
View file

@ -1173,11 +1173,11 @@ try:
app.state.rag_tokenizer = get_rag_tokenizer(tokenizer_model_name)
if app.state.rag_tokenizer is None:
log.error(
f'RAG tokenizer model \'{tokenizer_model_name}\' could not be loaded at startup. '
f"RAG tokenizer model '{tokenizer_model_name}' could not be loaded at startup. "
f'Uploads using token_transformers splitter will fail until the model is available.'
)
else:
log.info(f'RAG tokenizer model \'{tokenizer_model_name}\' loaded successfully')
log.info(f"RAG tokenizer model '{tokenizer_model_name}' loaded successfully")
except Exception as e:
log.error(f'Error updating models: {e}')
pass

14
backend/open_webui/routers/retrieval.py
View file

@ -1097,7 +1097,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
request.app.state.config.CHUNK_OVERLAP = (
form_data.CHUNK_OVERLAP if form_data.CHUNK_OVERLAP is not None else request.app.state.config.CHUNK_OVERLAP
)
if form_data.RAG_TOKENIZER_MODEL is not None and form_data.RAG_TOKENIZER_MODEL != str(request.app.state.config.RAG_TOKENIZER_MODEL):
if form_data.RAG_TOKENIZER_MODEL is not None and form_data.RAG_TOKENIZER_MODEL != str(
request.app.state.config.RAG_TOKENIZER_MODEL
):
request.app.state.config.RAG_TOKENIZER_MODEL = form_data.RAG_TOKENIZER_MODEL
request.app.state.rag_tokenizer = None # invalidate cache so it reloads on next ingestion
@ -1370,13 +1372,13 @@ def get_transformers_tokenizer_for_text_splitter(request: Request):
if tokenizer is None:
raise ValueError(
f"RAG_TOKENIZER_MODEL is set to '{tokenizer_model}' but failed to load — "
f"check the application logs for details. "
f"Ensure the model name is correct and the model can be downloaded or is already cached locally."
f'check the application logs for details. '
f'Ensure the model name is correct and the model can be downloaded or is already cached locally.'
)
request.app.state.rag_tokenizer = tokenizer
return tokenizer
if request.app.state.ef is not None and hasattr(request.app.state.ef, "tokenizer"):
if request.app.state.ef is not None and hasattr(request.app.state.ef, 'tokenizer'):
return request.app.state.ef.tokenizer
return None
@ -1408,7 +1410,9 @@ def merge_docs_to_target_size(
if tokenizer is not None:
measure_chunk_size = lambda text: len(tokenizer.encode(text, add_special_tokens=False))
else:
log.warning('token_transformers tokenizer unavailable in merge_docs_to_target_size, falling back to character counting')
log.warning(
'token_transformers tokenizer unavailable in merge_docs_to_target_size, falling back to character counting'
)
processed_chunks: list[Document] = []

6
src/lib/components/admin/Settings/Documents.svelte
View file

@ -827,7 +827,9 @@
>
<option value="">{$i18n.t('Default')} ({$i18n.t('Character')})</option>
<option value="token">{$i18n.t('Token')} ({$i18n.t('Tiktoken')})</option>
<option value="token_transformers">{$i18n.t('Token')} ({$i18n.t('Transformers')})</option>
<option value="token_transformers"
>{$i18n.t('Token')} ({$i18n.t('Transformers')})</option
>
</select>
</div>
</div>
@ -840,7 +842,7 @@
<Tooltip
placement="top-start"
content={$i18n.t(
'HuggingFace repository name of a model to load its tokenizer locally for exact token length calculation (e.g. sentence-transformers/all-MiniLM-L6-v2). Takes priority over the local embedding model\'s tokenizer when set. Required when using an external embedding API.'
"HuggingFace repository name of a model to load its tokenizer locally for exact token length calculation (e.g. sentence-transformers/all-MiniLM-L6-v2). Takes priority over the local embedding model's tokenizer when set. Required when using an external embedding API."
)}
>
{$i18n.t('Tokenizer Model (HuggingFace Repo)')}