diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py
index 75d240d700..1fcd9ae977 100644
--- a/backend/open_webui/main.py
+++ b/backend/open_webui/main.py
@@ -1173,11 +1173,11 @@ try:
app.state.rag_tokenizer = get_rag_tokenizer(tokenizer_model_name)
if app.state.rag_tokenizer is None:
log.error(
- f'RAG tokenizer model \'{tokenizer_model_name}\' could not be loaded at startup. '
+ f"RAG tokenizer model '{tokenizer_model_name}' could not be loaded at startup. "
f'Uploads using token_transformers splitter will fail until the model is available.'
)
else:
- log.info(f'RAG tokenizer model \'{tokenizer_model_name}\' loaded successfully')
+ log.info(f"RAG tokenizer model '{tokenizer_model_name}' loaded successfully")
except Exception as e:
log.error(f'Error updating models: {e}')
pass
diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py
index 40b459bf7c..c06bab0d7f 100644
--- a/backend/open_webui/routers/retrieval.py
+++ b/backend/open_webui/routers/retrieval.py
@@ -1097,7 +1097,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
request.app.state.config.CHUNK_OVERLAP = (
form_data.CHUNK_OVERLAP if form_data.CHUNK_OVERLAP is not None else request.app.state.config.CHUNK_OVERLAP
)
- if form_data.RAG_TOKENIZER_MODEL is not None and form_data.RAG_TOKENIZER_MODEL != str(request.app.state.config.RAG_TOKENIZER_MODEL):
+ if form_data.RAG_TOKENIZER_MODEL is not None and form_data.RAG_TOKENIZER_MODEL != str(
+ request.app.state.config.RAG_TOKENIZER_MODEL
+ ):
request.app.state.config.RAG_TOKENIZER_MODEL = form_data.RAG_TOKENIZER_MODEL
request.app.state.rag_tokenizer = None # invalidate cache so it reloads on next ingestion
@@ -1370,13 +1372,13 @@ def get_transformers_tokenizer_for_text_splitter(request: Request):
if tokenizer is None:
raise ValueError(
f"RAG_TOKENIZER_MODEL is set to '{tokenizer_model}' but failed to load — "
- f"check the application logs for details. "
- f"Ensure the model name is correct and the model can be downloaded or is already cached locally."
+ f'check the application logs for details. '
+ f'Ensure the model name is correct and the model can be downloaded or is already cached locally.'
)
request.app.state.rag_tokenizer = tokenizer
return tokenizer
- if request.app.state.ef is not None and hasattr(request.app.state.ef, "tokenizer"):
+ if request.app.state.ef is not None and hasattr(request.app.state.ef, 'tokenizer'):
return request.app.state.ef.tokenizer
return None
@@ -1408,7 +1410,9 @@ def merge_docs_to_target_size(
if tokenizer is not None:
measure_chunk_size = lambda text: len(tokenizer.encode(text, add_special_tokens=False))
else:
- log.warning('token_transformers tokenizer unavailable in merge_docs_to_target_size, falling back to character counting')
+ log.warning(
+ 'token_transformers tokenizer unavailable in merge_docs_to_target_size, falling back to character counting'
+ )
processed_chunks: list[Document] = []
diff --git a/src/lib/components/admin/Settings/Documents.svelte b/src/lib/components/admin/Settings/Documents.svelte
index 93372526cd..1dd1c90faa 100644
--- a/src/lib/components/admin/Settings/Documents.svelte
+++ b/src/lib/components/admin/Settings/Documents.svelte
@@ -827,7 +827,9 @@
>
-
+
@@ -840,7 +842,7 @@
{$i18n.t('Tokenizer Model (HuggingFace Repo)')}