From d4030a8aa5d48c2a1cb06c461566844aca2530ab Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Sun, 31 May 2026 15:10:48 -0700 Subject: [PATCH] refac --- backend/open_webui/config.py | 6 +++++ backend/open_webui/main.py | 2 ++ backend/open_webui/retrieval/loaders/main.py | 2 +- backend/open_webui/routers/retrieval.py | 7 +++++ .../admin/Settings/Documents.svelte | 26 ++++++++++++++++++- 5 files changed, 41 insertions(+), 2 deletions(-) diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index 67a76b1e54..8613a6a46b 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -1109,6 +1109,12 @@ MINERU_PARAMS = ConfigVar( mineru_params, ) +MINERU_FILE_EXTENSIONS = ConfigVar( + 'MINERU_FILE_EXTENSIONS', + 'rag.mineru_file_extensions', + [ext.strip() for ext in os.getenv('MINERU_FILE_EXTENSIONS', 'pdf').split(',') if ext.strip()], +) + EXTERNAL_DOCUMENT_LOADER_URL = ConfigVar( 'EXTERNAL_DOCUMENT_LOADER_URL', 'rag.external_document_loader_url', diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index 6342aacd35..2c1cbf227a 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -281,6 +281,7 @@ from open_webui.config import ( MINERU_API_MODE, MINERU_API_TIMEOUT, MINERU_API_URL, + MINERU_FILE_EXTENSIONS, MINERU_PARAMS, MISTRAL_OCR_API_BASE_URL, MISTRAL_OCR_API_KEY, @@ -1022,6 +1023,7 @@ app.state.config.MINERU_API_URL = MINERU_API_URL app.state.config.MINERU_API_KEY = MINERU_API_KEY app.state.config.MINERU_API_TIMEOUT = MINERU_API_TIMEOUT app.state.config.MINERU_PARAMS = MINERU_PARAMS +app.state.config.MINERU_FILE_EXTENSIONS = MINERU_FILE_EXTENSIONS app.state.config.TEXT_SPLITTER = RAG_TEXT_SPLITTER app.state.config.ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER = ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER diff --git a/backend/open_webui/retrieval/loaders/main.py b/backend/open_webui/retrieval/loaders/main.py index b322c84279..b262e66317 100644 --- a/backend/open_webui/retrieval/loaders/main.py +++ b/backend/open_webui/retrieval/loaders/main.py @@ -371,7 +371,7 @@ class Loader: azure_credential=DefaultAzureCredential(), api_model=self.kwargs.get('DOCUMENT_INTELLIGENCE_MODEL'), ) - elif self.engine == 'mineru' and file_ext in ['pdf']: # MinerU currently only supports PDF + elif self.engine == 'mineru' and file_ext in self.kwargs.get('MINERU_FILE_EXTENSIONS', ['pdf']): mineru_timeout = self.kwargs.get('MINERU_API_TIMEOUT', 300) if mineru_timeout: try: diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index f9a27b2529..f95736a953 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -466,6 +466,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)): 'MINERU_API_KEY': request.app.state.config.MINERU_API_KEY, 'MINERU_API_TIMEOUT': request.app.state.config.MINERU_API_TIMEOUT, 'MINERU_PARAMS': request.app.state.config.MINERU_PARAMS, + 'MINERU_FILE_EXTENSIONS': request.app.state.config.MINERU_FILE_EXTENSIONS, # Reranking settings 'RAG_RERANKING_MODEL': request.app.state.config.RAG_RERANKING_MODEL, 'RAG_RERANKING_ENGINE': request.app.state.config.RAG_RERANKING_ENGINE, @@ -681,6 +682,7 @@ class ConfigForm(BaseModel): MINERU_API_KEY: str | None = None MINERU_API_TIMEOUT: str | None = None MINERU_PARAMS: dict | None = None + MINERU_FILE_EXTENSIONS: list[str] | None = None # Reranking settings RAG_RERANKING_MODEL: str | None = None @@ -904,6 +906,11 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend request.app.state.config.MINERU_PARAMS = ( form_data.MINERU_PARAMS if form_data.MINERU_PARAMS is not None else request.app.state.config.MINERU_PARAMS ) + request.app.state.config.MINERU_FILE_EXTENSIONS = ( + form_data.MINERU_FILE_EXTENSIONS + if form_data.MINERU_FILE_EXTENSIONS is not None + else request.app.state.config.MINERU_FILE_EXTENSIONS + ) # Reranking settings if request.app.state.config.RAG_RERANKING_ENGINE == '': diff --git a/src/lib/components/admin/Settings/Documents.svelte b/src/lib/components/admin/Settings/Documents.svelte index e173c74969..92706ff94d 100644 --- a/src/lib/components/admin/Settings/Documents.svelte +++ b/src/lib/components/admin/Settings/Documents.svelte @@ -244,7 +244,10 @@ MINERU_PARAMS: typeof RAGConfig.MINERU_PARAMS === 'string' && RAGConfig.MINERU_PARAMS.trim() !== '' ? JSON.parse(RAGConfig.MINERU_PARAMS) - : {} + : {}, + MINERU_FILE_EXTENSIONS: RAGConfig.MINERU_FILE_EXTENSIONS.split(',') + .map((ext) => ext.trim()) + .filter((ext) => ext !== '') }); dispatch('save'); }; @@ -286,6 +289,8 @@ ? JSON.stringify(config.MINERU_PARAMS ?? {}, null, 2) : config.MINERU_PARAMS; + config.MINERU_FILE_EXTENSIONS = (config?.MINERU_FILE_EXTENSIONS ?? ['pdf']).join(', '); + RAGConfig = config; }); @@ -763,6 +768,25 @@ /> + + +
+
+ + {$i18n.t('File Extensions')} + +
+ +
{/if}