From b171b0216b916745420c7caf513093a315ed9560 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Tue, 17 Mar 2026 17:54:59 -0500 Subject: [PATCH] refac --- backend/open_webui/config.py | 9 ++++++++ backend/open_webui/main.py | 2 ++ backend/open_webui/routers/retrieval.py | 6 +++++ backend/open_webui/tools/builtin.py | 6 ++--- .../admin/Settings/WebSearch.svelte | 22 +++++++++++++++++++ 5 files changed, 42 insertions(+), 3 deletions(-) diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index fb313e2785..13de48555a 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -3345,6 +3345,15 @@ WEB_SEARCH_CONCURRENT_REQUESTS = PersistentConfig( int(os.getenv("WEB_SEARCH_CONCURRENT_REQUESTS", "0")), ) +WEB_FETCH_MAX_CONTENT_LENGTH = PersistentConfig( + "WEB_FETCH_MAX_CONTENT_LENGTH", + "rag.web.search.fetch_url_max_content_length", + ( + int(os.environ.get("WEB_FETCH_MAX_CONTENT_LENGTH")) + if os.environ.get("WEB_FETCH_MAX_CONTENT_LENGTH") + else None + ), +) WEB_LOADER_ENGINE = PersistentConfig( "WEB_LOADER_ENGINE", diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index 18981940ad..93b3b5bece 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -305,6 +305,7 @@ from open_webui.config import ( BYPASS_WEB_SEARCH_WEB_LOADER, WEB_SEARCH_RESULT_COUNT, WEB_SEARCH_CONCURRENT_REQUESTS, + WEB_FETCH_MAX_CONTENT_LENGTH, WEB_SEARCH_TRUST_ENV, WEB_SEARCH_DOMAIN_FILTER_LIST, OLLAMA_CLOUD_WEB_SEARCH_API_KEY, @@ -1045,6 +1046,7 @@ app.state.config.WEB_SEARCH_ENGINE = WEB_SEARCH_ENGINE app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST = WEB_SEARCH_DOMAIN_FILTER_LIST app.state.config.WEB_SEARCH_RESULT_COUNT = WEB_SEARCH_RESULT_COUNT app.state.config.WEB_SEARCH_CONCURRENT_REQUESTS = WEB_SEARCH_CONCURRENT_REQUESTS +app.state.config.WEB_FETCH_MAX_CONTENT_LENGTH = WEB_FETCH_MAX_CONTENT_LENGTH app.state.config.WEB_LOADER_ENGINE = WEB_LOADER_ENGINE app.state.config.WEB_LOADER_CONCURRENT_REQUESTS = WEB_LOADER_CONCURRENT_REQUESTS diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index f63871224a..3e509d2f88 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -537,6 +537,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)): "WEB_SEARCH_TRUST_ENV": request.app.state.config.WEB_SEARCH_TRUST_ENV, "WEB_SEARCH_RESULT_COUNT": request.app.state.config.WEB_SEARCH_RESULT_COUNT, "WEB_SEARCH_CONCURRENT_REQUESTS": request.app.state.config.WEB_SEARCH_CONCURRENT_REQUESTS, + "WEB_FETCH_MAX_CONTENT_LENGTH": request.app.state.config.WEB_FETCH_MAX_CONTENT_LENGTH, "WEB_LOADER_CONCURRENT_REQUESTS": request.app.state.config.WEB_LOADER_CONCURRENT_REQUESTS, "WEB_SEARCH_DOMAIN_FILTER_LIST": request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST, "BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL": request.app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL, @@ -604,6 +605,7 @@ class WebConfig(BaseModel): WEB_SEARCH_TRUST_ENV: Optional[bool] = None WEB_SEARCH_RESULT_COUNT: Optional[int] = None WEB_SEARCH_CONCURRENT_REQUESTS: Optional[int] = None + WEB_FETCH_MAX_CONTENT_LENGTH: Optional[int] = None WEB_LOADER_CONCURRENT_REQUESTS: Optional[int] = None WEB_SEARCH_DOMAIN_FILTER_LIST: Optional[List[str]] = [] BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL: Optional[bool] = None @@ -1109,6 +1111,9 @@ async def update_rag_config( request.app.state.config.WEB_SEARCH_CONCURRENT_REQUESTS = ( form_data.web.WEB_SEARCH_CONCURRENT_REQUESTS ) + request.app.state.config.WEB_FETCH_MAX_CONTENT_LENGTH = ( + form_data.web.WEB_FETCH_MAX_CONTENT_LENGTH + ) request.app.state.config.WEB_LOADER_CONCURRENT_REQUESTS = ( form_data.web.WEB_LOADER_CONCURRENT_REQUESTS ) @@ -1293,6 +1298,7 @@ async def update_rag_config( "WEB_SEARCH_TRUST_ENV": request.app.state.config.WEB_SEARCH_TRUST_ENV, "WEB_SEARCH_RESULT_COUNT": request.app.state.config.WEB_SEARCH_RESULT_COUNT, "WEB_SEARCH_CONCURRENT_REQUESTS": request.app.state.config.WEB_SEARCH_CONCURRENT_REQUESTS, + "FETCH_URL_MAX_CONTENT_LENGTH": request.app.state.config.FETCH_URL_MAX_CONTENT_LENGTH, "WEB_LOADER_CONCURRENT_REQUESTS": request.app.state.config.WEB_LOADER_CONCURRENT_REQUESTS, "WEB_SEARCH_DOMAIN_FILTER_LIST": request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST, "BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL": request.app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL, diff --git a/backend/open_webui/tools/builtin.py b/backend/open_webui/tools/builtin.py index 594a7d24ac..e556d8a355 100644 --- a/backend/open_webui/tools/builtin.py +++ b/backend/open_webui/tools/builtin.py @@ -202,9 +202,9 @@ async def fetch_url( try: content, _ = await asyncio.to_thread(get_content_from_url, __request__, url) - # Truncate if too long (avoid overwhelming context) - max_length = 50000 - if len(content) > max_length: + # Truncate if configured (WEB_FETCH_MAX_CONTENT_LENGTH) + max_length = getattr(__request__.app.state.config, "WEB_FETCH_MAX_CONTENT_LENGTH", None) + if max_length and max_length > 0 and len(content) > max_length: content = content[:max_length] + "\n\n[Content truncated...]" return content diff --git a/src/lib/components/admin/Settings/WebSearch.svelte b/src/lib/components/admin/Settings/WebSearch.svelte index ebe4dc0ced..bf2fd33768 100644 --- a/src/lib/components/admin/Settings/WebSearch.svelte +++ b/src/lib/components/admin/Settings/WebSearch.svelte @@ -872,6 +872,27 @@ +
+
+ + {$i18n.t('Fetch URL Content Length Limit')} + +
+ + +
+
{$i18n.t('Domain Filter List')} @@ -1161,6 +1182,7 @@
+ {/if}