scriptoriumadmin/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-09-23 00:41:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

fix: continue on per-URL errors in SafeFireCrawlLoader

Browse source 
The try/except wrapped the entire for loop, so the first URL that
returned an HTTP error from Firecrawl (e.g. 402, 403, 404, 400)
terminated the loop and silently dropped every URL that came after it.
The `continue_on_failure` flag was effectively a no-op: it suppressed
the exception but did not actually continue iterating.
This commit is contained in:
Álvaro Justen (@turicas) 2026-04-27 16:42:06 -03:00
parent c3edd41d1f
commit 03edefb816
1 changed files with 23 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
backend/open_webui/retrieval/web/utils.py
View file

@ -218,8 +218,8 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
self.params = params or {}
def lazy_load(self) -> Iterator[Document]:
try:
for url in self.web_paths:
for url in self.web_paths:
try:
doc = scrape_firecrawl_url(
self.api_url,
self.api_key,
@ -230,21 +230,30 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
)
if doc is not None:
yield doc
except Exception as e:
if self.continue_on_failure:
log.warning(f'Error extracting content from URLs with Firecrawl: {e}')
else:
except Exception as e:
if self.continue_on_failure:
log.warning(f'Error extracting content from {url} with Firecrawl: {e}')
continue
raise e
async def alazy_load(self):
try:
docs = await run_in_threadpool(lambda: list(self.lazy_load()))
for doc in docs:
yield doc
except Exception as e:
if self.continue_on_failure:
log.warning(f'Error extracting content from URLs with Firecrawl: {e}')
else:
for url in self.web_paths:
try:
doc = await run_in_threadpool(
scrape_firecrawl_url,
self.api_url,
self.api_key,
url,
verify_ssl=self.verify_ssl,
timeout=self.timeout,
params=self.params,
)
if doc is not None:
yield doc
except Exception as e:
if self.continue_on_failure:
log.warning(f'Error extracting content from {url} with Firecrawl: {e}')
continue
raise e