From 03edefb8165917f2aa0993d3245d664a3ae11158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Justen=20=28=40turicas=29?= Date: Mon, 27 Apr 2026 16:42:06 -0300 Subject: [PATCH] fix: continue on per-URL errors in SafeFireCrawlLoader The try/except wrapped the entire for loop, so the first URL that returned an HTTP error from Firecrawl (e.g. 402, 403, 404, 400) terminated the loop and silently dropped every URL that came after it. The `continue_on_failure` flag was effectively a no-op: it suppressed the exception but did not actually continue iterating. --- backend/open_webui/retrieval/web/utils.py | 37 ++++++++++++++--------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index 6ee0e3781a..660e6739a4 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -218,8 +218,8 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): self.params = params or {} def lazy_load(self) -> Iterator[Document]: - try: - for url in self.web_paths: + for url in self.web_paths: + try: doc = scrape_firecrawl_url( self.api_url, self.api_key, @@ -230,21 +230,30 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): ) if doc is not None: yield doc - except Exception as e: - if self.continue_on_failure: - log.warning(f'Error extracting content from URLs with Firecrawl: {e}') - else: + except Exception as e: + if self.continue_on_failure: + log.warning(f'Error extracting content from {url} with Firecrawl: {e}') + continue raise e async def alazy_load(self): - try: - docs = await run_in_threadpool(lambda: list(self.lazy_load())) - for doc in docs: - yield doc - except Exception as e: - if self.continue_on_failure: - log.warning(f'Error extracting content from URLs with Firecrawl: {e}') - else: + for url in self.web_paths: + try: + doc = await run_in_threadpool( + scrape_firecrawl_url, + self.api_url, + self.api_key, + url, + verify_ssl=self.verify_ssl, + timeout=self.timeout, + params=self.params, + ) + if doc is not None: + yield doc + except Exception as e: + if self.continue_on_failure: + log.warning(f'Error extracting content from {url} with Firecrawl: {e}') + continue raise e