diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index 6ee0e3781a..660e6739a4 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -218,8 +218,8 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): self.params = params or {} def lazy_load(self) -> Iterator[Document]: - try: - for url in self.web_paths: + for url in self.web_paths: + try: doc = scrape_firecrawl_url( self.api_url, self.api_key, @@ -230,21 +230,30 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): ) if doc is not None: yield doc - except Exception as e: - if self.continue_on_failure: - log.warning(f'Error extracting content from URLs with Firecrawl: {e}') - else: + except Exception as e: + if self.continue_on_failure: + log.warning(f'Error extracting content from {url} with Firecrawl: {e}') + continue raise e async def alazy_load(self): - try: - docs = await run_in_threadpool(lambda: list(self.lazy_load())) - for doc in docs: - yield doc - except Exception as e: - if self.continue_on_failure: - log.warning(f'Error extracting content from URLs with Firecrawl: {e}') - else: + for url in self.web_paths: + try: + doc = await run_in_threadpool( + scrape_firecrawl_url, + self.api_url, + self.api_key, + url, + verify_ssl=self.verify_ssl, + timeout=self.timeout, + params=self.params, + ) + if doc is not None: + yield doc + except Exception as e: + if self.continue_on_failure: + log.warning(f'Error extracting content from {url} with Firecrawl: {e}') + continue raise e