From 1682b9727b4d3e0990d484f8773369c661ee15f5 Mon Sep 17 00:00:00 2001 From: Thomas Menard Date: Sun, 3 May 2026 17:20:03 +0200 Subject: [PATCH] fix(ci): resolve httpx.AsyncClient and code quality issues - Replace httpx.AsyncClient with get_async_httpx_client (performance) - Remove unused imports (uuid, WebFetchResponse, AllMessageValues, List) - Fix inline json imports - Update tests to mock get_async_httpx_client instead of httpx.AsyncClient --- .../webfetch_interception/handler.py | 5 +- .../webfetch_interception/tools.py | 2 +- .../llms/firecrawl/fetch/transformation.py | 107 ++++-------------- tests/webfetch_tests/test_firecrawl_fetch.py | 22 ++-- 4 files changed, 38 insertions(+), 98 deletions(-) diff --git a/litellm/integrations/webfetch_interception/handler.py b/litellm/integrations/webfetch_interception/handler.py index 3996b2e8d07..82a728a91ed 100644 --- a/litellm/integrations/webfetch_interception/handler.py +++ b/litellm/integrations/webfetch_interception/handler.py @@ -8,7 +8,6 @@ server-side using litellm router's fetch tools. import asyncio import math -import uuid from typing import Any, Dict, List, Optional, Tuple, Union, cast import litellm @@ -25,12 +24,11 @@ from litellm.integrations.webfetch_interception.tools import ( from litellm.integrations.webfetch_interception.transformation import ( WebFetchTransformation, ) -from litellm.llms.base_llm.fetch.transformation import BaseFetchConfig, WebFetchResponse +from litellm.llms.base_llm.fetch.transformation import BaseFetchConfig from litellm.types.integrations.custom_logger import ( AgenticLoopPlan, AgenticLoopRequestPatch, ) -from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager @@ -893,7 +891,6 @@ class WebFetchInterceptionLogger(CustomLogger): url = args.get("url") elif isinstance(args, str): import json - try: args_dict = json.loads(args) url = args_dict.get("url") diff --git a/litellm/integrations/webfetch_interception/tools.py b/litellm/integrations/webfetch_interception/tools.py index cee5cd0d5a4..3b9f69efb47 100644 --- a/litellm/integrations/webfetch_interception/tools.py +++ b/litellm/integrations/webfetch_interception/tools.py @@ -6,7 +6,7 @@ Native provider tools (like Anthropic's web_fetch_20250305) are converted to this format for consistent interception and execution. """ -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional from litellm.constants import LITELLM_WEB_FETCH_TOOL_NAME diff --git a/litellm/llms/firecrawl/fetch/transformation.py b/litellm/llms/firecrawl/fetch/transformation.py index 98714374403..003381187f8 100644 --- a/litellm/llms/firecrawl/fetch/transformation.py +++ b/litellm/llms/firecrawl/fetch/transformation.py @@ -2,14 +2,13 @@ Firecrawl Fetch API module. Calls Firecrawl's /scrape endpoint to fetch and scrape web content. +Supports both Firecrawl Cloud and self-hosted instances. """ -import json from typing import Any, Dict, Optional -import httpx - from litellm.llms.base_llm.fetch.transformation import BaseFetchConfig, WebFetchResponse +from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.secret_managers.main import get_secret_str @@ -95,88 +94,32 @@ class FirecrawlFetchConfig(BaseFetchConfig): # Allow passing api_base via optional_params for dynamic override api_base = optional_params.get("api_base") - async with httpx.AsyncClient(timeout=30.0) as client: - response = await client.post( - self.get_complete_url(api_base=api_base), - headers=headers, - json=request_data, - ) - headers["Authorization"] = f"Bearer {api_key}" - headers["Content-Type"] = "application/json" - return headers - - def get_complete_url(self, api_base: Optional[str] = None) -> str: - """Get complete URL for Fetch endpoint.""" - api_base = ( - api_base or get_secret_str("FIRECRAWL_API_BASE") or self.FIRECRAWL_API_BASE + client = get_async_httpx_client() + response = await client.post( + self.get_complete_url(api_base=api_base), + headers=headers, + json=request_data, + timeout=30.0, ) - # Append "/scrape" to the api base if it's not already there - if not api_base.endswith("/scrape"): - api_base = f"{api_base}/scrape" - - return api_base - - async def afetch_url( - self, - url: str, - headers: Dict[str, str], - optional_params: Dict[str, Any], - ) -> WebFetchResponse: - """ - Fetch content from a URL using Firecrawl's /scrape endpoint. - - Args: - url: URL to fetch - headers: HTTP headers (including auth) - optional_params: Optional parameters for the request - - formats: List of formats to return (default: ["markdown"]) - - onlyMainContent: bool (default: True) - - includeTags: List[str] - tags to include - - excludeTags: List[str] - tags to exclude - - headers: Dict[str, str] - custom headers - - waitFor: int - time to wait in ms - - timeout: int - timeout in ms - - Returns: - WebFetchResponse with content - """ - request_data = { - "url": url, - "formats": optional_params.get("formats", ["markdown"]), - "onlyMainContent": optional_params.get("onlyMainContent", True), - } - - # Add optional parameters - for key in ["includeTags", "excludeTags", "headers", "waitFor", "timeout"]: - if key in optional_params: - request_data[key] = optional_params[key] - - async with httpx.AsyncClient(timeout=30.0) as client: - response = await client.post( - self.get_complete_url(), - headers=headers, - json=request_data, + if response.status_code != 200: + raise self.get_error_class( + error_message=f"Firecrawl fetch failed: {response.status_code} - {response.text}", + status_code=response.status_code, + headers=dict(response.headers), ) - if response.status_code != 200: - raise self.get_error_class( - error_message=f"Firecrawl fetch failed: {response.status_code} - {response.text}", - status_code=response.status_code, - headers=dict(response.headers), - ) + data = response.json() - data = response.json() + # Firecrawl response format: {"data": {"markdown": "...", "metadata": {...}}} + response_data = data.get("data", {}) + content = response_data.get("markdown") or response_data.get("content", "") + metadata = response_data.get("metadata", {}) + title = metadata.get("title") if isinstance(metadata, dict) else None - # Firecrawl response format: {"data": {"markdown": "...", "metadata": {...}}} - response_data = data.get("data", {}) - content = response_data.get("markdown") or response_data.get("content", "") - metadata = response_data.get("metadata", {}) - title = metadata.get("title") if isinstance(metadata, dict) else None - - return WebFetchResponse( - url=url, - title=title, - content=content, - metadata=metadata if isinstance(metadata, dict) else None, - ) + return WebFetchResponse( + url=url, + title=title, + content=content, + metadata=metadata if isinstance(metadata, dict) else None, + ) diff --git a/tests/webfetch_tests/test_firecrawl_fetch.py b/tests/webfetch_tests/test_firecrawl_fetch.py index 37f0849e003..1359a45836d 100644 --- a/tests/webfetch_tests/test_firecrawl_fetch.py +++ b/tests/webfetch_tests/test_firecrawl_fetch.py @@ -3,7 +3,7 @@ WebFetch tests for Firecrawl fetch provider. """ import pytest -from unittest.mock import AsyncMock, Mock, patch +from unittest.mock import AsyncMock, MagicMock, Mock, patch from litellm.llms.firecrawl.fetch.transformation import FirecrawlFetchConfig from litellm.llms.base_llm.fetch.transformation import WebFetchResponse @@ -16,16 +16,14 @@ class TestFirecrawlFetchConfig: def fetch_config(self): return FirecrawlFetchConfig() - @pytest.mark.asyncio - async def test_validate_environment_with_api_key(self, fetch_config): + def test_validate_environment_with_api_key(self, fetch_config): """Test that API key validation works.""" headers = {} result = fetch_config.validate_environment(headers, api_key="test-key") assert result["Authorization"] == "Bearer test-key" assert result["Content-Type"] == "application/json" - @pytest.mark.asyncio - async def test_validate_environment_without_api_key(self, fetch_config): + def test_validate_environment_without_api_key(self, fetch_config): """Test that missing API key raises ValueError.""" headers = {} with patch( @@ -59,11 +57,12 @@ class TestFirecrawlFetchConfig: mock_response.headers = {} mock_client = AsyncMock() - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=None) mock_client.post = AsyncMock(return_value=mock_response) - with patch("httpx.AsyncClient", return_value=mock_client): + with patch( + "litellm.llms.firecrawl.fetch.transformation.get_async_httpx_client", + return_value=mock_client, + ): result = await fetch_config.afetch_url( url="https://example.com", headers={"Authorization": "Bearer test-key"}, @@ -84,11 +83,12 @@ class TestFirecrawlFetchConfig: mock_response.headers = {} mock_client = AsyncMock() - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=None) mock_client.post = AsyncMock(return_value=mock_response) - with patch("httpx.AsyncClient", return_value=mock_client): + with patch( + "litellm.llms.firecrawl.fetch.transformation.get_async_httpx_client", + return_value=mock_client, + ): with pytest.raises(Exception, match="Firecrawl fetch failed"): await fetch_config.afetch_url( url="https://example.com",