fix(ci): resolve httpx.AsyncClient and code quality issues

- Replace httpx.AsyncClient with get_async_httpx_client (performance)
- Remove unused imports (uuid, WebFetchResponse, AllMessageValues, List)
- Fix inline json imports
- Update tests to mock get_async_httpx_client instead of httpx.AsyncClient

litellm/integrations/webfetch_interception/handler.py

@@ -8,7 +8,6 @@ server-side using litellm router's fetch tools.
 
 import asyncio
 import math
-import uuid
 from typing import Any, Dict, List, Optional, Tuple, Union, cast
 
 import litellm
@@ -25,12 +24,11 @@ from litellm.integrations.webfetch_interception.tools import (
 from litellm.integrations.webfetch_interception.transformation import (
     WebFetchTransformation,
 )
-from litellm.llms.base_llm.fetch.transformation import BaseFetchConfig, WebFetchResponse
+from litellm.llms.base_llm.fetch.transformation import BaseFetchConfig
 from litellm.types.integrations.custom_logger import (
     AgenticLoopPlan,
     AgenticLoopRequestPatch,
 )
-from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import LlmProviders
 from litellm.utils import ProviderConfigManager
 
@@ -893,7 +891,6 @@ class WebFetchInterceptionLogger(CustomLogger):
                         url = args.get("url")
                     elif isinstance(args, str):
                         import json
-
                         try:
                             args_dict = json.loads(args)
                             url = args_dict.get("url")

litellm/integrations/webfetch_interception/tools.py

@@ -6,7 +6,7 @@ Native provider tools (like Anthropic's web_fetch_20250305) are converted
 to this format for consistent interception and execution.
 """
 
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Optional
 
 from litellm.constants import LITELLM_WEB_FETCH_TOOL_NAME
 

litellm/llms/firecrawl/fetch/transformation.py

@@ -2,14 +2,13 @@
 Firecrawl Fetch API module.
 
 Calls Firecrawl's /scrape endpoint to fetch and scrape web content.
+Supports both Firecrawl Cloud and self-hosted instances.
 """
 
-import json
 from typing import Any, Dict, Optional
 
-import httpx
-
 from litellm.llms.base_llm.fetch.transformation import BaseFetchConfig, WebFetchResponse
+from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.secret_managers.main import get_secret_str
 
 
@@ -95,88 +94,32 @@ class FirecrawlFetchConfig(BaseFetchConfig):
         # Allow passing api_base via optional_params for dynamic override
         api_base = optional_params.get("api_base")
 
-        async with httpx.AsyncClient(timeout=30.0) as client:
-            response = await client.post(
-                self.get_complete_url(api_base=api_base),
-                headers=headers,
-                json=request_data,
-            )
-        headers["Authorization"] = f"Bearer {api_key}"
-        headers["Content-Type"] = "application/json"
-        return headers
-
-    def get_complete_url(self, api_base: Optional[str] = None) -> str:
-        """Get complete URL for Fetch endpoint."""
-        api_base = (
-            api_base or get_secret_str("FIRECRAWL_API_BASE") or self.FIRECRAWL_API_BASE
+        client = get_async_httpx_client()
+        response = await client.post(
+            self.get_complete_url(api_base=api_base),
+            headers=headers,
+            json=request_data,
+            timeout=30.0,
         )
 
-        # Append "/scrape" to the api base if it's not already there
-        if not api_base.endswith("/scrape"):
-            api_base = f"{api_base}/scrape"
-
-        return api_base
-
-    async def afetch_url(
-        self,
-        url: str,
-        headers: Dict[str, str],
-        optional_params: Dict[str, Any],
-    ) -> WebFetchResponse:
-        """
-        Fetch content from a URL using Firecrawl's /scrape endpoint.
-
-        Args:
-            url: URL to fetch
-            headers: HTTP headers (including auth)
-            optional_params: Optional parameters for the request
-                - formats: List of formats to return (default: ["markdown"])
-                - onlyMainContent: bool (default: True)
-                - includeTags: List[str] - tags to include
-                - excludeTags: List[str] - tags to exclude
-                - headers: Dict[str, str] - custom headers
-                - waitFor: int - time to wait in ms
-                - timeout: int - timeout in ms
-
-        Returns:
-            WebFetchResponse with content
-        """
-        request_data = {
-            "url": url,
-            "formats": optional_params.get("formats", ["markdown"]),
-            "onlyMainContent": optional_params.get("onlyMainContent", True),
-        }
-
-        # Add optional parameters
-        for key in ["includeTags", "excludeTags", "headers", "waitFor", "timeout"]:
-            if key in optional_params:
-                request_data[key] = optional_params[key]
-
-        async with httpx.AsyncClient(timeout=30.0) as client:
-            response = await client.post(
-                self.get_complete_url(),
-                headers=headers,
-                json=request_data,
+        if response.status_code != 200:
+            raise self.get_error_class(
+                error_message=f"Firecrawl fetch failed: {response.status_code} - {response.text}",
+                status_code=response.status_code,
+                headers=dict(response.headers),
             )
 
-            if response.status_code != 200:
-                raise self.get_error_class(
-                    error_message=f"Firecrawl fetch failed: {response.status_code} - {response.text}",
-                    status_code=response.status_code,
-                    headers=dict(response.headers),
-                )
+        data = response.json()
 
-            data = response.json()
+        # Firecrawl response format: {"data": {"markdown": "...", "metadata": {...}}}
+        response_data = data.get("data", {})
+        content = response_data.get("markdown") or response_data.get("content", "")
+        metadata = response_data.get("metadata", {})
+        title = metadata.get("title") if isinstance(metadata, dict) else None
 
-            # Firecrawl response format: {"data": {"markdown": "...", "metadata": {...}}}
-            response_data = data.get("data", {})
-            content = response_data.get("markdown") or response_data.get("content", "")
-            metadata = response_data.get("metadata", {})
-            title = metadata.get("title") if isinstance(metadata, dict) else None
-
-            return WebFetchResponse(
-                url=url,
-                title=title,
-                content=content,
-                metadata=metadata if isinstance(metadata, dict) else None,
-            )
+        return WebFetchResponse(
+            url=url,
+            title=title,
+            content=content,
+            metadata=metadata if isinstance(metadata, dict) else None,
+        )

tests/webfetch_tests/test_firecrawl_fetch.py

@@ -3,7 +3,7 @@ WebFetch tests for Firecrawl fetch provider.
 """
 
 import pytest
-from unittest.mock import AsyncMock, Mock, patch
+from unittest.mock import AsyncMock, MagicMock, Mock, patch
 
 from litellm.llms.firecrawl.fetch.transformation import FirecrawlFetchConfig
 from litellm.llms.base_llm.fetch.transformation import WebFetchResponse
@@ -16,16 +16,14 @@ class TestFirecrawlFetchConfig:
     def fetch_config(self):
         return FirecrawlFetchConfig()
 
-    @pytest.mark.asyncio
-    async def test_validate_environment_with_api_key(self, fetch_config):
+    def test_validate_environment_with_api_key(self, fetch_config):
         """Test that API key validation works."""
         headers = {}
         result = fetch_config.validate_environment(headers, api_key="test-key")
         assert result["Authorization"] == "Bearer test-key"
         assert result["Content-Type"] == "application/json"
 
-    @pytest.mark.asyncio
-    async def test_validate_environment_without_api_key(self, fetch_config):
+    def test_validate_environment_without_api_key(self, fetch_config):
         """Test that missing API key raises ValueError."""
         headers = {}
         with patch(
@@ -59,11 +57,12 @@ class TestFirecrawlFetchConfig:
         mock_response.headers = {}
 
         mock_client = AsyncMock()
-        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
-        mock_client.__aexit__ = AsyncMock(return_value=None)
         mock_client.post = AsyncMock(return_value=mock_response)
 
-        with patch("httpx.AsyncClient", return_value=mock_client):
+        with patch(
+            "litellm.llms.firecrawl.fetch.transformation.get_async_httpx_client",
+            return_value=mock_client,
+        ):
             result = await fetch_config.afetch_url(
                 url="https://example.com",
                 headers={"Authorization": "Bearer test-key"},
@@ -84,11 +83,12 @@ class TestFirecrawlFetchConfig:
         mock_response.headers = {}
 
         mock_client = AsyncMock()
-        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
-        mock_client.__aexit__ = AsyncMock(return_value=None)
         mock_client.post = AsyncMock(return_value=mock_response)
 
-        with patch("httpx.AsyncClient", return_value=mock_client):
+        with patch(
+            "litellm.llms.firecrawl.fetch.transformation.get_async_httpx_client",
+            return_value=mock_client,
+        ):
             with pytest.raises(Exception, match="Firecrawl fetch failed"):
                 await fetch_config.afetch_url(
                     url="https://example.com",