diff --git a/docs/my-website/docs/search/index.md b/docs/my-website/docs/search/index.md index 551a495261a..8a71edead06 100644 --- a/docs/my-website/docs/search/index.md +++ b/docs/my-website/docs/search/index.md @@ -276,6 +276,7 @@ The response follows Perplexity's search format with the following structure: | Firecrawl | `FIRECRAWL_API_KEY` | `firecrawl` | | SearXNG | `SEARXNG_API_BASE` (required) | `searxng` | | Linkup | `LINKUP_API_KEY` | `linkup` | +| DuckDuckGo | `DUCKDUCKGO_API_BASE` | `duckduckgo` | See the individual provider documentation for detailed setup instructions and provider-specific parameters. diff --git a/litellm/llms/duckduckgo/search/__init__.py b/litellm/llms/duckduckgo/search/__init__.py new file mode 100644 index 00000000000..c0019637838 --- /dev/null +++ b/litellm/llms/duckduckgo/search/__init__.py @@ -0,0 +1,6 @@ +""" +DuckDuckGo Search API module. +""" +from litellm.llms.duckduckgo.search.transformation import DuckDuckGoSearchConfig + +__all__ = ["DuckDuckGoSearchConfig"] diff --git a/litellm/llms/duckduckgo/search/transformation.py b/litellm/llms/duckduckgo/search/transformation.py new file mode 100644 index 00000000000..509d69041fb --- /dev/null +++ b/litellm/llms/duckduckgo/search/transformation.py @@ -0,0 +1,252 @@ +""" +Calls DuckDuckGo's Instant Answer API to search the web. + +DuckDuckGo API Reference: https://duckduckgo.com/api +""" +from typing import Dict, List, Literal, Optional, TypedDict, Union +from urllib.parse import urlencode + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig, + SearchResponse, + SearchResult, +) +from litellm.secret_managers.main import get_secret_str + + +class _DuckDuckGoSearchRequestRequired(TypedDict): + """Required fields for DuckDuckGo Search API request.""" + q: str # Required - search query + + +class DuckDuckGoSearchRequest(_DuckDuckGoSearchRequestRequired, total=False): + """ + DuckDuckGo Instant Answer API request format. + Based on: https://duckduckgo.com/api + """ + format: str # Optional - output format ('json', 'xml'), default 'json' + pretty: int # Optional - pretty print (0 or 1), default 1 + no_redirect: int # Optional - skip HTTP redirects (0 or 1), default 0 + no_html: int # Optional - remove HTML from text (0 or 1), default 0 + skip_disambig: int # Optional - skip disambiguation results (0 or 1), default 0 + + +class DuckDuckGoSearchConfig(BaseSearchConfig): + DUCKDUCKGO_API_BASE = "https://api.duckduckgo.com" + + @staticmethod + def ui_friendly_name() -> str: + return "DuckDuckGo" + + def get_http_method(self) -> Literal["GET", "POST"]: + """ + Get HTTP method for search requests. + DuckDuckGo Instant Answer API uses GET requests. + + Returns: + HTTP method 'GET' + """ + return "GET" + + def validate_environment( + self, + headers: Dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + **kwargs, + ) -> Dict: + """ + Validate environment and return headers. + DuckDuckGo Instant Answer API does not require authentication. + """ + # DuckDuckGo API is free and doesn't require API key + headers["Content-Type"] = "application/json" + return headers + + def get_complete_url( + self, + api_base: Optional[str], + optional_params: dict, + data: Optional[Union[Dict, List[Dict]]] = None, + **kwargs, + ) -> str: + """ + Get complete URL for Search endpoint. + DuckDuckGo uses query parameters, so we construct the URL with the query. + """ + api_base = api_base or get_secret_str("DUCKDUCKGO_API_BASE") or self.DUCKDUCKGO_API_BASE + + # Build query parameters from the transformed request body + if data and isinstance(data, dict) and "_duckduckgo_params" in data: + params = data["_duckduckgo_params"] + query_string = urlencode(params, doseq=True) + return f"{api_base}/?{query_string}" + + return api_base + + + def transform_search_request( + self, + query: Union[str, List[str]], + optional_params: dict, + **kwargs, + ) -> Dict: + """ + Transform Search request to DuckDuckGo API format. + + Args: + query: Search query (string or list of strings). DuckDuckGo only supports single string queries. + optional_params: Optional parameters for the request + - max_results: Maximum number of search results (DuckDuckGo API doesn't directly support this, used for filtering) + - format: Output format ('json', 'xml') + - pretty: Pretty print (0 or 1) + - no_redirect: Skip HTTP redirects (0 or 1) + - no_html: Remove HTML from text (0 or 1) + - skip_disambig: Skip disambiguation results (0 or 1) + + Returns: + Dict with typed request data following DuckDuckGoSearchRequest spec + """ + if isinstance(query, list): + # DuckDuckGo only supports single string queries + query = " ".join(query) + + request_data: DuckDuckGoSearchRequest = { + "q": query, + "format": "json", # Always use JSON format + } + + # Convert to dict before dynamic key assignments + result_data = dict(request_data) + + if "max_results" in optional_params: + result_data["_max_results"] = optional_params["max_results"] + + # Pass through DuckDuckGo-specific parameters + ddg_params = ["pretty", "no_redirect", "no_html", "skip_disambig"] + for param in ddg_params: + if param in optional_params: + result_data[param] = optional_params[param] + + return { + "_duckduckgo_params": result_data, + } + + def transform_search_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs, + ) -> SearchResponse: + """ + Transform DuckDuckGo API response to LiteLLM unified SearchResponse format. + + DuckDuckGo → LiteLLM mappings: + - RelatedTopics[].Text → SearchResult.title + snippet + - RelatedTopics[].FirstURL → SearchResult.url + - RelatedTopics[].Text → SearchResult.snippet + - No date/last_updated fields in DuckDuckGo response (set to None) + + Args: + raw_response: Raw httpx response from DuckDuckGo API + logging_obj: Logging object for tracking + + Returns: + SearchResponse with standardized format + """ + response_json = raw_response.json() + + # Extract max_results from the request URL params + query_params = raw_response.request.url.params if raw_response.request else {} + max_results = None + if "_max_results" in query_params: + try: + max_results = int(query_params["_max_results"]) + except (ValueError, TypeError): + pass + + # Transform results to SearchResult objects + results = [] + + # DuckDuckGo can return results in different fields + # Priority: Abstract > Answer > RelatedTopics + + # Check if there's an Abstract with URL + if response_json.get("AbstractURL") and response_json.get("AbstractText"): + abstract_result = SearchResult( + title=response_json.get("Heading", ""), + url=response_json.get("AbstractURL", ""), + snippet=response_json.get("AbstractText", ""), + date=None, + last_updated=None, + ) + results.append(abstract_result) + + # Process RelatedTopics + related_topics = response_json.get("RelatedTopics", []) + for topic in related_topics: + # Stop if we've reached max_results + if max_results is not None and len(results) >= max_results: + break + + if isinstance(topic, dict): + # Check if it's a direct result + if "FirstURL" in topic and "Text" in topic: + text = topic.get("Text", "") + url = topic.get("FirstURL", "") + + # Try to split title and snippet + if " - " in text: + parts = text.split(" - ", 1) + title = parts[0] + snippet = parts[1] if len(parts) > 1 else text + else: + title = text[:50] + "..." if len(text) > 50 else text + snippet = text + + search_result = SearchResult( + title=title, + url=url, + snippet=snippet, + date=None, + last_updated=None, + ) + results.append(search_result) + + # Check if it contains nested topics + elif "Topics" in topic: + nested_topics = topic.get("Topics", []) + for nested_topic in nested_topics: + # Stop if we've reached max_results + if max_results is not None and len(results) >= max_results: + break + + if "FirstURL" in nested_topic and "Text" in nested_topic: + text = nested_topic.get("Text", "") + url = nested_topic.get("FirstURL", "") + + # Try to split title and snippet + if " - " in text: + parts = text.split(" - ", 1) + title = parts[0] + snippet = parts[1] if len(parts) > 1 else text + else: + title = text[:50] + "..." if len(text) > 50 else text + snippet = text + + search_result = SearchResult( + title=title, + url=url, + snippet=snippet, + date=None, + last_updated=None, + ) + results.append(search_result) + + return SearchResponse( + results=results, + object="search", + ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9a9acb91986..04183e398eb 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -37270,5 +37270,13 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 } + }, + "duckduckgo/search": { + "litellm_provider": "duckduckgo", + "mode": "search", + "input_cost_per_query": 0.0, + "metadata": { + "notes": "DuckDuckGo Instant Answer API is free and does not require an API key." + } } } \ No newline at end of file diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5f8798c7712..f393686a7ea 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3197,7 +3197,7 @@ class SearchProviders(str, Enum): FIRECRAWL = "firecrawl" SEARXNG = "searxng" LINKUP = "linkup" - + DUCKDUCKGO = "duckduckgo" # Create a set of all search provider values for quick lookup SearchProvidersSet = {provider.value for provider in SearchProviders} diff --git a/litellm/utils.py b/litellm/utils.py index 61402a1b30b..6a18fcc9e35 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8778,6 +8778,7 @@ class ProviderConfigManager: """ from litellm.llms.brave.search.transformation import BraveSearchConfig from litellm.llms.dataforseo.search.transformation import DataForSEOSearchConfig + from litellm.llms.duckduckgo.search.transformation import DuckDuckGoSearchConfig from litellm.llms.exa_ai.search.transformation import ExaAISearchConfig from litellm.llms.firecrawl.search.transformation import FirecrawlSearchConfig from litellm.llms.google_pse.search.transformation import GooglePSESearchConfig @@ -8800,6 +8801,7 @@ class ProviderConfigManager: SearchProviders.FIRECRAWL: FirecrawlSearchConfig, SearchProviders.SEARXNG: SearXNGSearchConfig, SearchProviders.LINKUP: LinkupSearchConfig, + SearchProviders.DUCKDUCKGO: DuckDuckGoSearchConfig, } config_class = PROVIDER_TO_CONFIG_MAP.get(provider, None) if config_class is None: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 471d4b9ab7a..ae89a82e964 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -37312,5 +37312,13 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 } + }, + "duckduckgo/search": { + "litellm_provider": "duckduckgo", + "mode": "search", + "input_cost_per_query": 0.0, + "metadata": { + "notes": "DuckDuckGo Instant Answer API is free and does not require an API key." + } } } \ No newline at end of file diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 366da0c0b46..328398a296a 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -761,6 +761,23 @@ "interactions": true } }, + "duckduckgo": { + "display_name": "DuckDuckGo (`duckduckgo`)", + "url": "https://docs.litellm.ai/docs/search/duckduckgo", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, "elevenlabs": { "display_name": "ElevenLabs (`elevenlabs`)", "url": "https://docs.litellm.ai/docs/providers/elevenlabs", diff --git a/tests/code_coverage_tests/enforce_llms_folder_style.py b/tests/code_coverage_tests/enforce_llms_folder_style.py index f684d884a6b..7e6fd8e6fd6 100644 --- a/tests/code_coverage_tests/enforce_llms_folder_style.py +++ b/tests/code_coverage_tests/enforce_llms_folder_style.py @@ -16,6 +16,7 @@ SEARCH_PROVIDERS = [ "firecrawl", "searxng", "linkup", + "duckduckgo", ] ALLOWED_FILES_IN_LLMS_FOLDER = [ @@ -73,8 +74,8 @@ def run_lint_check(unique_names): def main(): - llms_dir = "./litellm/llms/" # Update this path if needed - # llms_dir = "../../litellm/llms/" # LOCAL TESTING + # llms_dir = "./litellm/llms/" # Update this path if needed + llms_dir = "litellm/litellm/llms" # LOCAL TESTING unique_names = get_unique_names_from_llms_dir(llms_dir) print("Unique names in llms directory:", sorted(list(unique_names))) diff --git a/tests/search_tests/test_duckduckgo_search.py b/tests/search_tests/test_duckduckgo_search.py new file mode 100644 index 00000000000..13df1cff9d4 --- /dev/null +++ b/tests/search_tests/test_duckduckgo_search.py @@ -0,0 +1,356 @@ +""" +Tests for DuckDuckGo Search API integration. +""" +import os +import sys +import pytest +from unittest.mock import AsyncMock, patch, MagicMock + +sys.path.insert( + 0, os.path.abspath("../..") +) + +import litellm +from tests.search_tests.base_search_unit_tests import BaseSearchTest + + +class TestDuckDuckGoSearch(BaseSearchTest): + """ + Tests for DuckDuckGo Search functionality. + """ + + def get_search_provider(self) -> str: + """ + Return search_provider for DuckDuckGo Search. + """ + return "duckduckgo" + + @pytest.mark.asyncio + async def test_basic_search(self): + """ + Test basic search functionality with a simple query. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm._turn_on_debug() + search_provider = self.get_search_provider() + print("Search Provider=", search_provider) + + try: + response = await litellm.asearch( + query="india", + search_provider=search_provider, + ) + print("Search response=", response.model_dump_json(indent=4)) + + print(f"\n{'='*80}") + print(f"Response type: {type(response)}") + print(f"Response object: {response.object if hasattr(response, 'object') else 'N/A'}") + + # Check if response has expected Search format + assert hasattr(response, "results"), "Response should have 'results' attribute" + assert hasattr(response, "object"), "Response should have 'object' attribute" + assert response.object == "search", f"Expected object='search', got '{response.object}'" + + # Validate results structure + assert isinstance(response.results, list), "results should be a list" + assert len(response.results) > 0, "Should have at least one result" + + # Check first result structure + first_result = response.results[0] + assert hasattr(first_result, "title"), "Result should have 'title' attribute" + assert hasattr(first_result, "url"), "Result should have 'url' attribute" + assert hasattr(first_result, "snippet"), "Result should have 'snippet' attribute" + + print(f"Total results: {len(response.results)}") + print(f"First result title: {first_result.title}") + print(f"First result URL: {first_result.url}") + print(f"First result snippet: {first_result.snippet[:100]}...") + print(f"{'='*80}\n") + + assert len(first_result.title) > 0, "Title should not be empty" + assert len(first_result.url) > 0, "URL should not be empty" + assert len(first_result.snippet) > 0, "Snippet should not be empty" + + # Validate cost tracking in _hidden_params + assert hasattr(response, "_hidden_params"), "Response should have '_hidden_params' attribute" + hidden_params = response._hidden_params + assert "response_cost" in hidden_params, "_hidden_params should contain 'response_cost'" + + response_cost = hidden_params["response_cost"] + assert response_cost is not None, "response_cost should not be None" + assert isinstance(response_cost, (int, float)), "response_cost should be a number" + assert response_cost == 0, "response_cost should be 0" + + print(f"Cost tracking: ${response_cost:.6f}") + + except Exception as e: + pytest.fail(f"Search call failed: {str(e)}") + + + def test_search_response_structure(self): + """ + Test that the Search response has the correct structure. + """ + litellm.set_verbose = True + search_provider = self.get_search_provider() + + response = litellm.search( + query="india", + search_provider=search_provider, + ) + + # Validate response structure + assert hasattr(response, "results"), "Response should have 'results' attribute" + assert hasattr(response, "object"), "Response should have 'object' attribute" + + assert isinstance(response.results, list), "results should be a list" + assert len(response.results) > 0, "Should have at least one result" + assert response.object == "search", "object should be 'search'" + + # Validate first result structure + first_result = response.results[0] + assert hasattr(first_result, "title"), "Result should have 'title' attribute" + assert hasattr(first_result, "url"), "Result should have 'url' attribute" + assert hasattr(first_result, "snippet"), "Result should have 'snippet' attribute" + assert isinstance(first_result.title, str), "title should be a string" + assert isinstance(first_result.url, str), "url should be a string" + assert isinstance(first_result.snippet, str), "snippet should be a string" + + print(f"\nResponse structure validated:") + print(f" - object: {response.object}") + print(f" - results: {len(response.results)}") + print(f" - first result has all required fields") + +class TestDuckDuckGoSearchMocked: + """ + Tests for DuckDuckGo Search functionality with mocked network responses. + """ + + @pytest.mark.asyncio + async def test_duckduckgo_search_request_payload(self): + """ + Test that validates the DuckDuckGo search request payload structure without making real API calls. + """ + # Create a mock response matching DuckDuckGo API format + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "Abstract": "", + "AbstractSource": "Wikipedia", + "AbstractText": "Python is a high-level programming language.", + "AbstractURL": "https://en.wikipedia.org/wiki/Python_(programming_language)", + "Answer": "", + "AnswerType": "", + "Definition": "", + "DefinitionSource": "", + "DefinitionURL": "", + "Entity": "", + "Heading": "Python (programming language)", + "Image": "", + "ImageHeight": 0, + "ImageIsLogo": 0, + "ImageWidth": 0, + "Infobox": "", + "Redirect": "", + "RelatedTopics": [ + { + "FirstURL": "https://duckduckgo.com/Python_programming", + "Icon": { + "Height": "", + "URL": "/i/python.png", + "Width": "" + }, + "Result": "Python Programming A general-purpose programming language.", + "Text": "Python Programming - A general-purpose programming language." + }, + { + "FirstURL": "https://duckduckgo.com/Python_packages", + "Icon": { + "Height": "", + "URL": "", + "Width": "" + }, + "Result": "Python Packages Package management in Python.", + "Text": "Python Packages - Package management in Python." + } + ], + "Results": [], + "Type": "A", + "meta": { + "attribution": None, + "blockgroup": None, + "created_date": None, + "description": "Wikipedia", + "designer": None, + "dev_date": None, + "dev_milestone": "live", + "developer": [ + { + "name": "DDG Team", + "type": "ddg", + "url": "http://www.duckduckhack.com" + } + ], + "example_query": "python programming", + "id": "wikipedia_fathead", + "is_stackexchange": None, + "js_callback_name": "wikipedia", + "live_date": None, + "maintainer": { + "github": "duckduckgo" + }, + "name": "Wikipedia", + "perl_module": "DDG::Fathead::Wikipedia", + "producer": None, + "production_state": "online", + "repo": "fathead", + "signal_from": "wikipedia_fathead", + "src_domain": "en.wikipedia.org", + "src_id": 1, + "src_name": "Wikipedia", + "src_options": { + "directory": "", + "is_fanon": 0, + "is_mediawiki": 1, + "is_wikipedia": 1, + "language": "en", + "min_abstract_length": "20", + "skip_abstract": 0, + "skip_abstract_paren": 0, + "skip_end": "0", + "skip_icon": 0, + "skip_image_name": 0, + "skip_qr": "", + "source_skip": "", + "src_info": "" + }, + "src_url": None, + "status": "live", + "tab": "About", + "topic": [ + "productivity" + ], + "unsafe": 0 + } + } + + # Mock the httpx AsyncClient get method (DuckDuckGo uses GET) + with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get", new_callable=AsyncMock) as mock_get: + mock_get.return_value = mock_response + + # Make the search call + response = await litellm.asearch( + query="python programming", + search_provider="duckduckgo", + max_results=5 + ) + + # Verify the get method was called once + assert mock_get.call_count == 1 + + # Get the actual call arguments + call_args = mock_get.call_args + + # Verify URL contains the query with proper URL encoding + url = call_args.kwargs["url"] + assert "api.duckduckgo.com" in url + # URL should be properly encoded with %20 for spaces + assert ("q=python+programming" in url or "q=python%20programming" in url) + assert "format=json" in url + + # Verify response structure + assert hasattr(response, "results") + assert hasattr(response, "object") + assert response.object == "search" + assert len(response.results) > 0 + + # Verify first result (Abstract) + first_result = response.results[0] + assert first_result.title == "Python (programming language)" + assert first_result.url == "https://en.wikipedia.org/wiki/Python_(programming_language)" + assert "Python is a high-level programming language" in first_result.snippet + + # Verify related topics are included + assert len(response.results) >= 2 # Abstract + at least one related topic + + @pytest.mark.asyncio + async def test_duckduckgo_search_disambiguation(self): + """ + Test handling of disambiguation results from DuckDuckGo. + """ + # Create a mock response with disambiguation type + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "Abstract": "", + "AbstractSource": "Wikipedia", + "AbstractText": "", + "AbstractURL": "https://en.wikipedia.org/wiki/India_(disambiguation)", + "Answer": "", + "AnswerType": "", + "Definition": "", + "DefinitionSource": "", + "DefinitionURL": "", + "Entity": "", + "Heading": "India", + "Image": "", + "ImageHeight": 0, + "ImageIsLogo": 0, + "ImageWidth": 0, + "Infobox": "", + "Redirect": "", + "RelatedTopics": [ + { + "FirstURL": "https://duckduckgo.com/India", + "Icon": { + "Height": "", + "URL": "/i/cef47a13.png", + "Width": "" + }, + "Result": "India A country in South Asia.", + "Text": "India - A country in South Asia." + }, + { + "Name": "Related Topics", + "Topics": [ + { + "FirstURL": "https://duckduckgo.com/d/Indus", + "Icon": { + "Height": "", + "URL": "", + "Width": "" + }, + "Result": "Indus See related meanings for the word 'Indus'.", + "Text": "Indus - See related meanings for the word 'Indus'." + } + ] + } + ], + "Results": [], + "Type": "D", + "meta": {} + } + + # Mock the httpx AsyncClient get method + with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get", new_callable=AsyncMock) as mock_get: + mock_get.return_value = mock_response + + # Make the search call + response = await litellm.asearch( + query="India", + search_provider="duckduckgo" + ) + + # Verify response structure + assert hasattr(response, "results") + assert hasattr(response, "object") + assert response.object == "search" + + # Should have results from both direct topics and nested topics + assert len(response.results) >= 2 + + # Verify nested topics are processed + urls = [result.url for result in response.results] + assert any("India" in url for url in urls) + assert any("Indus" in url for url in urls)