Add duckduckgo as search tool
This commit is contained in:
Sameer Kankute
parent
2517c069ca
commit
fae95eee88
6
litellm/llms/duckduckgo/search/__init__.py
Normal file
6
litellm/llms/duckduckgo/search/__init__.py
Normal file
@ -0,0 +1,6 @@
|
||||
"""
|
||||
DuckDuckGo Search API module.
|
||||
"""
|
||||
from litellm.llms.duckduckgo.search.transformation import DuckDuckGoSearchConfig
|
||||
|
||||
__all__ = ["DuckDuckGoSearchConfig"]
|
||||
253
litellm/llms/duckduckgo/search/transformation.py
Normal file
253
litellm/llms/duckduckgo/search/transformation.py
Normal file
@ -0,0 +1,253 @@
|
||||
"""
|
||||
Calls DuckDuckGo's Instant Answer API to search the web.
|
||||
|
||||
DuckDuckGo API Reference: https://duckduckgo.com/api
|
||||
"""
|
||||
from typing import Dict, List, Literal, Optional, TypedDict, Union
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.search.transformation import (
|
||||
BaseSearchConfig,
|
||||
SearchResponse,
|
||||
SearchResult,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
|
||||
class _DuckDuckGoSearchRequestRequired(TypedDict):
|
||||
"""Required fields for DuckDuckGo Search API request."""
|
||||
q: str # Required - search query
|
||||
|
||||
|
||||
class DuckDuckGoSearchRequest(_DuckDuckGoSearchRequestRequired, total=False):
|
||||
"""
|
||||
DuckDuckGo Instant Answer API request format.
|
||||
Based on: https://duckduckgo.com/api
|
||||
"""
|
||||
format: str # Optional - output format ('json', 'xml'), default 'json'
|
||||
pretty: int # Optional - pretty print (0 or 1), default 1
|
||||
no_redirect: int # Optional - skip HTTP redirects (0 or 1), default 0
|
||||
no_html: int # Optional - remove HTML from text (0 or 1), default 0
|
||||
skip_disambig: int # Optional - skip disambiguation results (0 or 1), default 0
|
||||
|
||||
|
||||
class DuckDuckGoSearchConfig(BaseSearchConfig):
|
||||
DUCKDUCKGO_API_BASE = "https://api.duckduckgo.com"
|
||||
|
||||
@staticmethod
|
||||
def ui_friendly_name() -> str:
|
||||
return "DuckDuckGo"
|
||||
|
||||
def get_http_method(self) -> Literal["GET", "POST"]:
|
||||
"""
|
||||
Get HTTP method for search requests.
|
||||
DuckDuckGo Instant Answer API uses GET requests.
|
||||
|
||||
Returns:
|
||||
HTTP method 'GET'
|
||||
"""
|
||||
return "GET"
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: Dict,
|
||||
api_key: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> Dict:
|
||||
"""
|
||||
Validate environment and return headers.
|
||||
DuckDuckGo Instant Answer API does not require authentication.
|
||||
"""
|
||||
# DuckDuckGo API is free and doesn't require API key
|
||||
headers["Content-Type"] = "application/json"
|
||||
return headers
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
optional_params: dict,
|
||||
data: Optional[Union[Dict, List[Dict]]] = None,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
"""
|
||||
Get complete URL for Search endpoint.
|
||||
DuckDuckGo uses query parameters, so we construct the URL with the query.
|
||||
"""
|
||||
api_base = api_base or get_secret_str("DUCKDUCKGO_API_BASE") or self.DUCKDUCKGO_API_BASE
|
||||
|
||||
# Ensure URL ends without trailing slash for query parameters
|
||||
if api_base.endswith("/"):
|
||||
api_base = api_base.rstrip("/")
|
||||
|
||||
# Construct URL with query parameters
|
||||
if data and isinstance(data, dict):
|
||||
query_params = []
|
||||
for key, value in data.items():
|
||||
if isinstance(value, list):
|
||||
# Join list values with commas
|
||||
value = ",".join(str(v) for v in value)
|
||||
query_params.append(f"{key}={value}")
|
||||
|
||||
if query_params:
|
||||
api_base = f"{api_base}/?{'&'.join(query_params)}"
|
||||
|
||||
return api_base
|
||||
|
||||
|
||||
def transform_search_request(
|
||||
self,
|
||||
query: Union[str, List[str]],
|
||||
optional_params: dict,
|
||||
**kwargs,
|
||||
) -> Dict:
|
||||
"""
|
||||
Transform Search request to DuckDuckGo API format.
|
||||
|
||||
Args:
|
||||
query: Search query (string or list of strings). DuckDuckGo only supports single string queries.
|
||||
optional_params: Optional parameters for the request
|
||||
- max_results: Maximum number of search results (DuckDuckGo API doesn't directly support this, used for filtering)
|
||||
- format: Output format ('json', 'xml')
|
||||
- pretty: Pretty print (0 or 1)
|
||||
- no_redirect: Skip HTTP redirects (0 or 1)
|
||||
- no_html: Remove HTML from text (0 or 1)
|
||||
- skip_disambig: Skip disambiguation results (0 or 1)
|
||||
|
||||
Returns:
|
||||
Dict with typed request data following DuckDuckGoSearchRequest spec
|
||||
"""
|
||||
if isinstance(query, list):
|
||||
# DuckDuckGo only supports single string queries
|
||||
query = " ".join(query)
|
||||
|
||||
request_data: DuckDuckGoSearchRequest = {
|
||||
"q": query,
|
||||
"format": "json", # Always use JSON format
|
||||
}
|
||||
|
||||
# Store max_results for response filtering if provided
|
||||
if "max_results" in optional_params:
|
||||
# DuckDuckGo API doesn't support max_results directly
|
||||
# We'll filter the results in transform_search_response
|
||||
pass
|
||||
|
||||
# Convert to dict before dynamic key assignments
|
||||
result_data = dict(request_data)
|
||||
|
||||
# Pass through DuckDuckGo-specific parameters
|
||||
ddg_params = ["pretty", "no_redirect", "no_html", "skip_disambig"]
|
||||
for param in ddg_params:
|
||||
if param in optional_params:
|
||||
result_data[param] = optional_params[param]
|
||||
|
||||
return result_data
|
||||
|
||||
def transform_search_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
**kwargs,
|
||||
) -> SearchResponse:
|
||||
"""
|
||||
Transform DuckDuckGo API response to LiteLLM unified SearchResponse format.
|
||||
|
||||
DuckDuckGo → LiteLLM mappings:
|
||||
- RelatedTopics[].Text → SearchResult.title + snippet
|
||||
- RelatedTopics[].FirstURL → SearchResult.url
|
||||
- RelatedTopics[].Text → SearchResult.snippet
|
||||
- No date/last_updated fields in DuckDuckGo response (set to None)
|
||||
|
||||
Args:
|
||||
raw_response: Raw httpx response from DuckDuckGo API
|
||||
logging_obj: Logging object for tracking
|
||||
|
||||
Returns:
|
||||
SearchResponse with standardized format
|
||||
"""
|
||||
response_json = raw_response.json()
|
||||
|
||||
# Transform results to SearchResult objects
|
||||
results = []
|
||||
|
||||
# DuckDuckGo can return results in different fields
|
||||
# Priority: Abstract > Answer > RelatedTopics
|
||||
|
||||
# Check if there's an Abstract with URL
|
||||
if response_json.get("AbstractURL") and response_json.get("AbstractText"):
|
||||
abstract_result = SearchResult(
|
||||
title=response_json.get("Heading", ""),
|
||||
url=response_json.get("AbstractURL", ""),
|
||||
snippet=response_json.get("AbstractText", ""),
|
||||
date=None,
|
||||
last_updated=None,
|
||||
)
|
||||
results.append(abstract_result)
|
||||
|
||||
# Process RelatedTopics
|
||||
related_topics = response_json.get("RelatedTopics", [])
|
||||
for topic in related_topics:
|
||||
# RelatedTopics can contain nested topics or direct results
|
||||
if isinstance(topic, dict):
|
||||
# Check if it's a direct result
|
||||
if "FirstURL" in topic and "Text" in topic:
|
||||
# Extract title and snippet from Text
|
||||
# Text format is usually "Title - Snippet"
|
||||
text = topic.get("Text", "")
|
||||
url = topic.get("FirstURL", "")
|
||||
|
||||
# Try to split title and snippet
|
||||
if " - " in text:
|
||||
parts = text.split(" - ", 1)
|
||||
title = parts[0]
|
||||
snippet = parts[1] if len(parts) > 1 else text
|
||||
else:
|
||||
title = text[:50] + "..." if len(text) > 50 else text
|
||||
snippet = text
|
||||
|
||||
search_result = SearchResult(
|
||||
title=title,
|
||||
url=url,
|
||||
snippet=snippet,
|
||||
date=None,
|
||||
last_updated=None,
|
||||
)
|
||||
results.append(search_result)
|
||||
|
||||
# Check if it contains nested topics
|
||||
elif "Topics" in topic:
|
||||
nested_topics = topic.get("Topics", [])
|
||||
for nested_topic in nested_topics:
|
||||
if "FirstURL" in nested_topic and "Text" in nested_topic:
|
||||
text = nested_topic.get("Text", "")
|
||||
url = nested_topic.get("FirstURL", "")
|
||||
|
||||
# Try to split title and snippet
|
||||
if " - " in text:
|
||||
parts = text.split(" - ", 1)
|
||||
title = parts[0]
|
||||
snippet = parts[1] if len(parts) > 1 else text
|
||||
else:
|
||||
title = text[:50] + "..." if len(text) > 50 else text
|
||||
snippet = text
|
||||
|
||||
search_result = SearchResult(
|
||||
title=title,
|
||||
url=url,
|
||||
snippet=snippet,
|
||||
date=None,
|
||||
last_updated=None,
|
||||
)
|
||||
results.append(search_result)
|
||||
|
||||
# Apply max_results filtering if provided in kwargs
|
||||
max_results = kwargs.get("max_results")
|
||||
if max_results is not None and isinstance(max_results, int):
|
||||
results = results[:max_results]
|
||||
|
||||
return SearchResponse(
|
||||
results=results,
|
||||
object="search",
|
||||
)
|
||||
@ -3197,7 +3197,7 @@ class SearchProviders(str, Enum):
|
||||
FIRECRAWL = "firecrawl"
|
||||
SEARXNG = "searxng"
|
||||
LINKUP = "linkup"
|
||||
|
||||
DUCKDUCKGO = "duckduckgo"
|
||||
|
||||
# Create a set of all search provider values for quick lookup
|
||||
SearchProvidersSet = {provider.value for provider in SearchProviders}
|
||||
|
||||
@ -8771,6 +8771,7 @@ class ProviderConfigManager:
|
||||
"""
|
||||
from litellm.llms.brave.search.transformation import BraveSearchConfig
|
||||
from litellm.llms.dataforseo.search.transformation import DataForSEOSearchConfig
|
||||
from litellm.llms.duckduckgo.search.transformation import DuckDuckGoSearchConfig
|
||||
from litellm.llms.exa_ai.search.transformation import ExaAISearchConfig
|
||||
from litellm.llms.firecrawl.search.transformation import FirecrawlSearchConfig
|
||||
from litellm.llms.google_pse.search.transformation import GooglePSESearchConfig
|
||||
@ -8793,6 +8794,7 @@ class ProviderConfigManager:
|
||||
SearchProviders.FIRECRAWL: FirecrawlSearchConfig,
|
||||
SearchProviders.SEARXNG: SearXNGSearchConfig,
|
||||
SearchProviders.LINKUP: LinkupSearchConfig,
|
||||
SearchProviders.DUCKDUCKGO: DuckDuckGoSearchConfig,
|
||||
}
|
||||
config_class = PROVIDER_TO_CONFIG_MAP.get(provider, None)
|
||||
if config_class is None:
|
||||
|
||||
259
tests/search_tests/test_duckduckgo_search.py
Normal file
259
tests/search_tests/test_duckduckgo_search.py
Normal file
@ -0,0 +1,259 @@
|
||||
"""
|
||||
Tests for DuckDuckGo Search API integration.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, patch, MagicMock
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../..")
|
||||
)
|
||||
|
||||
import litellm
|
||||
from tests.search_tests.base_search_unit_tests import BaseSearchTest
|
||||
|
||||
|
||||
class TestDuckDuckGoSearch(BaseSearchTest):
|
||||
"""
|
||||
Tests for DuckDuckGo Search functionality.
|
||||
"""
|
||||
|
||||
def get_search_provider(self) -> str:
|
||||
"""
|
||||
Return search_provider for DuckDuckGo Search.
|
||||
"""
|
||||
return "duckduckgo"
|
||||
|
||||
|
||||
class TestDuckDuckGoSearchMocked:
|
||||
"""
|
||||
Tests for DuckDuckGo Search functionality with mocked network responses.
|
||||
"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duckduckgo_search_request_payload(self):
|
||||
"""
|
||||
Test that validates the DuckDuckGo search request payload structure without making real API calls.
|
||||
"""
|
||||
# Create a mock response matching DuckDuckGo API format
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"Abstract": "",
|
||||
"AbstractSource": "Wikipedia",
|
||||
"AbstractText": "Python is a high-level programming language.",
|
||||
"AbstractURL": "https://en.wikipedia.org/wiki/Python_(programming_language)",
|
||||
"Answer": "",
|
||||
"AnswerType": "",
|
||||
"Definition": "",
|
||||
"DefinitionSource": "",
|
||||
"DefinitionURL": "",
|
||||
"Entity": "",
|
||||
"Heading": "Python (programming language)",
|
||||
"Image": "",
|
||||
"ImageHeight": 0,
|
||||
"ImageIsLogo": 0,
|
||||
"ImageWidth": 0,
|
||||
"Infobox": "",
|
||||
"Redirect": "",
|
||||
"RelatedTopics": [
|
||||
{
|
||||
"FirstURL": "https://duckduckgo.com/Python_programming",
|
||||
"Icon": {
|
||||
"Height": "",
|
||||
"URL": "/i/python.png",
|
||||
"Width": ""
|
||||
},
|
||||
"Result": "<a href=\"https://duckduckgo.com/Python_programming\">Python Programming</a> A general-purpose programming language.",
|
||||
"Text": "Python Programming - A general-purpose programming language."
|
||||
},
|
||||
{
|
||||
"FirstURL": "https://duckduckgo.com/Python_packages",
|
||||
"Icon": {
|
||||
"Height": "",
|
||||
"URL": "",
|
||||
"Width": ""
|
||||
},
|
||||
"Result": "<a href=\"https://duckduckgo.com/Python_packages\">Python Packages</a> Package management in Python.",
|
||||
"Text": "Python Packages - Package management in Python."
|
||||
}
|
||||
],
|
||||
"Results": [],
|
||||
"Type": "A",
|
||||
"meta": {
|
||||
"attribution": None,
|
||||
"blockgroup": None,
|
||||
"created_date": None,
|
||||
"description": "Wikipedia",
|
||||
"designer": None,
|
||||
"dev_date": None,
|
||||
"dev_milestone": "live",
|
||||
"developer": [
|
||||
{
|
||||
"name": "DDG Team",
|
||||
"type": "ddg",
|
||||
"url": "http://www.duckduckhack.com"
|
||||
}
|
||||
],
|
||||
"example_query": "python programming",
|
||||
"id": "wikipedia_fathead",
|
||||
"is_stackexchange": None,
|
||||
"js_callback_name": "wikipedia",
|
||||
"live_date": None,
|
||||
"maintainer": {
|
||||
"github": "duckduckgo"
|
||||
},
|
||||
"name": "Wikipedia",
|
||||
"perl_module": "DDG::Fathead::Wikipedia",
|
||||
"producer": None,
|
||||
"production_state": "online",
|
||||
"repo": "fathead",
|
||||
"signal_from": "wikipedia_fathead",
|
||||
"src_domain": "en.wikipedia.org",
|
||||
"src_id": 1,
|
||||
"src_name": "Wikipedia",
|
||||
"src_options": {
|
||||
"directory": "",
|
||||
"is_fanon": 0,
|
||||
"is_mediawiki": 1,
|
||||
"is_wikipedia": 1,
|
||||
"language": "en",
|
||||
"min_abstract_length": "20",
|
||||
"skip_abstract": 0,
|
||||
"skip_abstract_paren": 0,
|
||||
"skip_end": "0",
|
||||
"skip_icon": 0,
|
||||
"skip_image_name": 0,
|
||||
"skip_qr": "",
|
||||
"source_skip": "",
|
||||
"src_info": ""
|
||||
},
|
||||
"src_url": None,
|
||||
"status": "live",
|
||||
"tab": "About",
|
||||
"topic": [
|
||||
"productivity"
|
||||
],
|
||||
"unsafe": 0
|
||||
}
|
||||
}
|
||||
|
||||
# Mock the httpx AsyncClient get method (DuckDuckGo uses GET)
|
||||
with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get", new_callable=AsyncMock) as mock_get:
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
# Make the search call
|
||||
response = await litellm.asearch(
|
||||
query="python programming",
|
||||
search_provider="duckduckgo",
|
||||
max_results=5
|
||||
)
|
||||
|
||||
# Verify the get method was called once
|
||||
assert mock_get.call_count == 1
|
||||
|
||||
# Get the actual call arguments
|
||||
call_args = mock_get.call_args
|
||||
|
||||
# Verify URL contains the query
|
||||
url = call_args.kwargs["url"]
|
||||
assert "api.duckduckgo.com" in url
|
||||
assert "q=python" in url or "q=python%20programming" in url
|
||||
assert "format=json" in url
|
||||
|
||||
# Verify response structure
|
||||
assert hasattr(response, "results")
|
||||
assert hasattr(response, "object")
|
||||
assert response.object == "search"
|
||||
assert len(response.results) > 0
|
||||
|
||||
# Verify first result (Abstract)
|
||||
first_result = response.results[0]
|
||||
assert first_result.title == "Python (programming language)"
|
||||
assert first_result.url == "https://en.wikipedia.org/wiki/Python_(programming_language)"
|
||||
assert "Python is a high-level programming language" in first_result.snippet
|
||||
|
||||
# Verify related topics are included
|
||||
assert len(response.results) >= 2 # Abstract + at least one related topic
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duckduckgo_search_disambiguation(self):
|
||||
"""
|
||||
Test handling of disambiguation results from DuckDuckGo.
|
||||
"""
|
||||
# Create a mock response with disambiguation type
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"Abstract": "",
|
||||
"AbstractSource": "Wikipedia",
|
||||
"AbstractText": "",
|
||||
"AbstractURL": "https://en.wikipedia.org/wiki/India_(disambiguation)",
|
||||
"Answer": "",
|
||||
"AnswerType": "",
|
||||
"Definition": "",
|
||||
"DefinitionSource": "",
|
||||
"DefinitionURL": "",
|
||||
"Entity": "",
|
||||
"Heading": "India",
|
||||
"Image": "",
|
||||
"ImageHeight": 0,
|
||||
"ImageIsLogo": 0,
|
||||
"ImageWidth": 0,
|
||||
"Infobox": "",
|
||||
"Redirect": "",
|
||||
"RelatedTopics": [
|
||||
{
|
||||
"FirstURL": "https://duckduckgo.com/India",
|
||||
"Icon": {
|
||||
"Height": "",
|
||||
"URL": "/i/cef47a13.png",
|
||||
"Width": ""
|
||||
},
|
||||
"Result": "<a href=\"https://duckduckgo.com/India\">India</a> A country in South Asia.",
|
||||
"Text": "India - A country in South Asia."
|
||||
},
|
||||
{
|
||||
"Name": "Related Topics",
|
||||
"Topics": [
|
||||
{
|
||||
"FirstURL": "https://duckduckgo.com/d/Indus",
|
||||
"Icon": {
|
||||
"Height": "",
|
||||
"URL": "",
|
||||
"Width": ""
|
||||
},
|
||||
"Result": "<a href=\"https://duckduckgo.com/d/Indus\">Indus</a> See related meanings for the word 'Indus'.",
|
||||
"Text": "Indus - See related meanings for the word 'Indus'."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"Results": [],
|
||||
"Type": "D",
|
||||
"meta": {}
|
||||
}
|
||||
|
||||
# Mock the httpx AsyncClient get method
|
||||
with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get", new_callable=AsyncMock) as mock_get:
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
# Make the search call
|
||||
response = await litellm.asearch(
|
||||
query="India",
|
||||
search_provider="duckduckgo"
|
||||
)
|
||||
|
||||
# Verify response structure
|
||||
assert hasattr(response, "results")
|
||||
assert hasattr(response, "object")
|
||||
assert response.object == "search"
|
||||
|
||||
# Should have results from both direct topics and nested topics
|
||||
assert len(response.results) >= 2
|
||||
|
||||
# Verify nested topics are processed
|
||||
urls = [result.url for result in response.results]
|
||||
assert any("India" in url for url in urls)
|
||||
assert any("Indus" in url for url in urls)
|
||||
Loading…
Reference in New Issue
Block a user