diff --git a/litellm/llms/azure/search/__init__.py b/litellm/llms/azure/search/__init__.py new file mode 100644 index 00000000000..2414ba2b1e8 --- /dev/null +++ b/litellm/llms/azure/search/__init__.py @@ -0,0 +1,3 @@ +from litellm.llms.azure.search.transformation import BingGroundingSearchConfig + +__all__ = ("BingGroundingSearchConfig",) diff --git a/litellm/llms/azure/search/transformation.py b/litellm/llms/azure/search/transformation.py new file mode 100644 index 00000000000..2caee9b50a0 --- /dev/null +++ b/litellm/llms/azure/search/transformation.py @@ -0,0 +1,353 @@ +""" +Calls the Microsoft Foundry Responses API with the `bing_grounding` or `web_search` +tool to search the web (Grounding with Bing Search). + +Microsoft docs: https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/bing-grounding + +Setup: + 1. Set BING_GROUNDING_PROJECT_ENDPOINT to the Foundry project endpoint, e.g. + https://.services.ai.azure.com/api/projects/ + 2. Set BING_GROUNDING_MODEL to a model deployment in that project (e.g. gpt-4.1); + it runs the grounded search and its tokens are billed on that deployment + 3. Optional: set BING_GROUNDING_CONNECTION_ID to a Grounding with Bing Search + project connection id to use the `bing_grounding` tool; without it the + project's built-in `web_search` tool is used + 4. Auth: pass api_key, or set BING_GROUNDING_TOKEN to an Entra bearer token for + scope https://ai.azure.com/.default, or configure azure-identity + (AZURE_CLIENT_ID / AZURE_CLIENT_SECRET / AZURE_TENANT_ID, managed identity, + or any DefaultAzureCredential source) and the token is minted automatically + +Usage: + response = litellm.search( + query="latest AI developments", + search_provider="bing_grounding", + max_results=5, + ) +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal + +import httpx +from pydantic import BaseModel, ConfigDict, ValidationError + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig, + SearchResponse, + SearchResult, +) +from litellm.secret_managers.main import get_secret_str + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +_DOCS_URL: Final = "https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/bing-grounding" + +PROJECT_ENDPOINT_ENV: Final = "BING_GROUNDING_PROJECT_ENDPOINT" +MODEL_ENV: Final = "BING_GROUNDING_MODEL" +CONNECTION_ID_ENV: Final = "BING_GROUNDING_CONNECTION_ID" +TOKEN_ENV: Final = "BING_GROUNDING_TOKEN" + +ENTRA_SCOPE: Final = "https://ai.azure.com/.default" + +_RESPONSES_PATH: Final = "/openai/v1/responses" +_SNIPPET_FALLBACK_LENGTH: Final = 300 + + +class _Annotation(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + + type: str = "" + url: str | None = None + title: str | None = None + start_index: int | None = None + end_index: int | None = None + + +class _ContentPart(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + + type: str = "" + text: str = "" + annotations: tuple[_Annotation, ...] = () + + +class _OutputItem(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + + type: str = "" + content: tuple[_ContentPart, ...] = () + + +class _ResponsesEnvelope(BaseModel): + """A Foundry Responses API body. `output` is required: a body without it is not a + Responses API response and must not be reported as a successful empty search.""" + + model_config = ConfigDict(extra="ignore", frozen=True) + + output: tuple[_OutputItem, ...] + + +class _ErrorBody(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + + message: str | None = None + + +class _ErrorEnvelope(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + + error: _ErrorBody | None = None + + +def _unwrap_error_detail(error_message: str) -> str: + """ + Surface the human-readable message inside Foundry's error envelope. + + Tool failures nest a second JSON document as a string inside `error.message` + (observed live for `bing_grounding` connection errors), so the unwrap runs twice. + Falls back to the raw body for anything else. + """ + try: + envelope: Final = _ErrorEnvelope.model_validate_json(error_message) + except ValidationError: + return error_message + message: Final = envelope.error.message if envelope.error else None + if message is None: + return error_message + try: + nested: Final = _ErrorBody.model_validate_json(message) + except ValidationError: + return message + return nested.message or message + + +def _snippet(text: str, annotation: _Annotation) -> str: + """ + The text a citation supports, not the citation marker itself. + + A url_citation's start/end indices span the inline marker ("([host](url))"), + which follows the claim it backs, so the snippet is the marker's own line up + to where the marker starts. + """ + start: Final = annotation.start_index + marker_start: Final = start if start is not None and 0 <= start <= len(text) else len(text) + claim: Final = text[:marker_start].rsplit("\n", 1)[-1].strip() + if claim: + return claim[-_SNIPPET_FALLBACK_LENGTH:] + return text[:_SNIPPET_FALLBACK_LENGTH] + + +def _citation_results(envelope: _ResponsesEnvelope) -> tuple[SearchResult, ...]: + """One result per cited URL: first occurrence wins, order preserved as answered.""" + cited: Final = tuple( + SearchResult( + title=annotation.title or "", + url=annotation.url or "", + snippet=_snippet(part.text, annotation), + date=None, + last_updated=None, + ) + for item in envelope.output + if item.type == "message" + for part in item.content + if part.type == "output_text" + for annotation in part.annotations + if annotation.type == "url_citation" and annotation.url + ) + first_by_url: Final = MappingProxyType({result.url: result for result in reversed(cited)}) + return tuple(first_by_url[url] for url in dict.fromkeys(result.url for result in cited)) + + +class _SearchConfiguration(BaseModel): + model_config = ConfigDict(frozen=True) + + project_connection_id: str + count: int | None = None + + +class _BingGroundingParams(BaseModel): + model_config = ConfigDict(frozen=True) + + search_configurations: tuple[_SearchConfiguration, ...] + + +class _BingGroundingTool(BaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["bing_grounding"] = "bing_grounding" + bing_grounding: _BingGroundingParams + + +class _UserLocation(BaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["approximate"] = "approximate" + country: str + + +class _WebSearchTool(BaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["web_search"] = "web_search" + user_location: _UserLocation | None = None + + +class _ResponsesRequest(BaseModel): + model_config = ConfigDict(frozen=True) + + model: str + input: str + tools: tuple[_BingGroundingTool | _WebSearchTool, ...] + + +def _search_tool(optional_params: Mapping[str, object]) -> _BingGroundingTool | _WebSearchTool: + connection_id: Final = get_secret_str(CONNECTION_ID_ENV) + max_results: Final = optional_params.get("max_results") + country: Final = optional_params.get("country") + if connection_id: + configuration: Final = _SearchConfiguration( + project_connection_id=connection_id, + count=max_results if isinstance(max_results, int) else None, + ) + return _BingGroundingTool(bing_grounding=_BingGroundingParams(search_configurations=(configuration,))) + location: Final = _UserLocation(country=country.upper()) if isinstance(country, str) else None + return _WebSearchTool(user_location=location) + + +def _default_entra_token_minter() -> str: + from litellm.secret_managers.get_azure_ad_token_provider import get_azure_ad_token_provider + + return get_azure_ad_token_provider(azure_scope=ENTRA_SCOPE)() + + +class BingGroundingSearchConfig(BaseSearchConfig): + def __init__(self, entra_token_minter: Callable[[], str] | None = None) -> None: + super().__init__() + self._entra_token_minter = entra_token_minter + + @staticmethod + def ui_friendly_name() -> str: + return "Grounding with Bing Search" + + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: BaseSearchConfig.validate_environment signature + api_key: str | None = None, + api_base: str | None = None, + **kwargs: object, # kwargs-ok: BaseSearchConfig.validate_environment signature + ) -> dict[str, str]: # mutable-ok: the http handler passes this straight to httpx as headers + """ + Validate environment and return headers. + + Returns a new dict rather than mutating ``headers``: the http handler calls this + a second time after ``litellm/search/main.py`` already did, so it has to be idempotent. + """ + resolved_token: Final = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=(TOKEN_ENV,), + base_env_var=PROJECT_ENDPOINT_ENV, + default_api_base=None, + ) or self._mint_entra_token(api_base) + return { # mutable-ok: httpx requires a plain dict of headers + **headers, + "Authorization": f"Bearer {resolved_token}", + "Content-Type": "application/json", + } + + def _mint_entra_token(self, caller_api_base: str | None) -> str: + self._assert_trusted_api_base_for_server_credential( + caller_api_base, None, PROJECT_ENDPOINT_ENV, "Azure AD token" + ) + minter: Final = self._entra_token_minter or _default_entra_token_minter + try: + return minter() + except Exception as e: + raise ValueError( + f"Grounding with Bing Search: no credential available. Pass api_key, set {TOKEN_ENV} " + f"to an Entra bearer token, or configure azure-identity (AZURE_CLIENT_ID / " + f"AZURE_CLIENT_SECRET / AZURE_TENANT_ID or any DefaultAzureCredential source) " + f"for scope {ENTRA_SCOPE}. Underlying error: {e}" + ) from e + + def get_complete_url( + self, + api_base: str | None, + optional_params: dict[str, object], # mutable-ok: BaseSearchConfig.get_complete_url signature + data: dict[str, object] | list[dict[str, object]] | None = None, # mutable-ok: base signature + **kwargs: object, # kwargs-ok: BaseSearchConfig.get_complete_url signature + ) -> str: + resolved_base: Final = api_base or get_secret_str(PROJECT_ENDPOINT_ENV) + if not resolved_base: + raise ValueError( + f"{PROJECT_ENDPOINT_ENV} is not set. Set it to your Microsoft Foundry project " + f"endpoint, e.g. https://.services.ai.azure.com/api/projects/." + ) + trimmed: Final = resolved_base.rstrip("/") + if trimmed.endswith(_RESPONSES_PATH): + return trimmed + return f"{trimmed}{_RESPONSES_PATH}" + + def transform_search_request( + self, + query: str | list[str], # mutable-ok: BaseSearchConfig.transform_search_request signature + optional_params: dict[str, object], # mutable-ok: base signature + **kwargs: object, # kwargs-ok: BaseSearchConfig.transform_search_request signature + ) -> dict[str, object]: # mutable-ok: the http handler passes this straight to httpx as the JSON body + """ + Transform Search request to the Foundry Responses API format. + + The unified params map as far as the API allows: + - max_results -> the bing_grounding search configuration's `count` (the built-in + web_search tool has no result-count knob, so it is dropped in that mode) + - country -> web_search's approximate `user_location` (bing_grounding's `market` + wants a full locale like en-US, which a bare country code cannot fill) + - search_domain_filter, max_tokens_per_page -> no API equivalent, dropped + """ + model: Final = get_secret_str(MODEL_ENV) + if not model: + raise ValueError( + f"{MODEL_ENV} is not set. Set it to a model deployment in the Foundry project " + f"that runs the grounded search, e.g. gpt-4.1." + ) + request: Final = _ResponsesRequest( + model=model, + input=" ".join(query) if isinstance(query, list) else query, + tools=(_search_tool(optional_params),), + ) + return request.model_dump(mode="json", exclude_none=True) + + def transform_search_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs: object, # kwargs-ok: BaseSearchConfig.transform_search_response signature + ) -> SearchResponse: + try: + parsed: Final = _ResponsesEnvelope.model_validate_json(raw_response.content) + except ValidationError as e: + raise self.get_error_class( + error_message=f"response does not match the Foundry Responses API schema: {e}", + status_code=raw_response.status_code, + headers=dict(raw_response.headers), # mutable-ok: BaseSearchConfig.get_error_class signature + ) + results: Final = list(_citation_results(parsed)) # mutable-ok: SearchResponse.results is list[SearchResult] + return SearchResponse(results=results, object="search") + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, str], # mutable-ok: BaseSearchConfig.get_error_class signature + ) -> Exception: + detail: Final = _unwrap_error_detail(error_message).rstrip(". ") + return BaseLLMException( + status_code=status_code, + message=f"Grounding with Bing Search: {detail}. See {_DOCS_URL} for details.", + headers=headers, + ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3af7d9e5019..9ba8b846e11 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16890,6 +16890,14 @@ "notes": "Web Search on Amazon Bedrock AgentCore, billed by AWS on the gateway" } }, + "bing_grounding/search": { + "input_cost_per_query": 0.035, + "litellm_provider": "bing_grounding", + "mode": "search", + "metadata": { + "notes": "Grounding with Bing Search (G1 SKU): $35 per 1,000 transactions. Tokens for the Foundry model deployment that runs the grounded search are billed separately on that deployment." + } + }, "tinyfish/search": { "input_cost_per_query": 0.0, "litellm_provider": "tinyfish", diff --git a/litellm/proxy/example_config_yaml/bing_grounding_websearch_config.yaml b/litellm/proxy/example_config_yaml/bing_grounding_websearch_config.yaml new file mode 100644 index 00000000000..5ab18723f7f --- /dev/null +++ b/litellm/proxy/example_config_yaml/bing_grounding_websearch_config.yaml @@ -0,0 +1,40 @@ +# Web search via Microsoft Foundry: Grounding with Bing Search / the built-in +# web_search tool, called through the Foundry Responses API. +# See litellm/llms/azure/search/transformation.py for details. +# +# Required environment variables (the search router forwards only +# search_provider / api_key / api_base from the litellm_params block, so +# provider configuration rides env vars): +# BING_GROUNDING_PROJECT_ENDPOINT: the Foundry project endpoint, e.g. +# https://.services.ai.azure.com/api/projects/ +# BING_GROUNDING_MODEL: a model deployment in that project (e.g. gpt-4.1); +# it runs the grounded search, its tokens are billed on that deployment +# Optional: +# BING_GROUNDING_CONNECTION_ID: a Grounding with Bing Search project +# connection id; set it to use the bing_grounding tool ($35 per 1,000 +# transactions on the G1 SKU). Without it the project's built-in +# web_search tool is used +# BING_GROUNDING_TOKEN: an Entra bearer token for scope +# https://ai.azure.com/.default. Without it (and without api_key below) +# the token is minted via azure-identity (AZURE_CLIENT_ID / +# AZURE_CLIENT_SECRET / AZURE_TENANT_ID, managed identity, or any other +# DefaultAzureCredential source) + +model_list: + - model_name: claude-sonnet + litellm_params: + model: bedrock/us.anthropic.claude-sonnet-5 + aws_region_name: us-east-1 + +search_tools: + - search_tool_name: bing-grounding-search + litellm_params: + search_provider: bing_grounding + # Alternative to BING_GROUNDING_TOKEN / azure-identity: + # api_key: os.environ/BING_GROUNDING_TOKEN + +litellm_settings: + callbacks: ["websearch_interception"] + websearch_interception_params: + enabled_providers: ["bedrock"] + search_tool_name: bing-grounding-search diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 67eae2b4f21..371ec7d3375 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3844,6 +3844,7 @@ class SearchProviders(str, Enum): TINYFISH = "tinyfish" AGENTCORE = "agentcore" NIMBLE = "nimble" + BING_GROUNDING = "bing_grounding" # Create a set of all search provider values for quick lookup diff --git a/litellm/utils.py b/litellm/utils.py index e5ce7157e77..006717df187 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9111,6 +9111,7 @@ class ProviderConfigManager: from litellm.llms.apiserpent.search.transformation import ( APISerpentSearchConfig, ) + from litellm.llms.azure.search.transformation import BingGroundingSearchConfig from litellm.llms.bedrock.search.transformation import AgentCoreSearchConfig from litellm.llms.brave.search.transformation import BraveSearchConfig from litellm.llms.dataforseo.search.transformation import DataForSEOSearchConfig @@ -9152,6 +9153,7 @@ class ProviderConfigManager: SearchProviders.TINYFISH: TinyfishSearchConfig, SearchProviders.AGENTCORE: AgentCoreSearchConfig, SearchProviders.NIMBLE: NimbleSearchConfig, + SearchProviders.BING_GROUNDING: BingGroundingSearchConfig, } config_class: Final = PROVIDER_TO_CONFIG_MAP.get(provider, None) if config_class is None: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3af7d9e5019..9ba8b846e11 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -16890,6 +16890,14 @@ "notes": "Web Search on Amazon Bedrock AgentCore, billed by AWS on the gateway" } }, + "bing_grounding/search": { + "input_cost_per_query": 0.035, + "litellm_provider": "bing_grounding", + "mode": "search", + "metadata": { + "notes": "Grounding with Bing Search (G1 SKU): $35 per 1,000 transactions. Tokens for the Foundry model deployment that runs the grounded search are billed separately on that deployment." + } + }, "tinyfish/search": { "input_cost_per_query": 0.0, "litellm_provider": "tinyfish", diff --git a/tests/search_tests/test_bing_grounding_search.py b/tests/search_tests/test_bing_grounding_search.py new file mode 100644 index 00000000000..00e5e382eef --- /dev/null +++ b/tests/search_tests/test_bing_grounding_search.py @@ -0,0 +1,187 @@ +""" +Tests for the Grounding with Bing Search (Microsoft Foundry) integration. +""" + +import json +from unittest.mock import AsyncMock, Mock, patch + +import pytest + +import litellm +from tests.search_tests.base_search_unit_tests import BaseSearchTest + +PROJECT_ENDPOINT = "https://acct.services.ai.azure.com/api/projects/proj" + +_ANSWER_TEXT = ( + "LiteLLM is an open source LLM gateway ([github.com](https://github.com/BerriAI/litellm))\n" + "The docs live on docs.litellm.ai ([docs.litellm.ai](https://docs.litellm.ai/))" +) + + +def _annotation(marker: str, url: str, title: str) -> dict: + start = _ANSWER_TEXT.index(marker) + return { + "type": "url_citation", + "url": url, + "title": title, + "start_index": start, + "end_index": start + len(marker), + } + + +MOCK_BING_GROUNDING_RESPONSE = { + "id": "resp_mock", + "object": "response", + "status": "completed", + "model": "gpt-4.1", + "output": [ + {"type": "web_search_call", "status": "completed"}, + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": _ANSWER_TEXT, + "annotations": [ + _annotation( + "([github.com](https://github.com/BerriAI/litellm))", + "https://github.com/BerriAI/litellm", + "BerriAI/litellm - GitHub", + ), + _annotation( + "([docs.litellm.ai](https://docs.litellm.ai/))", + "https://docs.litellm.ai/", + "LiteLLM Docs", + ), + ], + } + ], + }, + ], + "usage": {"input_tokens": 100, "output_tokens": 50}, +} + + +def _mock_response(): + response = Mock() + response.status_code = 200 + response.headers = {} + response.content = json.dumps(MOCK_BING_GROUNDING_RESPONSE).encode() + return response + + +@pytest.mark.skip(reason="Local only tested search providers") +class TestBingGroundingSearch(BaseSearchTest): + """ + E2E tests for Grounding with Bing Search that make real API calls. + Inherits from BaseSearchTest to run standard search tests. + """ + + def get_search_provider(self) -> str: + return "bing_grounding" + + +class TestBingGroundingSearchTransformation: + """ + Full-stack tests through `litellm.search` / `litellm.asearch` with the HTTP layer mocked. + Transformation details are unit-tested in tests/test_litellm/llms/azure/search/. + """ + + @pytest.fixture(autouse=True) + def _server_env(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("BING_GROUNDING_PROJECT_ENDPOINT", PROJECT_ENDPOINT) + monkeypatch.setenv("BING_GROUNDING_MODEL", "gpt-4.1") + monkeypatch.setenv("BING_GROUNDING_TOKEN", "test-entra-token") + monkeypatch.delenv("BING_GROUNDING_CONNECTION_ID", raising=False) + + def test_bing_grounding_search_request_and_response(self): + with patch( # test-quality-ok: litellm.search has no client injection seam + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", + return_value=_mock_response(), + ) as mock_post: + response = litellm.search( + query="what is litellm", + search_provider="bing_grounding", + max_results=5, + country="us", + ) + + assert mock_post.called + call_kwargs = mock_post.call_args.kwargs + assert call_kwargs["url"] == f"{PROJECT_ENDPOINT}/openai/v1/responses" + assert call_kwargs["headers"]["Authorization"] == "Bearer test-entra-token" + + request_body = call_kwargs["json"] + assert request_body["model"] == "gpt-4.1" + assert request_body["input"] == "what is litellm" + assert request_body["tools"] == [ + {"type": "web_search", "user_location": {"type": "approximate", "country": "US"}} + ] + + assert response.object == "search" + assert len(response.results) == 2 + assert response.results[0].url == "https://github.com/BerriAI/litellm" + assert response.results[0].title == "BerriAI/litellm - GitHub" + assert response.results[0].snippet == "LiteLLM is an open source LLM gateway" + assert response.results[1].url == "https://docs.litellm.ai/" + assert response.results[1].snippet == "The docs live on docs.litellm.ai" + + def test_connection_mode_sends_the_bing_grounding_tool(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv( + "BING_GROUNDING_CONNECTION_ID", + "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.CognitiveServices" + "/accounts/acct/projects/proj/connections/bing-conn", + ) + with patch( # test-quality-ok: litellm.search has no client injection seam + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", + return_value=_mock_response(), + ) as mock_post: + litellm.search( + query="what is litellm", + search_provider="bing_grounding", + max_results=3, + ) + + request_body = mock_post.call_args.kwargs["json"] + assert request_body["tools"] == [ + { + "type": "bing_grounding", + "bing_grounding": { + "search_configurations": [ + { + "project_connection_id": ( + "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.CognitiveServices" + "/accounts/acct/projects/proj/connections/bing-conn" + ), + "count": 3, + } + ] + }, + } + ] + + @pytest.mark.asyncio + async def test_bing_grounding_asearch(self): + with patch( # test-quality-ok: litellm.asearch has no client injection seam + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new=AsyncMock(return_value=_mock_response()), + ) as mock_post: + response = await litellm.asearch( + query="what is litellm", + search_provider="bing_grounding", + ) + + assert mock_post.call_args.kwargs["json"]["tools"] == [{"type": "web_search"}] + assert len(response.results) == 2 + + def test_bing_grounding_search_tracks_cost(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + with patch( # test-quality-ok: litellm.search has no client injection seam + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", + return_value=_mock_response(), + ): + response = litellm.search(query="pricing check", search_provider="bing_grounding") + + assert response._hidden_params["response_cost"] == pytest.approx(0.035) diff --git a/tests/test_litellm/llms/azure/search/foundry_responses_web_search_fixture.json b/tests/test_litellm/llms/azure/search/foundry_responses_web_search_fixture.json new file mode 100644 index 00000000000..2dba691d038 --- /dev/null +++ b/tests/test_litellm/llms/azure/search/foundry_responses_web_search_fixture.json @@ -0,0 +1,78 @@ +{ + "id": "resp_04bcf916b629f9be006a8c7c053e9881959a4efe7118cb3a34", + "object": "response", + "status": "completed", + "model": "gpt-4.1", + "output": [ + { + "type": "web_search_call", + "id": "ws_04bcf916b629f9be006a8c7c080e108195b3366ab048231129", + "response_id": "resp_04bcf916b629f9be006a8c7c053e9881959a4efe7118cb3a34", + "status": "completed", + "action": { + "type": "search", + "queries": [ + "latest LiteLLM release GitHub LiteLLM", + "LiteLLM github releases", + "LiteLLM repo LiteLLM" + ], + "query": "latest LiteLLM release GitHub" + } + }, + { + "type": "message", + "id": "msg_04bcf916b629f9be006a8c7c0c7b448195979bfafb3b28f1ba", + "response_id": "resp_04bcf916b629f9be006a8c7c053e9881959a4efe7118cb3a34", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "You asked: “What is the latest LiteLLM release on GitHub? Answer with the version number.”\n\nI verified the most recent release information from the official GitHub repository for LiteLLM (BerriAI/litellm) using multiple sources, including GitHub itself and secondary trackers:\n\n- On the GitHub **Releases** page for BerriAI/litellm, the topmost entry is **v1.99.0‑rc.1**, marked as a pre‑release, published “yesterday” (relative to today, August 24, 2026) ([github.com](https://github.com/BerriAI/litellm/releases)). This indicates that version **v1.99.0‑rc.1** is the most recent tag available.\n- An external release‑tracking site (ReleaseAlert) confirms: **Latest version: v1.99.0‑rc.1**, last published August 22, 2026 ([releasealert.dev](https://releasealert.dev/github/BerriAI/litellm)).\n- The GitHub API (via `releases/latest`) currently points to **v1.98.0** as the latest **stable** release, with published date August 23, 2026 ([api.github.com](https://api.github.com/repos/BerriAI/litellm/releases/latest)).\n\nTo summarize:\n\n- The absolute **latest** release tag on GitHub is **v1.99.0‑rc.1** (release candidate), published recently (August 22, 2026) ([github.com](https://github.com/BerriAI/litellm/releases)).\n- The most recent **stable** release is **v1.98.0**, published August 23, 2026 ([api.github.com](https://api.github.com/repos/BerriAI/litellm/releases/latest)).\n\nSince you asked for the “latest LiteLLM release on GitHub,” without specifying stable vs. pre‑release, the correct answer is:\n\n**v1.99.0‑rc.1**\n\nLet me know if you'd like details on what's new in that release, or if you'd prefer the latest stable version.", + "annotations": [ + { + "type": "url_citation", + "url": "https://github.com/BerriAI/litellm/releases", + "start_index": 456, + "end_index": 515, + "title": "Releases · BerriAI/litellm - GitHub" + }, + { + "type": "url_citation", + "url": "https://releasealert.dev/github/BerriAI/litellm", + "start_index": 722, + "end_index": 791, + "title": "BerriAI/litellm on GitHub | Release Alert" + }, + { + "type": "url_citation", + "url": "https://api.github.com/repos/BerriAI/litellm/releases/latest", + "start_index": 936, + "end_index": 1016, + "title": "api.github.com" + }, + { + "type": "url_citation", + "url": "https://github.com/BerriAI/litellm/releases", + "start_index": 1160, + "end_index": 1219, + "title": "Releases · BerriAI/litellm - GitHub" + }, + { + "type": "url_citation", + "url": "https://api.github.com/repos/BerriAI/litellm/releases/latest", + "start_index": 1300, + "end_index": 1380, + "title": "api.github.com" + } + ], + "logprobs": [] + } + ], + "status": "completed" + } + ], + "usage": { + "input_tokens": 15195, + "output_tokens": 467 + } +} diff --git a/tests/test_litellm/llms/azure/search/test_bing_grounding_search_transformation.py b/tests/test_litellm/llms/azure/search/test_bing_grounding_search_transformation.py new file mode 100644 index 00000000000..25dfa5fbe2b --- /dev/null +++ b/tests/test_litellm/llms/azure/search/test_bing_grounding_search_transformation.py @@ -0,0 +1,311 @@ +import json +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from litellm.llms.azure.search.transformation import BingGroundingSearchConfig + +REAL_FIXTURE = json.loads((Path(__file__).parent / "foundry_responses_web_search_fixture.json").read_text()) + +RESPONSES_URL = "https://acct.services.ai.azure.com/api/projects/proj/openai/v1/responses" + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch: pytest.MonkeyPatch): + for var in ( + "BING_GROUNDING_PROJECT_ENDPOINT", + "BING_GROUNDING_MODEL", + "BING_GROUNDING_CONNECTION_ID", + "BING_GROUNDING_TOKEN", + ): + monkeypatch.delenv(var, raising=False) + + +def _config(entra_token_minter=None) -> BingGroundingSearchConfig: + return BingGroundingSearchConfig(entra_token_minter=entra_token_minter) + + +def _resp(payload, status_code: int = 200): + r = Mock() + r.status_code = status_code + r.headers = {} + r.content = (payload if isinstance(payload, str) else json.dumps(payload)).encode() + return r + + +def _message_response(text: str, annotations: list) -> dict: + return { + "output": [ + {"type": "web_search_call", "status": "completed"}, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": text, "annotations": annotations}], + }, + ] + } + + +def _citation(url: str, title: str, start: int, end: int) -> dict: + return {"type": "url_citation", "url": url, "title": title, "start_index": start, "end_index": end} + + +def test_ui_friendly_name(): + assert _config().ui_friendly_name() == "Grounding with Bing Search" + + +def test_validate_environment_with_explicit_key(): + headers = _config().validate_environment({}, api_key="explicit-token") + assert headers["Authorization"] == "Bearer explicit-token" + assert headers["Content-Type"] == "application/json" + + +def test_validate_environment_reads_env_token(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("BING_GROUNDING_TOKEN", "env-token") + assert _config().validate_environment({})["Authorization"] == "Bearer env-token" + + +def test_validate_environment_falls_back_to_entra_minter(): + headers = _config(entra_token_minter=lambda: "entra-token").validate_environment({}) + assert headers["Authorization"] == "Bearer entra-token" + + +def test_validate_environment_api_key_beats_env_token(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("BING_GROUNDING_TOKEN", "env-token") + minter = Mock(return_value="entra-token") + headers = _config(entra_token_minter=minter).validate_environment({}, api_key="explicit-token") + assert headers["Authorization"] == "Bearer explicit-token" + minter.assert_not_called() + + +def test_validate_environment_env_token_beats_entra_minter(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("BING_GROUNDING_TOKEN", "env-token") + minter = Mock(return_value="entra-token") + assert _config(entra_token_minter=minter).validate_environment({})["Authorization"] == "Bearer env-token" + minter.assert_not_called() + + +def test_validate_environment_refuses_entra_token_for_caller_api_base(): + minter = Mock(return_value="entra-token") + with pytest.raises(ValueError, match="Refusing to send the server-configured"): + _config(entra_token_minter=minter).validate_environment({}, api_base="https://attacker.example.com") + minter.assert_not_called() + + +def test_validate_environment_entra_minter_failure_names_the_options(): + def failing_minter() -> str: + raise RuntimeError("no az login") + + with pytest.raises(ValueError, match="no credential available") as excinfo: + _config(entra_token_minter=failing_minter).validate_environment({}) + message = str(excinfo.value) + assert "BING_GROUNDING_TOKEN" in message + assert "https://ai.azure.com/.default" in message + assert "no az login" in message + + +def test_validate_environment_does_not_mutate_and_is_idempotent(): + config = _config() + caller_headers = {"X-Custom": "keep-me"} + + once = config.validate_environment(caller_headers, api_key="k") + twice = config.validate_environment(once, api_key="k") + + assert caller_headers == {"X-Custom": "keep-me"} + assert once == twice + assert once["X-Custom"] == "keep-me" + + +def test_get_complete_url_from_api_base(): + url = _config().get_complete_url("https://acct.services.ai.azure.com/api/projects/proj", {}) + assert url == RESPONSES_URL + + +def test_get_complete_url_reads_env_endpoint(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("BING_GROUNDING_PROJECT_ENDPOINT", "https://acct.services.ai.azure.com/api/projects/proj/") + assert _config().get_complete_url(None, {}) == RESPONSES_URL + + +def test_get_complete_url_missing_endpoint_raises(): + with pytest.raises(ValueError, match="BING_GROUNDING_PROJECT_ENDPOINT"): + _config().get_complete_url(None, {}) + + +@pytest.mark.parametrize( + "api_base", + [ + "https://acct.services.ai.azure.com/api/projects/proj", + "https://acct.services.ai.azure.com/api/projects/proj/", + "https://acct.services.ai.azure.com/api/projects/proj/openai/v1/responses", + "https://acct.services.ai.azure.com/api/projects/proj/openai/v1/responses/", + ], +) +def test_get_complete_url_appends_responses_path_exactly_once(api_base: str): + assert _config().get_complete_url(api_base, {}) == RESPONSES_URL + + +def test_transform_search_request_missing_model_raises(): + with pytest.raises(ValueError, match="BING_GROUNDING_MODEL"): + _config().transform_search_request("q", {}) + + +def test_transform_search_request_web_search_mode_exact_body(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("BING_GROUNDING_MODEL", "gpt-4.1") + body = _config().transform_search_request("latest AI developments", {"max_results": 5}) + assert body == { + "model": "gpt-4.1", + "input": "latest AI developments", + "tools": [{"type": "web_search"}], + } + + +def test_transform_search_request_web_search_mode_maps_country(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("BING_GROUNDING_MODEL", "gpt-4.1") + body = _config().transform_search_request("q", {"country": "us"}) + assert body["tools"] == [{"type": "web_search", "user_location": {"type": "approximate", "country": "US"}}] + + +def test_transform_search_request_connection_mode_exact_body(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("BING_GROUNDING_MODEL", "gpt-4.1") + monkeypatch.setenv("BING_GROUNDING_CONNECTION_ID", "conn-id") + body = _config().transform_search_request("q", {"max_results": 5}) + assert body == { + "model": "gpt-4.1", + "input": "q", + "tools": [ + { + "type": "bing_grounding", + "bing_grounding": {"search_configurations": [{"project_connection_id": "conn-id", "count": 5}]}, + } + ], + } + + +def test_transform_search_request_connection_mode_omits_count_without_max_results( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setenv("BING_GROUNDING_MODEL", "gpt-4.1") + monkeypatch.setenv("BING_GROUNDING_CONNECTION_ID", "conn-id") + body = _config().transform_search_request("q", {}) + assert body["tools"][0]["bing_grounding"]["search_configurations"] == [{"project_connection_id": "conn-id"}] + + +def test_transform_search_request_joins_list_query(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("BING_GROUNDING_MODEL", "gpt-4.1") + assert _config().transform_search_request(["foo", "bar"], {})["input"] == "foo bar" + + +def test_transform_search_response_real_fixture_dedupes_and_preserves_order(): + resp = _config().transform_search_response(_resp(REAL_FIXTURE), logging_obj=Mock()) + + assert resp.object == "search" + assert [r.url for r in resp.results] == [ + "https://github.com/BerriAI/litellm/releases", + "https://releasealert.dev/github/BerriAI/litellm", + "https://api.github.com/repos/BerriAI/litellm/releases/latest", + ] + assert resp.results[0].title == "Releases · BerriAI/litellm - GitHub" + assert resp.results[1].title == "BerriAI/litellm on GitHub | Release Alert" + + +def test_transform_search_response_real_fixture_snippets_are_the_cited_claims(): + resp = _config().transform_search_response(_resp(REAL_FIXTURE), logging_obj=Mock()) + + assert resp.results[0].snippet.startswith("- On the GitHub **Releases** page for BerriAI/litellm") + assert resp.results[1].snippet.startswith("- An external release") + assert resp.results[2].snippet.startswith("- The GitHub API (via `releases/latest`)") + for result in resp.results: + assert "url_citation" not in result.snippet + assert not result.snippet.startswith("([") + + +def test_transform_search_response_snippet_falls_back_to_text_head_for_leading_citation(): + text = "([example.com](https://example.com)) trailing prose" + payload = _message_response(text, [_citation("https://example.com", "Example", 0, 36)]) + resp = _config().transform_search_response(_resp(payload), logging_obj=Mock()) + assert resp.results[0].snippet == text + + +def test_transform_search_response_snippet_without_indices_uses_last_line(): + payload = _message_response( + "first line\nthe claim on the last line", + [{"type": "url_citation", "url": "https://example.com", "title": "Example"}], + ) + resp = _config().transform_search_response(_resp(payload), logging_obj=Mock()) + assert resp.results[0].snippet == "the claim on the last line" + + +def test_transform_search_response_ignores_non_citation_annotations(): + payload = _message_response("text", [{"type": "file_citation", "url": "https://example.com"}]) + assert _config().transform_search_response(_resp(payload), logging_obj=Mock()).results == [] + + +def test_transform_search_response_ignores_citation_without_url(): + payload = _message_response("text", [{"type": "url_citation", "title": "no url"}]) + assert _config().transform_search_response(_resp(payload), logging_obj=Mock()).results == [] + + +def test_transform_search_response_no_message_output(): + payload = {"output": [{"type": "web_search_call", "status": "completed"}]} + assert _config().transform_search_response(_resp(payload), logging_obj=Mock()).results == [] + + +@pytest.mark.parametrize( + "body", + [ + "502 Bad Gateway", + '{"output": "garbage"}', + '{"output": null}', + "{}", + ], +) +def test_transform_search_response_malformed_body_raises_instead_of_reporting_empty(body: str): + with pytest.raises(Exception, match="Grounding with Bing Search"): + _config().transform_search_response(_resp(body, status_code=502), logging_obj=Mock()) + + +def test_get_error_class_attributes_the_provider(): + error = _config().get_error_class(error_message="quota exceeded", status_code=429, headers={}) + assert error.status_code == 429 + assert "Grounding with Bing Search: quota exceeded" in str(error) + assert "learn.microsoft.com" in str(error) + + +def test_get_error_class_unwraps_the_nested_tool_error(): + nested_tool_error = json.dumps( + { + "error": "Tool_User_Error", + "message": ( + "The specified connection ID 'conn-id' in tool config input was not found " + "in the project or account connections." + ), + "code": "invalid_tool_input", + "tool": "bing_grounding", + } + ) + live_400_shape = json.dumps( + { + "error": { + "message": nested_tool_error, + "type": "invalid_request_error", + "param": None, + "code": "tool_user_error", + } + } + ) + error = _config().get_error_class(error_message=live_400_shape, status_code=400, headers={}) + assert ( + "Grounding with Bing Search: The specified connection ID 'conn-id' in tool config input " + "was not found in the project or account connections" in str(error) + ) + assert "Tool_User_Error" not in str(error) + + +def test_get_error_class_unwraps_a_plain_error_envelope(): + error = _config().get_error_class( + error_message='{"error":{"message":"The api key is invalid.","code":"401"}}', + status_code=401, + headers={}, + ) + assert "Grounding with Bing Search: The api key is invalid" in str(error) diff --git a/tests/test_litellm/llms/base_llm/search/test_base_search_transformation.py b/tests/test_litellm/llms/base_llm/search/test_base_search_transformation.py index e4402bbec49..e6aad7688d1 100644 --- a/tests/test_litellm/llms/base_llm/search/test_base_search_transformation.py +++ b/tests/test_litellm/llms/base_llm/search/test_base_search_transformation.py @@ -16,6 +16,7 @@ import pytest import litellm from litellm.llms.apiserpent.search.transformation import APISerpentSearchConfig +from litellm.llms.azure.search.transformation import BingGroundingSearchConfig from litellm.llms.base_llm.search.transformation import ( BaseSearchConfig, _is_trusted_search_api_base, @@ -59,6 +60,7 @@ _BASE_ENV_VARS = ( "TINYFISH_API_BASE", "CRW_API_BASE", "NIMBLE_API_BASE", + "BING_GROUNDING_PROJECT_ENDPOINT", ) @@ -99,6 +101,7 @@ PROVIDERS: Tuple[ProviderSpec, ...] = ( (TinyfishSearchConfig, {"TINYFISH_API_KEY": "srv"}, "caller-key", {}), (FastCRWSearchConfig, {"CRW_API_KEY": "srv"}, "caller-key", {}), (NimbleSearchConfig, {"NIMBLE_API_KEY": "srv"}, "caller-key", {}), + (BingGroundingSearchConfig, {"BING_GROUNDING_TOKEN": "srv"}, "caller-key", {}), ) _IDS = tuple(spec[0].__name__ for spec in PROVIDERS) diff --git a/ui/litellm-dashboard/public/assets/logos/bing.png b/ui/litellm-dashboard/public/assets/logos/bing.png new file mode 100644 index 00000000000..ab1f4359281 Binary files /dev/null and b/ui/litellm-dashboard/public/assets/logos/bing.png differ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.tsx b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.tsx index 57148ee3a21..9c79e976e2b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.tsx @@ -27,6 +27,7 @@ import { useZodForm } from "@/lib/forms/useZodForm"; import SearchConnectionTest from "./SearchConnectionTest"; import { buildSearchToolPayload } from "./searchToolPayload"; import { AvailableSearchProvider, SearchTool } from "./types"; +import bingLogo from "../../../../../public/assets/logos/bing.png"; import dataforseoLogo from "../../../../../public/assets/logos/dataforseo.png"; import exaAiLogo from "../../../../../public/assets/logos/exa_ai.png"; import googlePseLogo from "../../../../../public/assets/logos/google_pse.png"; @@ -44,6 +45,7 @@ const searchProviderLogoMap: Record = { google_pse: googlePseLogo.src, dataforseo: dataforseoLogo.src, nimble: nimbleLogo.src, + bing_grounding: bingLogo.src, }; interface SearchProviderLabelProps {