diff --git a/strix/tools/web_search/tool.py b/strix/tools/web_search/tool.py index daee9874f..bc477d630 100644 --- a/strix/tools/web_search/tool.py +++ b/strix/tools/web_search/tool.py @@ -139,16 +139,38 @@ _PARALLEL_MAX_CHARS = 12000 def _parallel_result_content(results: list[tuple[str, str]]) -> str: header_chars = sum(len(header) for header, _ in results) + 2 * (len(results) - 1) remaining = max(0, _PARALLEL_MAX_CHARS - header_chars) + excerpts = [excerpt.rstrip() for _, excerpt in results] + allocations = [0] * len(results) + nonempty = [index for index, excerpt in enumerate(excerpts) if excerpt] + full_cost = sum(len(excerpts[index]) + 2 for index in nonempty) + + if full_cost <= remaining: + for index in nonempty: + allocations[index] = len(excerpts[index]) + else: + selected = nonempty[: min(len(nonempty), remaining // 3)] + content_budget = remaining - 2 * len(selected) + pending = set(selected) + while pending: + share = content_budget // len(pending) + satisfied = [index for index in pending if len(excerpts[index]) <= share] + if not satisfied: + base, extra = divmod(content_budget, len(pending)) + for offset, index in enumerate(sorted(pending)): + allocations[index] = base + (offset < extra) + break + for index in satisfied: + allocations[index] = len(excerpts[index]) + content_budget -= allocations[index] + pending.remove(index) + blocks: list[str] = [] - truncated = False - for index, (header, excerpts) in enumerate(results): - share = remaining // (len(results) - index) - excerpt_slice = excerpts[: max(0, share - 2)] - excerpt = excerpt_slice.rstrip() + for index, (header, _) in enumerate(results): + excerpt = excerpts[index][: allocations[index]].rstrip() blocks.append(f"{header}\n\n{excerpt}" if excerpt else header) - if excerpt: - remaining -= len(excerpt) + 2 - truncated |= len(excerpt_slice) < len(excerpts) + truncated = any( + allocation < len(excerpt) for allocation, excerpt in zip(allocations, excerpts, strict=True) + ) content = "\n\n".join(blocks) if truncated: content += "\n[excerpts truncated to preserve source links]" @@ -234,7 +256,10 @@ def _exa_page_text(api_key: str, urls: list[str]) -> tuple[str, set[str]]: def _parallel_page_text(api_key: str, urls: list[str]) -> tuple[str, set[str]]: """Fetch page text and report which of the requested URLs Parallel returned.""" headers = {"x-api-key": api_key, "Content-Type": "application/json"} - payload = {"urls": urls, "advanced_settings": {"full_content": True}} + payload = { + "urls": urls, + "advanced_settings": {"full_content": {"max_chars_per_result": _EXA_PAGE_MAX_CHARS}}, + } with requests.post( "https://api.parallel.ai/v1/extract", headers=headers, @@ -270,6 +295,10 @@ def _parallel_page_text(api_key: str, urls: list[str]) -> tuple[str, set[str]]: ) if not content: continue + if len(content) > _EXA_PAGE_MAX_CHARS: + content = ( + f"{content[:_EXA_PAGE_MAX_CHARS]}\n[truncated at {_EXA_PAGE_MAX_CHARS} characters]" + ) raw_title = result.get("title") title = raw_title.strip() if isinstance(raw_title, str) else "" blocks.append(f"### {title or url}\n{url}\n\n{content}") diff --git a/tests/test_web_search.py b/tests/test_web_search.py index 5a0efb0b9..5f65c593a 100644 --- a/tests/test_web_search.py +++ b/tests/test_web_search.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast import pytest import requests @@ -513,7 +513,7 @@ def test_parallel_page_text_requests_full_content_and_renders_pages( } assert captured["json"] == { "urls": ["https://vendor.example/advisory", "https://vendor.example/docs"], - "advanced_settings": {"full_content": True}, + "advanced_settings": {"full_content": {"max_chars_per_result": tool._EXA_PAGE_MAX_CHARS}}, } assert captured["timeout"] == 300 assert content == ( @@ -526,6 +526,107 @@ def test_parallel_page_text_requests_full_content_and_renders_pages( } +@pytest.mark.parametrize( + ("body", "exception", "message"), + [ + (cast("Any", []), TypeError, "must be an object"), + ({"results": []}, ValueError, "no page contents"), + ( + { + "results": [ + { + "url": "https://empty.example", + "full_content": " ", + "excerpts": [" "], + } + ] + }, + ValueError, + "no page contents", + ), + ], +) +def test_parallel_page_text_rejects_empty_or_contentless_responses( + monkeypatch: pytest.MonkeyPatch, + body: Any, + exception: type[Exception], + message: str, +) -> None: + monkeypatch.setattr(requests, "post", lambda *_a, **_kw: _FakeResponse(body)) + + with pytest.raises(exception, match=message): + tool._parallel_page_text("parallel-key", ["https://empty.example"]) + + +def test_parallel_page_text_truncates_a_long_page(monkeypatch: pytest.MonkeyPatch) -> None: + body = "A" * (tool._EXA_PAGE_MAX_CHARS + 500) + monkeypatch.setattr( + requests, + "post", + lambda *_a, **_kw: _FakeResponse( + { + "results": [ + { + "url": "https://long.example", + "full_content": body, + "excerpts": [], + } + ] + } + ), + ) + + content, fetched = tool._parallel_page_text("parallel-key", ["https://long.example"]) + + assert content.count("A") == tool._EXA_PAGE_MAX_CHARS + assert f"[truncated at {tool._EXA_PAGE_MAX_CHARS} characters]" in content + assert fetched == {"https://long.example"} + + +def test_do_get_contents_reports_parallel_extract_partial_failures( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _Settings: + integrations = IntegrationSettings( + PARALLEL_API_KEY="parallel-key", + STRIX_WEB_SEARCH_PROVIDER="parallel", + ) + + monkeypatch.setattr(tool, "load_settings", _Settings) + monkeypatch.setattr( + requests, + "post", + lambda *_a, **_kw: _FakeResponse( + { + "results": [ + { + "url": "https://ok.example/", + "full_content": "Page body.", + "excerpts": [], + } + ], + "errors": [ + { + "url": "https://blocked.example", + "error_type": "fetch_error", + "http_status_code": 403, + "content": None, + } + ], + } + ), + ) + + result = tool._do_get_contents(["https://ok.example", "https://blocked.example"]) + + assert result["success"] is True + assert result["provider"] == "parallel" + assert result["urls"] == ["https://ok.example"] + assert result["failed_urls"] == ["https://blocked.example"] + assert "Parallel Extract" in result["warning"] + assert "1 of 2" in result["warning"] + + def test_do_search_reports_the_provider_it_used(monkeypatch: pytest.MonkeyPatch) -> None: class _Settings: integrations = IntegrationSettings(EXA_API_KEY="ek") @@ -744,6 +845,20 @@ def test_parallel_spreads_excerpt_budget_and_strips_trailing_whitespace( assert content.endswith("[excerpts truncated to preserve source links]") +def test_parallel_redistributes_unused_excerpt_budget() -> None: + results = [ + ("### Long\nhttps://long.example", "a" * 3500), + *[(f"### Short {index}\nhttps://short{index}.example", "b" * 100) for index in range(4)], + ] + + content = tool._parallel_result_content(results) + + assert "a" * 3500 in content + assert content.count("b" * 100) == 4 + assert "[excerpts truncated" not in content + assert len(content) <= tool._PARALLEL_MAX_CHARS + + def test_parallel_strips_trailing_excerpt_whitespace() -> None: content = tool._parallel_result_content( [("### Result\nhttps://result.example", "Complete excerpt.\n\n")]