mirror of
https://github.com/usestrix/strix.git
synced 2026-11-14 09:01:25 +00:00
fix(web_search): cap and rebalance Parallel content
Co-authored-by: Rowan Aldean <RowanAldean@users.noreply.github.com>
This commit is contained in:
Cursor Agent
parent
90acb01e88
commit
e36603632e
2 changed files with 155 additions and 11 deletions
|
|
@ -139,16 +139,38 @@ _PARALLEL_MAX_CHARS = 12000
|
|||
def _parallel_result_content(results: list[tuple[str, str]]) -> str:
|
||||
header_chars = sum(len(header) for header, _ in results) + 2 * (len(results) - 1)
|
||||
remaining = max(0, _PARALLEL_MAX_CHARS - header_chars)
|
||||
excerpts = [excerpt.rstrip() for _, excerpt in results]
|
||||
allocations = [0] * len(results)
|
||||
nonempty = [index for index, excerpt in enumerate(excerpts) if excerpt]
|
||||
full_cost = sum(len(excerpts[index]) + 2 for index in nonempty)
|
||||
|
||||
if full_cost <= remaining:
|
||||
for index in nonempty:
|
||||
allocations[index] = len(excerpts[index])
|
||||
else:
|
||||
selected = nonempty[: min(len(nonempty), remaining // 3)]
|
||||
content_budget = remaining - 2 * len(selected)
|
||||
pending = set(selected)
|
||||
while pending:
|
||||
share = content_budget // len(pending)
|
||||
satisfied = [index for index in pending if len(excerpts[index]) <= share]
|
||||
if not satisfied:
|
||||
base, extra = divmod(content_budget, len(pending))
|
||||
for offset, index in enumerate(sorted(pending)):
|
||||
allocations[index] = base + (offset < extra)
|
||||
break
|
||||
for index in satisfied:
|
||||
allocations[index] = len(excerpts[index])
|
||||
content_budget -= allocations[index]
|
||||
pending.remove(index)
|
||||
|
||||
blocks: list[str] = []
|
||||
truncated = False
|
||||
for index, (header, excerpts) in enumerate(results):
|
||||
share = remaining // (len(results) - index)
|
||||
excerpt_slice = excerpts[: max(0, share - 2)]
|
||||
excerpt = excerpt_slice.rstrip()
|
||||
for index, (header, _) in enumerate(results):
|
||||
excerpt = excerpts[index][: allocations[index]].rstrip()
|
||||
blocks.append(f"{header}\n\n{excerpt}" if excerpt else header)
|
||||
if excerpt:
|
||||
remaining -= len(excerpt) + 2
|
||||
truncated |= len(excerpt_slice) < len(excerpts)
|
||||
truncated = any(
|
||||
allocation < len(excerpt) for allocation, excerpt in zip(allocations, excerpts, strict=True)
|
||||
)
|
||||
content = "\n\n".join(blocks)
|
||||
if truncated:
|
||||
content += "\n[excerpts truncated to preserve source links]"
|
||||
|
|
@ -234,7 +256,10 @@ def _exa_page_text(api_key: str, urls: list[str]) -> tuple[str, set[str]]:
|
|||
def _parallel_page_text(api_key: str, urls: list[str]) -> tuple[str, set[str]]:
|
||||
"""Fetch page text and report which of the requested URLs Parallel returned."""
|
||||
headers = {"x-api-key": api_key, "Content-Type": "application/json"}
|
||||
payload = {"urls": urls, "advanced_settings": {"full_content": True}}
|
||||
payload = {
|
||||
"urls": urls,
|
||||
"advanced_settings": {"full_content": {"max_chars_per_result": _EXA_PAGE_MAX_CHARS}},
|
||||
}
|
||||
with requests.post(
|
||||
"https://api.parallel.ai/v1/extract",
|
||||
headers=headers,
|
||||
|
|
@ -270,6 +295,10 @@ def _parallel_page_text(api_key: str, urls: list[str]) -> tuple[str, set[str]]:
|
|||
)
|
||||
if not content:
|
||||
continue
|
||||
if len(content) > _EXA_PAGE_MAX_CHARS:
|
||||
content = (
|
||||
f"{content[:_EXA_PAGE_MAX_CHARS]}\n[truncated at {_EXA_PAGE_MAX_CHARS} characters]"
|
||||
)
|
||||
raw_title = result.get("title")
|
||||
title = raw_title.strip() if isinstance(raw_title, str) else ""
|
||||
blocks.append(f"### {title or url}\n{url}\n\n{content}")
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
|
@ -513,7 +513,7 @@ def test_parallel_page_text_requests_full_content_and_renders_pages(
|
|||
}
|
||||
assert captured["json"] == {
|
||||
"urls": ["https://vendor.example/advisory", "https://vendor.example/docs"],
|
||||
"advanced_settings": {"full_content": True},
|
||||
"advanced_settings": {"full_content": {"max_chars_per_result": tool._EXA_PAGE_MAX_CHARS}},
|
||||
}
|
||||
assert captured["timeout"] == 300
|
||||
assert content == (
|
||||
|
|
@ -526,6 +526,107 @@ def test_parallel_page_text_requests_full_content_and_renders_pages(
|
|||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "exception", "message"),
|
||||
[
|
||||
(cast("Any", []), TypeError, "must be an object"),
|
||||
({"results": []}, ValueError, "no page contents"),
|
||||
(
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"url": "https://empty.example",
|
||||
"full_content": " ",
|
||||
"excerpts": [" "],
|
||||
}
|
||||
]
|
||||
},
|
||||
ValueError,
|
||||
"no page contents",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_parallel_page_text_rejects_empty_or_contentless_responses(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
body: Any,
|
||||
exception: type[Exception],
|
||||
message: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr(requests, "post", lambda *_a, **_kw: _FakeResponse(body))
|
||||
|
||||
with pytest.raises(exception, match=message):
|
||||
tool._parallel_page_text("parallel-key", ["https://empty.example"])
|
||||
|
||||
|
||||
def test_parallel_page_text_truncates_a_long_page(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
body = "A" * (tool._EXA_PAGE_MAX_CHARS + 500)
|
||||
monkeypatch.setattr(
|
||||
requests,
|
||||
"post",
|
||||
lambda *_a, **_kw: _FakeResponse(
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"url": "https://long.example",
|
||||
"full_content": body,
|
||||
"excerpts": [],
|
||||
}
|
||||
]
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
content, fetched = tool._parallel_page_text("parallel-key", ["https://long.example"])
|
||||
|
||||
assert content.count("A") == tool._EXA_PAGE_MAX_CHARS
|
||||
assert f"[truncated at {tool._EXA_PAGE_MAX_CHARS} characters]" in content
|
||||
assert fetched == {"https://long.example"}
|
||||
|
||||
|
||||
def test_do_get_contents_reports_parallel_extract_partial_failures(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
class _Settings:
|
||||
integrations = IntegrationSettings(
|
||||
PARALLEL_API_KEY="parallel-key",
|
||||
STRIX_WEB_SEARCH_PROVIDER="parallel",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(tool, "load_settings", _Settings)
|
||||
monkeypatch.setattr(
|
||||
requests,
|
||||
"post",
|
||||
lambda *_a, **_kw: _FakeResponse(
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"url": "https://ok.example/",
|
||||
"full_content": "Page body.",
|
||||
"excerpts": [],
|
||||
}
|
||||
],
|
||||
"errors": [
|
||||
{
|
||||
"url": "https://blocked.example",
|
||||
"error_type": "fetch_error",
|
||||
"http_status_code": 403,
|
||||
"content": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
result = tool._do_get_contents(["https://ok.example", "https://blocked.example"])
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["provider"] == "parallel"
|
||||
assert result["urls"] == ["https://ok.example"]
|
||||
assert result["failed_urls"] == ["https://blocked.example"]
|
||||
assert "Parallel Extract" in result["warning"]
|
||||
assert "1 of 2" in result["warning"]
|
||||
|
||||
|
||||
def test_do_search_reports_the_provider_it_used(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
class _Settings:
|
||||
integrations = IntegrationSettings(EXA_API_KEY="ek")
|
||||
|
|
@ -744,6 +845,20 @@ def test_parallel_spreads_excerpt_budget_and_strips_trailing_whitespace(
|
|||
assert content.endswith("[excerpts truncated to preserve source links]")
|
||||
|
||||
|
||||
def test_parallel_redistributes_unused_excerpt_budget() -> None:
|
||||
results = [
|
||||
("### Long\nhttps://long.example", "a" * 3500),
|
||||
*[(f"### Short {index}\nhttps://short{index}.example", "b" * 100) for index in range(4)],
|
||||
]
|
||||
|
||||
content = tool._parallel_result_content(results)
|
||||
|
||||
assert "a" * 3500 in content
|
||||
assert content.count("b" * 100) == 4
|
||||
assert "[excerpts truncated" not in content
|
||||
assert len(content) <= tool._PARALLEL_MAX_CHARS
|
||||
|
||||
|
||||
def test_parallel_strips_trailing_excerpt_whitespace() -> None:
|
||||
content = tool._parallel_result_content(
|
||||
[("### Result\nhttps://result.example", "Complete excerpt.\n\n")]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue