fix(spend): keep web search rounds inside interception follow-ups billed

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mateo 2026-11-03 01:47:04 +00:00
parent f56c469fa7
commit dff64ad609
4 changed files with 91 additions and 22 deletions

View file

@ -52,6 +52,18 @@ def internal_sub_call(parent_request_id: str | None = None) -> Generator[None]:
internal_parent_request_id.reset(parent_token)
@contextmanager
def billed_separately() -> Generator[None]:
"""A call that writes its own spend row even inside a sub-call whose parent bills the model response."""
token: Final = is_internal_call.set(False)
parent_token: Final = internal_parent_request_id.set(None)
try:
yield
finally:
is_internal_call.reset(token)
internal_parent_request_id.reset(parent_token)
@contextmanager
def post_response_phase() -> Generator[None]:
"""Work the caller no longer waits for (success callbacks, response-cache writes), including tasks it spawns."""

View file

@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast
from typing_extensions import Never, ReadOnly
import litellm
from litellm._internal_context import billed_separately
from litellm._logging import verbose_logger
from litellm.anthropic_interface import messages as anthropic_messages
from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
@ -1628,19 +1629,20 @@ class WebSearchInterceptionLogger(CustomLogger):
search_kwargs: Final = MappingProxyType(
{**configured_search_kwargs, **parent_correlation.as_search_kwargs()}
)
result: Final = (
await litellm.asearch(
query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs
with billed_separately():
result: Final = (
await litellm.asearch(
query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs
)
if search_metadata is None
else await litellm.asearch(
query=query_arg,
search_provider=search_provider,
litellm_metadata=search_metadata,
**_NO_ASEARCH_NAMED,
**search_kwargs,
)
)
if search_metadata is None
else await litellm.asearch(
query=query_arg,
search_provider=search_provider,
litellm_metadata=search_metadata,
**_NO_ASEARCH_NAMED,
**search_kwargs,
)
)
# Format using transformation function
search_result_text: Final = WebSearchTransformation.format_search_response(result)

View file

@ -27,7 +27,7 @@ JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_ITEMS: Final = TypeAdapter(list[dict[str, JsonValue]])
_CHAT_TARGET: Final = re.compile(r"^/openai/deployments/([^/]+)/chat/completions$")
Mode: TypeAlias = Literal["search", "plain", "search-down", "followup-fails", "missing"]
Mode: TypeAlias = Literal["search", "multi-search", "plain", "search-down", "followup-fails", "missing"]
def deployment(mode: Mode) -> str:
@ -42,8 +42,8 @@ def plain(name: str) -> str:
return f"plain-{name}"
def query(name: str) -> str:
return f"q-{name}"
def query(name: str, turn: int = 1) -> str:
return f"q-{name}" if turn == 1 else f"q-{name}-{turn}"
def _minted(prefix: str) -> str:
@ -98,9 +98,20 @@ def _chat(name: str, body: Mapping[str, JsonValue]) -> Reply:
content_type="text/event-stream", chunks=chat_stream(_minted(f"chatcmpl-{name}-s-"), "gpt-4o", plain(name))
)
messages: Final = _ITEMS.validate_python(body["messages"])
if any(message.get("role") == "tool" for message in messages):
tool_results: Final = tuple(message for message in messages if message.get("role") == "tool")
if tool_results:
if name.endswith("-followup-fails"):
return _azure_error(500, FOLLOWUP_FAILURE)
if name.endswith("-multi-search") and len(tool_results) == 1:
call: Final = {
"id": f"call_{name[4:36]}-2",
"type": "function",
"function": {"name": SEARCH_TOOL, "arguments": json.dumps({"query": query(name, turn=2)})},
}
message: Final = JSON_OBJECT.validate_python({"role": "assistant", "content": None, "tool_calls": [call]})
return _chat_completion(
_minted(f"chatcmpl-{name}-2-"), message, "tool_calls", FOLLOWUP_USAGE
)
return _chat_completion(
_minted(f"chatcmpl-{name}-2-"), {"role": "assistant", "content": answer(name)}, "stop", FOLLOWUP_USAGE
)
@ -153,9 +164,20 @@ def _responses(body: Mapping[str, JsonValue]) -> Reply:
content_type="text/event-stream", chunks=responses_stream(_minted(f"resp_{name}_s_"), name, plain(name))
)
items: Final = _ITEMS.validate_python(body["input"]) if isinstance(body.get("input"), list) else []
if any(item.get("type") == "function_call_output" for item in items):
function_call_outputs: Final = tuple(item for item in items if item.get("type") == "function_call_output")
if function_call_outputs:
if name.endswith("-followup-fails"):
return _azure_error(500, FOLLOWUP_FAILURE)
if name.endswith("-multi-search") and len(function_call_outputs) == 1:
call: Final = {
"type": "function_call",
"id": f"fc_{name[4:36]}-2",
"call_id": f"call_{name[4:36]}-2",
"name": SEARCH_TOOL,
"arguments": json.dumps({"query": query(name, turn=2)}),
"status": "completed",
}
return _response(_minted(f"resp_{name}_2_"), name, (call,), RESPONSES_FOLLOWUP_USAGE)
return _response(
_minted(f"resp_{name}_2_"), name, (_text_item(f"{name}_2", answer(name)),), RESPONSES_FOLLOWUP_USAGE
)

View file

@ -329,7 +329,7 @@ def _assert_response_landed_once(model: str, identity: str) -> None:
assert row["status"] == "success" and rv.same_response(string_value(row["request_id"]), identity), row
def _assert_single_model_spend_for_key(key: str, call_type: str) -> None:
def _assert_single_model_spend_for_key(key: str, call_type: str, *, expected_searches: int = 1) -> None:
digest: Final = sha256(key.encode()).hexdigest()
observed: Final = eventually(
lambda: (
@ -343,9 +343,9 @@ def _assert_single_model_spend_for_key(key: str, call_type: str) -> None:
lambda values: (
bool(values[0])
and (
len(values[0]) > 2
len(values[0]) > expected_searches + 1
or (
len(values[0]) == 2
len(values[0]) == expected_searches + 1
and len(values[1]) == 1
and values[1][0]["spend"] is not None
and float(str(values[1][0]["spend"]))
@ -360,10 +360,11 @@ def _assert_single_model_spend_for_key(key: str, call_type: str) -> None:
search_rows: Final = [row for row in rows if row["call_type"] == "asearch"]
spend: Final = sum(float(str(row["spend"])) for row in rows)
assert (
len(rows) == 2
len(rows) == expected_searches + 1
and len(model_rows) == 1
and len(search_rows) == 1
and len(search_rows) == expected_searches
and float(str(model_rows[0]["spend"])) > 0
and all(float(str(row["spend"])) > 0 for row in search_rows)
and float(str(balance[0]["spend"])) == pytest.approx(spend)
), (rows, balance)
@ -454,6 +455,22 @@ def test_chat_stream_through_web_search_has_single_spend_owner(rig: _Rig, scenar
_assert_single_model_spend_for_key(key, "acompletion")
def test_multi_round_chat_stream_through_web_search_bills_each_search(
rig: _Rig, scenario: Scenario
) -> None:
deployment: Final = _deploy(rig, scenario, "multi-search", priced=True)
key: Final = scenario.key(models=[deployment.model])
stream: Final = _post(rig, "/v1/chat/completions", _chat_body(deployment), key=key)
identity: Final = _assert_restreamed(stream, iv.answer(deployment.name), usage=None)
received: Final = _upstream(rig, deployment)
_assert_upstream(received, model_calls=3, searches=2)
assert [search["query"] for search in received.searches] == [
iv.query(deployment.name),
iv.query(deployment.name, turn=2),
], received.searches
_assert_single_model_spend_for_key(key, "acompletion", expected_searches=2)
def test_openai_sdk_responses_stream_through_web_search_keeps_stream_options_off_azure(
rig: _Rig, scenario: Scenario
) -> None:
@ -515,6 +532,22 @@ def test_responses_stream_through_web_search_has_single_spend_owner(rig: _Rig, s
_assert_single_model_spend_for_key(key, "aresponses")
def test_multi_round_responses_stream_through_web_search_bills_each_search(
rig: _Rig, scenario: Scenario
) -> None:
deployment: Final = _deploy(rig, scenario, "multi-search", priced=True)
key: Final = scenario.key(models=[deployment.model])
stream: Final = _post(rig, "/v1/responses", _responses_body(deployment), key=key)
_assert_responses_restreamed(stream, iv.answer(deployment.name))
received: Final = _upstream(rig, deployment)
_assert_upstream(received, model_calls=3, searches=2)
assert [search["query"] for search in received.searches] == [
iv.query(deployment.name),
iv.query(deployment.name, turn=2),
], received.searches
_assert_single_model_spend_for_key(key, "aresponses", expected_searches=2)
def test_openai_sdk_chat_without_streaming_through_web_search_answers_in_json(rig: _Rig, scenario: Scenario) -> None:
deployment: Final = _deploy(rig, scenario, "search")
completion: Final = _openai(rig).chat.completions.create(