mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(spend): keep web search rounds inside interception follow-ups billed
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
f56c469fa7
commit
dff64ad609
4 changed files with 91 additions and 22 deletions
|
|
@ -52,6 +52,18 @@ def internal_sub_call(parent_request_id: str | None = None) -> Generator[None]:
|
|||
internal_parent_request_id.reset(parent_token)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def billed_separately() -> Generator[None]:
|
||||
"""A call that writes its own spend row even inside a sub-call whose parent bills the model response."""
|
||||
token: Final = is_internal_call.set(False)
|
||||
parent_token: Final = internal_parent_request_id.set(None)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
is_internal_call.reset(token)
|
||||
internal_parent_request_id.reset(parent_token)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def post_response_phase() -> Generator[None]:
|
||||
"""Work the caller no longer waits for (success callbacks, response-cache writes), including tasks it spawns."""
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast
|
|||
from typing_extensions import Never, ReadOnly
|
||||
|
||||
import litellm
|
||||
from litellm._internal_context import billed_separately
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.anthropic_interface import messages as anthropic_messages
|
||||
from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
|
||||
|
|
@ -1628,19 +1629,20 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
search_kwargs: Final = MappingProxyType(
|
||||
{**configured_search_kwargs, **parent_correlation.as_search_kwargs()}
|
||||
)
|
||||
result: Final = (
|
||||
await litellm.asearch(
|
||||
query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs
|
||||
with billed_separately():
|
||||
result: Final = (
|
||||
await litellm.asearch(
|
||||
query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs
|
||||
)
|
||||
if search_metadata is None
|
||||
else await litellm.asearch(
|
||||
query=query_arg,
|
||||
search_provider=search_provider,
|
||||
litellm_metadata=search_metadata,
|
||||
**_NO_ASEARCH_NAMED,
|
||||
**search_kwargs,
|
||||
)
|
||||
)
|
||||
if search_metadata is None
|
||||
else await litellm.asearch(
|
||||
query=query_arg,
|
||||
search_provider=search_provider,
|
||||
litellm_metadata=search_metadata,
|
||||
**_NO_ASEARCH_NAMED,
|
||||
**search_kwargs,
|
||||
)
|
||||
)
|
||||
|
||||
# Format using transformation function
|
||||
search_result_text: Final = WebSearchTransformation.format_search_response(result)
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
|||
_ITEMS: Final = TypeAdapter(list[dict[str, JsonValue]])
|
||||
_CHAT_TARGET: Final = re.compile(r"^/openai/deployments/([^/]+)/chat/completions$")
|
||||
|
||||
Mode: TypeAlias = Literal["search", "plain", "search-down", "followup-fails", "missing"]
|
||||
Mode: TypeAlias = Literal["search", "multi-search", "plain", "search-down", "followup-fails", "missing"]
|
||||
|
||||
|
||||
def deployment(mode: Mode) -> str:
|
||||
|
|
@ -42,8 +42,8 @@ def plain(name: str) -> str:
|
|||
return f"plain-{name}"
|
||||
|
||||
|
||||
def query(name: str) -> str:
|
||||
return f"q-{name}"
|
||||
def query(name: str, turn: int = 1) -> str:
|
||||
return f"q-{name}" if turn == 1 else f"q-{name}-{turn}"
|
||||
|
||||
|
||||
def _minted(prefix: str) -> str:
|
||||
|
|
@ -98,9 +98,20 @@ def _chat(name: str, body: Mapping[str, JsonValue]) -> Reply:
|
|||
content_type="text/event-stream", chunks=chat_stream(_minted(f"chatcmpl-{name}-s-"), "gpt-4o", plain(name))
|
||||
)
|
||||
messages: Final = _ITEMS.validate_python(body["messages"])
|
||||
if any(message.get("role") == "tool" for message in messages):
|
||||
tool_results: Final = tuple(message for message in messages if message.get("role") == "tool")
|
||||
if tool_results:
|
||||
if name.endswith("-followup-fails"):
|
||||
return _azure_error(500, FOLLOWUP_FAILURE)
|
||||
if name.endswith("-multi-search") and len(tool_results) == 1:
|
||||
call: Final = {
|
||||
"id": f"call_{name[4:36]}-2",
|
||||
"type": "function",
|
||||
"function": {"name": SEARCH_TOOL, "arguments": json.dumps({"query": query(name, turn=2)})},
|
||||
}
|
||||
message: Final = JSON_OBJECT.validate_python({"role": "assistant", "content": None, "tool_calls": [call]})
|
||||
return _chat_completion(
|
||||
_minted(f"chatcmpl-{name}-2-"), message, "tool_calls", FOLLOWUP_USAGE
|
||||
)
|
||||
return _chat_completion(
|
||||
_minted(f"chatcmpl-{name}-2-"), {"role": "assistant", "content": answer(name)}, "stop", FOLLOWUP_USAGE
|
||||
)
|
||||
|
|
@ -153,9 +164,20 @@ def _responses(body: Mapping[str, JsonValue]) -> Reply:
|
|||
content_type="text/event-stream", chunks=responses_stream(_minted(f"resp_{name}_s_"), name, plain(name))
|
||||
)
|
||||
items: Final = _ITEMS.validate_python(body["input"]) if isinstance(body.get("input"), list) else []
|
||||
if any(item.get("type") == "function_call_output" for item in items):
|
||||
function_call_outputs: Final = tuple(item for item in items if item.get("type") == "function_call_output")
|
||||
if function_call_outputs:
|
||||
if name.endswith("-followup-fails"):
|
||||
return _azure_error(500, FOLLOWUP_FAILURE)
|
||||
if name.endswith("-multi-search") and len(function_call_outputs) == 1:
|
||||
call: Final = {
|
||||
"type": "function_call",
|
||||
"id": f"fc_{name[4:36]}-2",
|
||||
"call_id": f"call_{name[4:36]}-2",
|
||||
"name": SEARCH_TOOL,
|
||||
"arguments": json.dumps({"query": query(name, turn=2)}),
|
||||
"status": "completed",
|
||||
}
|
||||
return _response(_minted(f"resp_{name}_2_"), name, (call,), RESPONSES_FOLLOWUP_USAGE)
|
||||
return _response(
|
||||
_minted(f"resp_{name}_2_"), name, (_text_item(f"{name}_2", answer(name)),), RESPONSES_FOLLOWUP_USAGE
|
||||
)
|
||||
|
|
|
|||
|
|
@ -329,7 +329,7 @@ def _assert_response_landed_once(model: str, identity: str) -> None:
|
|||
assert row["status"] == "success" and rv.same_response(string_value(row["request_id"]), identity), row
|
||||
|
||||
|
||||
def _assert_single_model_spend_for_key(key: str, call_type: str) -> None:
|
||||
def _assert_single_model_spend_for_key(key: str, call_type: str, *, expected_searches: int = 1) -> None:
|
||||
digest: Final = sha256(key.encode()).hexdigest()
|
||||
observed: Final = eventually(
|
||||
lambda: (
|
||||
|
|
@ -343,9 +343,9 @@ def _assert_single_model_spend_for_key(key: str, call_type: str) -> None:
|
|||
lambda values: (
|
||||
bool(values[0])
|
||||
and (
|
||||
len(values[0]) > 2
|
||||
len(values[0]) > expected_searches + 1
|
||||
or (
|
||||
len(values[0]) == 2
|
||||
len(values[0]) == expected_searches + 1
|
||||
and len(values[1]) == 1
|
||||
and values[1][0]["spend"] is not None
|
||||
and float(str(values[1][0]["spend"]))
|
||||
|
|
@ -360,10 +360,11 @@ def _assert_single_model_spend_for_key(key: str, call_type: str) -> None:
|
|||
search_rows: Final = [row for row in rows if row["call_type"] == "asearch"]
|
||||
spend: Final = sum(float(str(row["spend"])) for row in rows)
|
||||
assert (
|
||||
len(rows) == 2
|
||||
len(rows) == expected_searches + 1
|
||||
and len(model_rows) == 1
|
||||
and len(search_rows) == 1
|
||||
and len(search_rows) == expected_searches
|
||||
and float(str(model_rows[0]["spend"])) > 0
|
||||
and all(float(str(row["spend"])) > 0 for row in search_rows)
|
||||
and float(str(balance[0]["spend"])) == pytest.approx(spend)
|
||||
), (rows, balance)
|
||||
|
||||
|
|
@ -454,6 +455,22 @@ def test_chat_stream_through_web_search_has_single_spend_owner(rig: _Rig, scenar
|
|||
_assert_single_model_spend_for_key(key, "acompletion")
|
||||
|
||||
|
||||
def test_multi_round_chat_stream_through_web_search_bills_each_search(
|
||||
rig: _Rig, scenario: Scenario
|
||||
) -> None:
|
||||
deployment: Final = _deploy(rig, scenario, "multi-search", priced=True)
|
||||
key: Final = scenario.key(models=[deployment.model])
|
||||
stream: Final = _post(rig, "/v1/chat/completions", _chat_body(deployment), key=key)
|
||||
identity: Final = _assert_restreamed(stream, iv.answer(deployment.name), usage=None)
|
||||
received: Final = _upstream(rig, deployment)
|
||||
_assert_upstream(received, model_calls=3, searches=2)
|
||||
assert [search["query"] for search in received.searches] == [
|
||||
iv.query(deployment.name),
|
||||
iv.query(deployment.name, turn=2),
|
||||
], received.searches
|
||||
_assert_single_model_spend_for_key(key, "acompletion", expected_searches=2)
|
||||
|
||||
|
||||
def test_openai_sdk_responses_stream_through_web_search_keeps_stream_options_off_azure(
|
||||
rig: _Rig, scenario: Scenario
|
||||
) -> None:
|
||||
|
|
@ -515,6 +532,22 @@ def test_responses_stream_through_web_search_has_single_spend_owner(rig: _Rig, s
|
|||
_assert_single_model_spend_for_key(key, "aresponses")
|
||||
|
||||
|
||||
def test_multi_round_responses_stream_through_web_search_bills_each_search(
|
||||
rig: _Rig, scenario: Scenario
|
||||
) -> None:
|
||||
deployment: Final = _deploy(rig, scenario, "multi-search", priced=True)
|
||||
key: Final = scenario.key(models=[deployment.model])
|
||||
stream: Final = _post(rig, "/v1/responses", _responses_body(deployment), key=key)
|
||||
_assert_responses_restreamed(stream, iv.answer(deployment.name))
|
||||
received: Final = _upstream(rig, deployment)
|
||||
_assert_upstream(received, model_calls=3, searches=2)
|
||||
assert [search["query"] for search in received.searches] == [
|
||||
iv.query(deployment.name),
|
||||
iv.query(deployment.name, turn=2),
|
||||
], received.searches
|
||||
_assert_single_model_spend_for_key(key, "aresponses", expected_searches=2)
|
||||
|
||||
|
||||
def test_openai_sdk_chat_without_streaming_through_web_search_answers_in_json(rig: _Rig, scenario: Scenario) -> None:
|
||||
deployment: Final = _deploy(rig, scenario, "search")
|
||||
completion: Final = _openai(rig).chat.completions.create(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue