mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(google_genai): keep pre-call rule rejections out of failure logging
A request a pre-call rule rejects never reaches a deployment, so the adapter now marks its failure as logged, the same as a rejected chat call. It also skips the rules for an empty conversation, as chat's setup does. On this line the Router cools a deployment down from that failure callback, so logging the rejection took healthy deployments out of rotation.
This commit is contained in:
parent
dd07c4abd9
commit
73729f703c
2 changed files with 58 additions and 10 deletions
|
|
@ -32,6 +32,15 @@ def _message_text(messages: Iterable[Mapping[str, object]]) -> str:
|
|||
return "".join(content for message in messages if isinstance(content := message.get("content"), str))
|
||||
|
||||
|
||||
def _run_pre_call_rules(logging_obj: LiteLLMLoggingObj, messages: Iterable[Mapping[str, object]], model: str) -> None:
|
||||
try:
|
||||
Rules().pre_call_rules(input=_message_text(messages), model=model)
|
||||
except Exception:
|
||||
logging_obj.mark_logging_complete(event_type="sync_failure")
|
||||
logging_obj.mark_logging_complete(event_type="async_failure")
|
||||
raise
|
||||
|
||||
|
||||
def _adapter_stream_options(requested: object) -> ChatCompletionStreamOptionsParam | None:
|
||||
match requested:
|
||||
case {"include_usage": True}:
|
||||
|
|
@ -78,11 +87,9 @@ class GenerateContentToCompletionHandler:
|
|||
**({"stream_options": stream_options} if stream_options is not None else {}),
|
||||
)
|
||||
)
|
||||
if isinstance(logging_obj, LiteLLMLoggingObj) and Rules.has_pre_call_rules():
|
||||
Rules().pre_call_rules(
|
||||
input=_message_text(completion_kwargs.get("messages", [])),
|
||||
model=model,
|
||||
)
|
||||
messages: Final = completion_kwargs.get("messages", ())
|
||||
if isinstance(logging_obj, LiteLLMLoggingObj) and messages and Rules.has_pre_call_rules():
|
||||
_run_pre_call_rules(logging_obj, messages, model)
|
||||
|
||||
return completion_kwargs
|
||||
|
||||
|
|
|
|||
|
|
@ -858,19 +858,60 @@ async def test_generate_content_adapter_keeps_the_request_no_log_and_session_id(
|
|||
async def test_generate_content_adapter_pre_call_rules_reject_the_request_before_the_upstream_call(
|
||||
_httpx_transport: None, monkeypatch: pytest.MonkeyPatch, entrypoint: str
|
||||
):
|
||||
logged: Final = _LoggedEvents()
|
||||
monkeypatch.setattr(litellm, "callbacks", [logged])
|
||||
monkeypatch.setattr(litellm, "pre_call_rules", [lambda text: "forbidden" not in text])
|
||||
generate: Final = getattr(_adapter_router(), entrypoint)
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.post("http://upstream.test/v1/chat/completions").mock(
|
||||
return_value=_upstream_reply(stream=entrypoint == "agenerate_content_stream")
|
||||
)
|
||||
with pytest.raises(litellm.APIResponseValidationError):
|
||||
await getattr(_adapter_router(), entrypoint)(
|
||||
model="grok-group",
|
||||
contents=[{"role": "user", "parts": [{"text": "a forbidden question"}]}],
|
||||
)
|
||||
await generate(model="grok-group", contents=[{"role": "user", "parts": [{"text": "a forbidden question"}]}])
|
||||
allowed: Final = await generate(
|
||||
model="grok-group",
|
||||
contents=[{"role": "user", "parts": [{"text": "hello"}]}],
|
||||
litellm_call_id=_ADAPTER_CALL_ID,
|
||||
)
|
||||
if entrypoint == "agenerate_content_stream":
|
||||
await _drain(allowed)
|
||||
await await_until(lambda: bool(logged.events), message="the allowed request was logged")
|
||||
|
||||
assert route.call_count == 0
|
||||
assert route.call_count == 1
|
||||
assert [(outcome, call_id) for outcome, _, call_id, _ in logged.events] == [("success", _ADAPTER_CALL_ID)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_content_adapter_pre_call_rule_rejections_leave_the_deployments_available(
|
||||
_httpx_transport: None, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "pre_call_rules", [lambda text: "forbidden" not in text])
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "grok-group",
|
||||
"litellm_params": {"model": "hosted_vllm/grok", "api_base": "http://upstream.test/v1", "api_key": "k"},
|
||||
"model_info": {"id": deployment_id},
|
||||
}
|
||||
for deployment_id in ("grok-1", "grok-2")
|
||||
],
|
||||
allowed_fails=1,
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.post("http://upstream.test/v1/chat/completions").mock(
|
||||
return_value=_upstream_reply(stream=False)
|
||||
)
|
||||
for _ in range(3):
|
||||
with pytest.raises(litellm.APIResponseValidationError):
|
||||
await router.agenerate_content(
|
||||
model="grok-group",
|
||||
contents=[{"role": "user", "parts": [{"text": "a forbidden question"}]}],
|
||||
)
|
||||
await router.agenerate_content(model="grok-group", contents=[{"role": "user", "parts": [{"text": "hello"}]}])
|
||||
|
||||
assert route.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entrypoint", ["agenerate_content", "agenerate_content_stream"])
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue