fix(google_genai): keep pre-call rule rejections out of failure logging

A request a pre-call rule rejects never reaches a deployment, so the
adapter now marks its failure as logged, the same as a rejected chat
call. It also skips the rules for an empty conversation, as chat's
setup does. On this line the Router cools a deployment down from that
failure callback, so logging the rejection took healthy deployments
out of rotation.
This commit is contained in:
mateo-berri 2026-11-07 04:53:53 -08:00
parent dd07c4abd9
commit 73729f703c
2 changed files with 58 additions and 10 deletions

View file

@ -32,6 +32,15 @@ def _message_text(messages: Iterable[Mapping[str, object]]) -> str:
return "".join(content for message in messages if isinstance(content := message.get("content"), str))
def _run_pre_call_rules(logging_obj: LiteLLMLoggingObj, messages: Iterable[Mapping[str, object]], model: str) -> None:
try:
Rules().pre_call_rules(input=_message_text(messages), model=model)
except Exception:
logging_obj.mark_logging_complete(event_type="sync_failure")
logging_obj.mark_logging_complete(event_type="async_failure")
raise
def _adapter_stream_options(requested: object) -> ChatCompletionStreamOptionsParam | None:
match requested:
case {"include_usage": True}:
@ -78,11 +87,9 @@ class GenerateContentToCompletionHandler:
**({"stream_options": stream_options} if stream_options is not None else {}),
)
)
if isinstance(logging_obj, LiteLLMLoggingObj) and Rules.has_pre_call_rules():
Rules().pre_call_rules(
input=_message_text(completion_kwargs.get("messages", [])),
model=model,
)
messages: Final = completion_kwargs.get("messages", ())
if isinstance(logging_obj, LiteLLMLoggingObj) and messages and Rules.has_pre_call_rules():
_run_pre_call_rules(logging_obj, messages, model)
return completion_kwargs

View file

@ -858,19 +858,60 @@ async def test_generate_content_adapter_keeps_the_request_no_log_and_session_id(
async def test_generate_content_adapter_pre_call_rules_reject_the_request_before_the_upstream_call(
_httpx_transport: None, monkeypatch: pytest.MonkeyPatch, entrypoint: str
):
logged: Final = _LoggedEvents()
monkeypatch.setattr(litellm, "callbacks", [logged])
monkeypatch.setattr(litellm, "pre_call_rules", [lambda text: "forbidden" not in text])
generate: Final = getattr(_adapter_router(), entrypoint)
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.post("http://upstream.test/v1/chat/completions").mock(
return_value=_upstream_reply(stream=entrypoint == "agenerate_content_stream")
)
with pytest.raises(litellm.APIResponseValidationError):
await getattr(_adapter_router(), entrypoint)(
model="grok-group",
contents=[{"role": "user", "parts": [{"text": "a forbidden question"}]}],
)
await generate(model="grok-group", contents=[{"role": "user", "parts": [{"text": "a forbidden question"}]}])
allowed: Final = await generate(
model="grok-group",
contents=[{"role": "user", "parts": [{"text": "hello"}]}],
litellm_call_id=_ADAPTER_CALL_ID,
)
if entrypoint == "agenerate_content_stream":
await _drain(allowed)
await await_until(lambda: bool(logged.events), message="the allowed request was logged")
assert route.call_count == 0
assert route.call_count == 1
assert [(outcome, call_id) for outcome, _, call_id, _ in logged.events] == [("success", _ADAPTER_CALL_ID)]
@pytest.mark.asyncio
async def test_generate_content_adapter_pre_call_rule_rejections_leave_the_deployments_available(
_httpx_transport: None, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setattr(litellm, "pre_call_rules", [lambda text: "forbidden" not in text])
router: Final = litellm.Router(
model_list=[
{
"model_name": "grok-group",
"litellm_params": {"model": "hosted_vllm/grok", "api_base": "http://upstream.test/v1", "api_key": "k"},
"model_info": {"id": deployment_id},
}
for deployment_id in ("grok-1", "grok-2")
],
allowed_fails=1,
)
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.post("http://upstream.test/v1/chat/completions").mock(
return_value=_upstream_reply(stream=False)
)
for _ in range(3):
with pytest.raises(litellm.APIResponseValidationError):
await router.agenerate_content(
model="grok-group",
contents=[{"role": "user", "parts": [{"text": "a forbidden question"}]}],
)
await router.agenerate_content(model="grok-group", contents=[{"role": "user", "parts": [{"text": "hello"}]}])
assert route.call_count == 1
@pytest.mark.parametrize("entrypoint", ["agenerate_content", "agenerate_content_stream"])