mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(google_genai): run pre-call rules on adapter requests
The adapter shares the request's logging object with its inner chat call, so that call skips the setup where litellm.pre_call_rules run. Run the rules on the translated messages in the adapter instead, so a rule can still reject a generateContent request before the upstream call.
This commit is contained in:
parent
94539aa5e6
commit
dd07c4abd9
2 changed files with 39 additions and 7 deletions
|
|
@ -1,10 +1,11 @@
|
|||
from collections.abc import AsyncIterator, Coroutine, Mapping
|
||||
from collections.abc import AsyncIterator, Coroutine, Iterable, Mapping
|
||||
from typing import Final, cast
|
||||
|
||||
from openai.types.chat import ChatCompletionStreamOptionsParam
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.rules import Rules
|
||||
from litellm.types.google_genai.adapters import GenerateContentCompletionKwargs
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
|
@ -27,6 +28,10 @@ _FORWARDED_REQUEST_KEYS: Final = (
|
|||
)
|
||||
|
||||
|
||||
def _message_text(messages: Iterable[Mapping[str, object]]) -> str:
|
||||
return "".join(content for message in messages if isinstance(content := message.get("content"), str))
|
||||
|
||||
|
||||
def _adapter_stream_options(requested: object) -> ChatCompletionStreamOptionsParam | None:
|
||||
match requested:
|
||||
case {"include_usage": True}:
|
||||
|
|
@ -65,14 +70,21 @@ class GenerateContentToCompletionHandler:
|
|||
logging_obj.update_messages(completion_request["messages"])
|
||||
if stream and isinstance(logging_obj, LiteLLMLoggingObj):
|
||||
logging_obj.stream_options = stream_options
|
||||
completion_kwargs: Final = dict(
|
||||
completion_request,
|
||||
**{key: request_kwargs[key] for key in _FORWARDED_REQUEST_KEYS if key in request_kwargs},
|
||||
**({"stream": True} if stream else {}),
|
||||
**({"stream_options": stream_options} if stream_options is not None else {}),
|
||||
completion_kwargs: Final = GenerateContentCompletionKwargs(
|
||||
**dict(
|
||||
completion_request,
|
||||
**{key: request_kwargs[key] for key in _FORWARDED_REQUEST_KEYS if key in request_kwargs},
|
||||
**({"stream": True} if stream else {}),
|
||||
**({"stream_options": stream_options} if stream_options is not None else {}),
|
||||
)
|
||||
)
|
||||
if isinstance(logging_obj, LiteLLMLoggingObj) and Rules.has_pre_call_rules():
|
||||
Rules().pre_call_rules(
|
||||
input=_message_text(completion_kwargs.get("messages", [])),
|
||||
model=model,
|
||||
)
|
||||
|
||||
return GenerateContentCompletionKwargs(**completion_kwargs)
|
||||
return completion_kwargs
|
||||
|
||||
@staticmethod
|
||||
async def async_generate_content_handler(
|
||||
|
|
|
|||
|
|
@ -853,6 +853,26 @@ async def test_generate_content_adapter_keeps_the_request_no_log_and_session_id(
|
|||
assert logged.session_ids == ["adapter-session"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entrypoint", ["agenerate_content", "agenerate_content_stream"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_content_adapter_pre_call_rules_reject_the_request_before_the_upstream_call(
|
||||
_httpx_transport: None, monkeypatch: pytest.MonkeyPatch, entrypoint: str
|
||||
):
|
||||
monkeypatch.setattr(litellm, "pre_call_rules", [lambda text: "forbidden" not in text])
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.post("http://upstream.test/v1/chat/completions").mock(
|
||||
return_value=_upstream_reply(stream=entrypoint == "agenerate_content_stream")
|
||||
)
|
||||
with pytest.raises(litellm.APIResponseValidationError):
|
||||
await getattr(_adapter_router(), entrypoint)(
|
||||
model="grok-group",
|
||||
contents=[{"role": "user", "parts": [{"text": "a forbidden question"}]}],
|
||||
)
|
||||
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entrypoint", ["agenerate_content", "agenerate_content_stream"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_content_adapter_failure_is_logged_once(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue