mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(decisions): keep main's answer when a blank request loses its image to the declared type check, and count null as no text
This commit is contained in:
parent
e92efed078
commit
96a80073eb
5 changed files with 88 additions and 8 deletions
|
|
@ -708,9 +708,10 @@ def _checked_image_types(call: DecisionsCall) -> DecisionsCall:
|
|||
error: Final = _media_type_error(media_types, input_images(call.ir_request.input))
|
||||
if media_types is None or error is None:
|
||||
return call
|
||||
return _dropped_or_refused(
|
||||
call, with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types)), error
|
||||
)
|
||||
kept_input: Final = with_images_where(call.ir_request.input, partial(_has_media_type_in, media_types))
|
||||
if _drops_params(call.litellm_params) and not _leaves_no_input(kept_input):
|
||||
return replace(call, ir_request=replace(call.ir_request, input=kept_input))
|
||||
raise _refusal(call, error)
|
||||
|
||||
|
||||
def _refused_over_count(call: DecisionsCall) -> DecisionsCall:
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from pydantic_core import ErrorDetails
|
|||
from typing_extensions import TypeIs, assert_never # noqa: TID251 # TypeIs reaches typing only on 3.13
|
||||
|
||||
import litellm
|
||||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
|
@ -255,13 +256,17 @@ def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: only
|
|||
return isinstance(value, Sequence)
|
||||
|
||||
|
||||
def _holds_no_text(value: object) -> bool:
|
||||
def _holds_no_text(value: object, depth: int = 0) -> bool:
|
||||
if value is None:
|
||||
return True
|
||||
if isinstance(value, str):
|
||||
return not value.strip()
|
||||
if depth >= DEFAULT_MAX_RECURSE_DEPTH:
|
||||
return False
|
||||
if _is_json_object(value):
|
||||
return all(map(_holds_no_text, value.values()))
|
||||
return all(_holds_no_text(item, depth + 1) for name, item in value.items() if name != "type")
|
||||
if _is_json_array(value):
|
||||
return all(map(_holds_no_text, value))
|
||||
return all(_holds_no_text(item, depth + 1) for item in value)
|
||||
return False
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ IGNORE_FUNCTIONS = [
|
|||
"_convert_to_json_serializable_dict", # max depth set (default 20) and circular reference protection to prevent infinite recursion.
|
||||
"dict", # max depth set. _LiteLLMParamsDictView.dict() calls builtin dict(), not itself.
|
||||
"_read_image_bytes", # max depth set.
|
||||
"_holds_no_text", # max depth set.
|
||||
"get_masked_values", # max depth set (default 20) to prevent infinite recursion while masking nested sensitive config dicts.
|
||||
"_redact_sensitive_litellm_params", # max depth set (default 10).
|
||||
"_redact_secret_values_in_obj", # max depth set (default 10, _REDACT_SECRET_MAX_DEPTH); fails closed by returning "REDACTED" at the cap.
|
||||
|
|
|
|||
|
|
@ -4807,8 +4807,19 @@ _NOT_AN_IMAGE_REQUEST: Final[Mapping[str, object]] = MappingProxyType(
|
|||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize(
|
||||
"state",
|
||||
("", [], {}, " ", [""], {"a": ""}, {"a": [" "]}),
|
||||
ids=("string", "list", "object", "blank", "list-of-empty", "object-of-empty", "nested-blank"),
|
||||
("", [], {}, " ", [""], {"a": ""}, {"a": [" "]}, [None], {"a": None}, [{"type": "input_text", "text": ""}]),
|
||||
ids=(
|
||||
"string",
|
||||
"list",
|
||||
"object",
|
||||
"blank",
|
||||
"list-of-empty",
|
||||
"object-of-empty",
|
||||
"nested-blank",
|
||||
"list-of-null",
|
||||
"object-of-null",
|
||||
"empty-text-part",
|
||||
),
|
||||
)
|
||||
async def test_an_empty_state_with_only_a_dropped_image_is_refused_when_the_model_always_took_images(
|
||||
state: object, respx_mock: respx.MockRouter
|
||||
|
|
@ -4836,6 +4847,27 @@ async def test_a_state_with_something_in_it_still_reaches_the_model_after_its_on
|
|||
assert sent.get("images", []) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
@pytest.mark.parametrize("state", ("", [], {}, " ", [""], {"a": ""}, [None]), ids=str)
|
||||
async def test_a_blank_state_still_reaches_the_model_when_its_only_image_drops_for_its_declared_type(
|
||||
state: object, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
await litellm.asystemone(
|
||||
model="cloudflare/clef",
|
||||
state=state,
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=[{"content_type": "image/gif", "base64": "R0lGODlh"}],
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert sent["state"] == state
|
||||
assert sent.get("images", []) == []
|
||||
|
||||
|
||||
_NOT_AN_IMAGE_DATA_URL: Final = f"data:image/png;base64,{_NOT_AN_IMAGE_BASE64}"
|
||||
_DECISIONS_PREDICATE: Final[list[dict[str, str]]] = [{"type": "predicate", "name": "red", "instructions": "Is it red?"}]
|
||||
|
||||
|
|
@ -4871,6 +4903,33 @@ async def test_decisions_input_with_no_text_and_only_a_dropped_image_is_refused(
|
|||
assert route.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
async def test_decisions_input_with_no_text_still_reaches_the_model_when_its_image_drops_for_its_declared_type(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(f"{_CLOUDFLARE_RUN_URL}/clef").respond(json=_RESPONSE)
|
||||
|
||||
await litellm.adecisions(
|
||||
model="cloudflare/clef",
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": ""},
|
||||
{"type": "input_image", "image_url": "data:image/gif;base64,R0lGODlh"},
|
||||
],
|
||||
}
|
||||
],
|
||||
questions=_DECISIONS_PREDICATE,
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert sent["state"] == ""
|
||||
assert sent.get("images", []) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("cloudflare_account")
|
||||
async def test_decisions_input_with_text_still_reaches_the_model_after_its_only_image_is_dropped(
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import json
|
||||
import sys
|
||||
from collections.abc import Mapping
|
||||
from functools import reduce
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -8,6 +10,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.decisions.transformation import (
|
||||
input_has_no_text,
|
||||
ir_to_systemone_request,
|
||||
ir_to_systemone_response,
|
||||
parse_systemone_response,
|
||||
|
|
@ -19,6 +22,7 @@ from litellm.types.decisions import (
|
|||
DecisionsIRRefusal,
|
||||
DecisionsIRRequest,
|
||||
DecisionsIRScoreAnswer,
|
||||
DecisionsIRState,
|
||||
DecisionsRequestBody,
|
||||
OpenAIDecisionRequestBody,
|
||||
UnsupportedDecisionsRequest,
|
||||
|
|
@ -419,3 +423,13 @@ def test_non_json_provider_response_is_an_api_error(raw_content: bytes, provider
|
|||
with pytest.raises(BaseLLMException, match="unexpected response") as error:
|
||||
config.transform_decisions_response("drex-v2.0", provider, httpx.Response(200, content=raw_content), request)
|
||||
assert error.value.status_code == 500
|
||||
|
||||
|
||||
def _nested_once(inner: object, _: int) -> object:
|
||||
return [inner]
|
||||
|
||||
|
||||
def test_input_has_no_text_stops_walking_a_state_nested_past_the_stack() -> None:
|
||||
deepest_state: Final = reduce(_nested_once, range(sys.getrecursionlimit() * 2), "")
|
||||
|
||||
assert input_has_no_text(DecisionsIRState(state=deepest_state, images=())) is False
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue