mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-11 08:38:24 +00:00
fix(decisions)!: honor a deployment's own supports_audio_input and supports_video_input flags (#49207)
* fix(decisions): honor a deployment's own supports_audio_input and supports_video_input flags A Decisions deployment's model_info flags for audio and video now win over the cost map, per media kind, the same way supports_box_questions already does. An unlisted model flagged true gets its clips sent upstream, and a false flag refuses them (or drops them under drop_params). OpenAI-format providers still refuse clips since their request shape has no field for them, and the SystemOne request builder no longer re-reads the cost map. * fix(decisions): ignore media flags on a deployment whose own mode is not decisions A chat-mode deployment's supports_audio_input describes chat input, so it no longer turns on audio or video clips for Decisions. The cost map decides for it, as before. Adds a proxy integration test for flagged, partly flagged, unflagged, and drop_params deployments on both /v1/decisions and /v1/systemone, and for caller-sent model_info. * fix(decisions): follow the cost map's mode for a flagged deployment without its own mode A chat deployment flagged for audio or video usually leaves mode unset, so its flags described chat input yet still let clips through on /v1/decisions. When the deployment sets no mode, the cost map's mode for the model now decides whether its media flags apply, and an unlisted model still takes them. The audio and video lookups also run only when the request carries a clip. * fix(decisions): read the mode a flagged deployment falls back to from the shipped cost map The router copies each deployment's model_info into the model's shared cost map row, so another deployment's mode there could switch off a flagged deployment's media flags. The fallback now reads the cost map as loaded, untouched by router registrations * fix(decisions): route clips in a load-balanced group to a deployment that takes them A Decisions or SystemOne request carrying audio or video now skips group deployments whose own flags or the cost map refuse that media, and keeps the whole group when none takes it, so the 400 still names the media. * fix(decisions): keep deployments that drop or cannot resolve clips in the media filter * fix(router): skip clip-refusing Decisions deployments after cooldown and tag filtering Running the media filter before cooldown turned the 400 a group answered when its only clip-taking deployment was cooling down into a 429. It now runs next to the image filter, just before order filtering, in both the async and sync selection paths. * fix(router): honor router default drop_params when skipping clip-refusing Decisions deployments * test(router): list the router default params helper in the router coverage check * test(bedrock): stop reloading the cost map module in the GovCloud fixture --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
mika-lt
GitHub
parent
3fc7969e8b
commit
c4b01b2ac8
9 changed files with 720 additions and 91 deletions
|
|
@ -3,10 +3,10 @@ from collections.abc import Mapping, Sequence
|
|||
from dataclasses import dataclass, field, replace
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeAlias
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, BeforeValidator, ConfigDict, TypeAdapter, ValidationError
|
||||
from pydantic_core import to_jsonable_python
|
||||
from typing_extensions import assert_never
|
||||
|
||||
|
|
@ -23,6 +23,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
|||
from litellm.llms.base_llm.decisions.transformation import (
|
||||
BaseDecisionsConfig,
|
||||
DecisionsResponseValidationError,
|
||||
catalog_mode,
|
||||
image_media_type,
|
||||
image_param,
|
||||
input_images,
|
||||
|
|
@ -61,7 +62,7 @@ from litellm.types.decisions import (
|
|||
UnsupportedDecisionsRequest,
|
||||
)
|
||||
from litellm.types.llms.openai import ChatCompletionUserMessage
|
||||
from litellm.types.utils import LlmProviders, ProviderSpecificHeader
|
||||
from litellm.types.utils import LlmProviders, ProviderSpecificHeader, is_decisions_model_mode
|
||||
from litellm.utils import ProviderConfigManager, get_model_cost_key
|
||||
|
||||
SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
|
||||
|
|
@ -257,19 +258,105 @@ def _request_safety_identifier(
|
|||
return None
|
||||
|
||||
|
||||
def _fields_without_unsupported_media(
|
||||
fields: RequestFields,
|
||||
def _bool_or_none(value: object) -> bool | None:
|
||||
return value if isinstance(value, bool) else None
|
||||
|
||||
|
||||
def _str_or_none(value: object) -> str | None:
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
_DeploymentFlag: TypeAlias = Annotated[bool | None, BeforeValidator(_bool_or_none)]
|
||||
|
||||
|
||||
class _DeploymentCapabilities(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, from_attributes=True)
|
||||
|
||||
mode: Annotated[str | None, BeforeValidator(_str_or_none)] = None
|
||||
supports_box_questions: _DeploymentFlag = None
|
||||
supports_audio_input: _DeploymentFlag = None
|
||||
supports_video_input: _DeploymentFlag = None
|
||||
|
||||
def decisions_media_flag(self, flag: bool | None, model: str, custom_llm_provider: str) -> bool | None:
|
||||
if flag is None:
|
||||
return None
|
||||
mode: Final = self.mode if self.mode is not None else catalog_mode(model, custom_llm_provider)
|
||||
return flag if mode is None or is_decisions_model_mode(mode) else None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ResolvedSupport:
|
||||
box_questions: bool
|
||||
audio: bool
|
||||
videos: bool
|
||||
|
||||
|
||||
def _takes_audio(
|
||||
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
|
||||
) -> bool:
|
||||
return provider_config.supports_audio_input(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider),
|
||||
)
|
||||
|
||||
|
||||
def _takes_videos(
|
||||
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
|
||||
) -> bool:
|
||||
return provider_config.supports_video_input(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider),
|
||||
)
|
||||
|
||||
|
||||
def deployment_refuses_media(
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
model_info: Mapping[str, object],
|
||||
call_kwargs: Mapping[str, object],
|
||||
*,
|
||||
audio: bool,
|
||||
videos: bool,
|
||||
) -> bool:
|
||||
if _drops_params(call_kwargs):
|
||||
return False
|
||||
provider_config: Final = get_decisions_provider_config(model, custom_llm_provider)
|
||||
canonical_model: Final = provider_config.canonical_model(model)
|
||||
deployment: Final = _DeploymentCapabilities.model_validate(model_info)
|
||||
return (audio and not _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) or (
|
||||
videos and not _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment)
|
||||
)
|
||||
|
||||
|
||||
def _resolved_support(
|
||||
provider_config: BaseDecisionsConfig,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
headers: Mapping[str, str],
|
||||
kwargs: Mapping[str, object],
|
||||
fields: RequestFields,
|
||||
) -> _ResolvedSupport:
|
||||
deployment: Final = _DeploymentCapabilities.model_validate(kwargs.get("model_info") or {})
|
||||
return _ResolvedSupport(
|
||||
box_questions=provider_config.supports_box_questions(
|
||||
model, custom_llm_provider, headers, deployment.supports_box_questions
|
||||
),
|
||||
audio=bool(fields.audio) and _takes_audio(provider_config, model, custom_llm_provider, deployment),
|
||||
videos=bool(fields.videos) and _takes_videos(provider_config, model, custom_llm_provider, deployment),
|
||||
)
|
||||
|
||||
|
||||
def _fields_without_unsupported_media(
|
||||
fields: RequestFields, support: _ResolvedSupport, kwargs: Mapping[str, object]
|
||||
) -> RequestFields:
|
||||
if not _drops_params(kwargs):
|
||||
return fields
|
||||
return replace(
|
||||
fields,
|
||||
audio=fields.audio if provider_config.supports_audio_input(model, custom_llm_provider) else None,
|
||||
videos=fields.videos if provider_config.supports_video_input(model, custom_llm_provider) else None,
|
||||
audio=fields.audio if support.audio else None,
|
||||
videos=fields.videos if support.videos else None,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -292,10 +379,8 @@ class _MalformedImages:
|
|||
def _unsupported_params(
|
||||
ir_request: DecisionsIRRequest,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
headers: Mapping[str, str],
|
||||
supports_box_questions: bool,
|
||||
support: _ResolvedSupport,
|
||||
) -> tuple[str, ...]:
|
||||
refuses_safety_identifier: Final = (
|
||||
ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier
|
||||
|
|
@ -304,13 +389,9 @@ def _unsupported_params(
|
|||
None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input)
|
||||
)
|
||||
refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers)
|
||||
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions
|
||||
refuses_audio: Final = bool(ir_request.audio) and not provider_config.supports_audio_input(
|
||||
model, custom_llm_provider
|
||||
)
|
||||
refuses_videos: Final = bool(ir_request.videos) and not provider_config.supports_video_input(
|
||||
model, custom_llm_provider
|
||||
)
|
||||
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not support.box_questions
|
||||
refuses_audio: Final = bool(ir_request.audio) and not support.audio
|
||||
refuses_videos: Final = bool(ir_request.videos) and not support.videos
|
||||
return (
|
||||
*(("safety_identifier",) if refuses_safety_identifier else ()),
|
||||
*(() if refused_image_param is None else (refused_image_param,)),
|
||||
|
|
@ -333,7 +414,7 @@ def _without_unsupported_params(
|
|||
ir_request: DecisionsIRRequest,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
headers: Mapping[str, str],
|
||||
supports_box_questions: bool,
|
||||
support: _ResolvedSupport,
|
||||
) -> DecisionsIRRequest:
|
||||
return replace(
|
||||
ir_request,
|
||||
|
|
@ -345,9 +426,7 @@ def _without_unsupported_params(
|
|||
safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None,
|
||||
assets=ir_request.assets if provider_config.supports_assets(headers) else MappingProxyType({}),
|
||||
questions=(
|
||||
ir_request.questions
|
||||
if supports_box_questions
|
||||
else tuple(filter(_is_not_box_question, ir_request.questions))
|
||||
ir_request.questions if support.box_questions else tuple(filter(_is_not_box_question, ir_request.questions))
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -368,39 +447,20 @@ def _without_malformed_images(
|
|||
return well_formed
|
||||
|
||||
|
||||
class _DeploymentBoxQuestions(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, from_attributes=True)
|
||||
|
||||
supports_box_questions: StrictBool | None = None
|
||||
|
||||
|
||||
def _deployment_supports_box_questions(kwargs: Mapping[str, object]) -> bool | None:
|
||||
try:
|
||||
return _DeploymentBoxQuestions.model_validate(kwargs.get("model_info") or {}).supports_box_questions
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _provider_ir_request(
|
||||
ir_request: DecisionsIRRequest,
|
||||
*,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
headers: Mapping[str, str],
|
||||
support: _ResolvedSupport,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> DecisionsIRRequest | _UnsupportedParams | _NothingLeftAfterDrop:
|
||||
supports_box_questions: Final = provider_config.supports_box_questions(
|
||||
model, custom_llm_provider, headers, _deployment_supports_box_questions(kwargs)
|
||||
)
|
||||
unsupported: Final = _unsupported_params(
|
||||
ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions
|
||||
)
|
||||
unsupported: Final = _unsupported_params(ir_request, provider_config, headers, support)
|
||||
if not unsupported:
|
||||
return ir_request
|
||||
if not _drops_params(kwargs):
|
||||
return _UnsupportedParams(names=unsupported)
|
||||
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions)
|
||||
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, support)
|
||||
if _leaves_no_input(supported.input):
|
||||
return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on")
|
||||
if not supported.questions:
|
||||
|
|
@ -479,10 +539,11 @@ def prepare_call(
|
|||
raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
|
||||
try:
|
||||
headers: Final = decisions_outbound_headers(extra_headers, provider, kwargs)
|
||||
support: Final = _resolved_support(provider_config, canonical_model, provider, headers, kwargs, fields)
|
||||
request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs)
|
||||
request: Final = _fields_without_unsupported_media(
|
||||
fields, provider_config, canonical_model, provider, kwargs
|
||||
).validate(safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs))
|
||||
request: Final = _fields_without_unsupported_media(fields, support, kwargs).validate(
|
||||
safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs)
|
||||
)
|
||||
except ValidationError as error:
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Invalid Decisions request: {error}",
|
||||
|
|
@ -523,12 +584,7 @@ def prepare_call(
|
|||
llm_provider=provider,
|
||||
)
|
||||
ir_request: Final = _provider_ir_request(
|
||||
caller_ir_request,
|
||||
provider_config=provider_config,
|
||||
model=canonical_model,
|
||||
custom_llm_provider=provider,
|
||||
headers=headers,
|
||||
kwargs=kwargs,
|
||||
caller_ir_request, provider_config=provider_config, headers=headers, support=support, kwargs=kwargs
|
||||
)
|
||||
if isinstance(ir_request, _UnsupportedParams):
|
||||
raise litellm.UnsupportedParamsError(
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from pydantic_core import ErrorDetails
|
|||
from typing_extensions import assert_never
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
|
@ -297,13 +298,7 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]:
|
|||
assert_never(question)
|
||||
|
||||
|
||||
def ir_to_systemone_request(
|
||||
model: str, request: DecisionsIRRequest, *, supports_audio: bool = False, supports_video: bool = False
|
||||
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
|
||||
if request.audio and not supports_audio:
|
||||
return UnsupportedDecisionsRequest(reason="audio input is not supported by this Decisions model")
|
||||
if request.videos and not supports_video:
|
||||
return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model")
|
||||
def ir_to_systemone_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object]:
|
||||
state: Final = _systemone_state(request.input)
|
||||
keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True)
|
||||
images: Final = input_images(request.input)
|
||||
|
|
@ -503,6 +498,16 @@ def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIR
|
|||
)
|
||||
|
||||
|
||||
def catalog_mode(model: str, custom_llm_provider: str) -> str | None:
|
||||
try:
|
||||
key: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)["key"]
|
||||
except Exception: # noqa: BLE001 # get_model_info raises bare Exception for an unmapped model, which the catalog does not list
|
||||
return None
|
||||
entry: Final = GetModelCostMap.loaded_model_cost_map().get(key)
|
||||
mode: Final = entry.get("mode") if entry is not None else None
|
||||
return mode if isinstance(mode, str) else None
|
||||
|
||||
|
||||
def _is_decisions_model(model: str, custom_llm_provider: str) -> bool:
|
||||
try:
|
||||
model_mode: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider).get("mode")
|
||||
|
|
@ -640,12 +645,16 @@ class BaseDecisionsConfig(ABC):
|
|||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
|
||||
if deployment_flag is not None:
|
||||
return deployment_flag
|
||||
return _is_decisions_model(model, custom_llm_provider) and litellm.supports_audio_input(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
|
||||
if deployment_flag is not None:
|
||||
return deployment_flag
|
||||
return _is_decisions_model(model, custom_llm_provider) and litellm.supports_video_input(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
|
@ -656,12 +665,7 @@ class BaseDecisionsConfig(ABC):
|
|||
request: DecisionsIRRequest,
|
||||
custom_llm_provider: str,
|
||||
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
|
||||
return ir_to_systemone_request(
|
||||
self.request_model(model),
|
||||
request,
|
||||
supports_audio=self.supports_audio_input(model, custom_llm_provider),
|
||||
supports_video=self.supports_video_input(model, custom_llm_provider),
|
||||
)
|
||||
return ir_to_systemone_request(self.request_model(model), request)
|
||||
|
||||
def unwrap_response(self, payload: object) -> object:
|
||||
return payload
|
||||
|
|
|
|||
|
|
@ -354,10 +354,10 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig):
|
|||
) -> str | None:
|
||||
return resolve_openai_bearer_token(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
|
||||
|
||||
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
|
||||
return False
|
||||
|
||||
def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
|
||||
return False
|
||||
|
||||
def transform_decisions_request(
|
||||
|
|
|
|||
|
|
@ -635,6 +635,7 @@ _PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"})
|
|||
_STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"})
|
||||
_video_call_deployment_id: Final[ContextVar[str | None]] = ContextVar("video_call_deployment_id", default=None)
|
||||
_routed_call_type: Final[ContextVar[str | None]] = ContextVar("routed_call_type", default=None)
|
||||
_DECISIONS_MEDIA_CALL_TYPES: Final = frozenset({"adecisions", "decisions", "asystemone", "systemone"})
|
||||
_ROUTE_CAPABILITY_RESOLVERS: Final[Mapping[str, Callable[[str, LlmProviders], object | None]]] = MappingProxyType(
|
||||
{
|
||||
"adecisions": ProviderConfigManager.get_provider_decisions_config,
|
||||
|
|
@ -4153,13 +4154,16 @@ class Router(DeprecatedRouterMembers):
|
|||
raise e.original_exception from e
|
||||
raise e
|
||||
|
||||
def _active_default_litellm_params(self) -> Mapping[str, object]:
|
||||
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params)
|
||||
return {key: value for key, value in default_params.items() if value is not None}
|
||||
|
||||
def _params_sent_to_deployment(
|
||||
self, request_kwargs: Mapping[str, object], deployment: Deployment | None
|
||||
) -> Mapping[str, object]:
|
||||
"""The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies."""
|
||||
deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {}
|
||||
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params)
|
||||
defaults: Final = {key: value for key, value in default_params.items() if value is not None}
|
||||
defaults: Final = self._active_default_litellm_params()
|
||||
deployment_tool_choice: Final = (
|
||||
{"tool_choice": deployment_params["tool_choice"]}
|
||||
if deployment_params.get("tool_choice") is not None
|
||||
|
|
@ -14911,6 +14915,62 @@ class Router(DeprecatedRouterMembers):
|
|||
return healthy_deployments
|
||||
return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates
|
||||
|
||||
@staticmethod
|
||||
def _decisions_deployment_refuses_media(
|
||||
deployment: DeploymentTypedDict, group: str, request_kwargs: Mapping[str, object], *, audio: bool, videos: bool
|
||||
) -> bool:
|
||||
from litellm.decisions.call import deployment_refuses_media
|
||||
|
||||
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
|
||||
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
|
||||
deployment_params: Final = cast(Mapping[str, object], deployment["litellm_params"]) # cast-ok: plain dict
|
||||
try:
|
||||
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
|
||||
return deployment_refuses_media(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
model_info,
|
||||
{**deployment_params, **request_kwargs},
|
||||
audio=audio,
|
||||
videos=videos,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # best-effort filter: a deployment it cannot resolve answers for itself
|
||||
verbose_router_logger.debug(
|
||||
"litellm.router.py::_decisions_deployment_refuses_media: keeping a deployment of model=%s. Got - %s",
|
||||
group,
|
||||
e,
|
||||
)
|
||||
return False
|
||||
|
||||
def _without_decisions_deployments_refusing_media(
|
||||
self, healthy_deployments: _RouteDeploymentsT, group: str, request_kwargs: Mapping[str, object] | None
|
||||
) -> _RouteDeploymentsT:
|
||||
call_kwargs: Final = request_kwargs or {}
|
||||
audio: Final = bool(call_kwargs.get("audio"))
|
||||
videos: Final = bool(call_kwargs.get("videos"))
|
||||
if (
|
||||
_routed_call_type.get() not in _DECISIONS_MEDIA_CALL_TYPES
|
||||
or not (audio or videos)
|
||||
or isinstance(healthy_deployments, Mapping)
|
||||
):
|
||||
return healthy_deployments
|
||||
|
||||
sent_kwargs: Final = {**self._active_default_litellm_params(), **call_kwargs}
|
||||
deployments_taking_media: Final = [
|
||||
deployment
|
||||
for deployment in healthy_deployments
|
||||
if not Router._decisions_deployment_refuses_media(
|
||||
cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape
|
||||
group,
|
||||
sent_kwargs,
|
||||
audio=audio,
|
||||
videos=videos,
|
||||
)
|
||||
]
|
||||
if not deployments_taking_media:
|
||||
return healthy_deployments
|
||||
return cast(_RouteDeploymentsT, deployments_taking_media) # cast-ok: same list shape as the input candidates
|
||||
|
||||
@staticmethod
|
||||
def _without_declared_text_only_deployments(
|
||||
healthy_deployments: _RouteDeploymentsT,
|
||||
|
|
@ -15635,10 +15695,14 @@ class Router(DeprecatedRouterMembers):
|
|||
|
||||
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
|
||||
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
|
||||
batch_candidates: Final = self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
batch_candidates: Final = self._without_decisions_deployments_refusing_media(
|
||||
self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
),
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
)
|
||||
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(
|
||||
|
|
@ -16878,10 +16942,14 @@ class Router(DeprecatedRouterMembers):
|
|||
|
||||
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
|
||||
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
|
||||
batch_candidates: Final = self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
batch_candidates: Final = self._without_decisions_deployments_refusing_media(
|
||||
self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
),
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
)
|
||||
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(
|
||||
|
|
|
|||
|
|
@ -183,6 +183,9 @@ ignored_function_names = [
|
|||
"_aresponses_background_stream_can_release", # Tested through the background Responses stream tests in test_router.py
|
||||
"_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py
|
||||
"_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests
|
||||
"_without_decisions_deployments_refusing_media", # Tested through the mixed media-flag group tests in tests/unit/decisions/test_main.py
|
||||
"_decisions_deployment_refuses_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests
|
||||
"_active_default_litellm_params", # Tested through the router default drop_params test in tests/unit/decisions/test_main.py
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,123 @@
|
|||
import json
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import Gateway, eventually, object_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_API_KEY: Final = "synthetic-decisions-key"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_INPUT: Final = "Listen to the attached clip."
|
||||
_AUDIO: Final = ("data:audio/wav;base64,UklGRiQAAABXQVZF",)
|
||||
_VIDEOS: Final = ("data:video/mp4;base64,AAAAIGZ0eXBpc29t",)
|
||||
_QUESTION: Final[dict[str, JsonValue]] = {
|
||||
"type": "predicate",
|
||||
"name": "banana",
|
||||
"instructions": "Does the clip name a banana?",
|
||||
}
|
||||
_SPEND_QUERY: Final = 'SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id = %s'
|
||||
|
||||
Route = Literal["/v1/decisions", "/v1/systemone"]
|
||||
Media = Literal["audio", "videos"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Upstream:
|
||||
model: str
|
||||
|
||||
def reply(self, request: Request) -> Reply:
|
||||
questions: Final = object_value(_JSON_OBJECT.validate_json(request.body)["questions"])
|
||||
plain: Final[dict[str, JsonValue]] = {
|
||||
"model": self.model,
|
||||
"answers": {name: {"type": "noul", "noul": 0.9} for name in questions},
|
||||
"usage": {"input_tokens": 41, "output_tokens": 0},
|
||||
}
|
||||
return Reply(body=json.dumps(plain).encode())
|
||||
|
||||
|
||||
def _unlisted_openrouter_model() -> _Upstream:
|
||||
return _Upstream(f"openrouter/example/omni-{uuid.uuid4().hex[:12]}")
|
||||
|
||||
|
||||
def _decide(gateway: Gateway, model: str, route: Route, media: Mapping[str, JsonValue]) -> httpx.Response:
|
||||
body: Final[dict[str, JsonValue]] = (
|
||||
{"model": model, "input": _INPUT, "questions": [_QUESTION], "num_retries": 0, **media}
|
||||
if route == "/v1/decisions"
|
||||
else {
|
||||
"model": model,
|
||||
"state": _INPUT,
|
||||
"questions": {"banana": {"type": "noul", "instructions": _QUESTION["instructions"]}},
|
||||
"num_retries": 0,
|
||||
**media,
|
||||
}
|
||||
)
|
||||
return gateway.request("POST", route, body)
|
||||
|
||||
|
||||
def _sent_media(request: Request) -> dict[str, JsonValue]:
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
return {kind: body[kind] for kind in ("audio", "videos") if kind in body}
|
||||
|
||||
|
||||
def _assert_refused(response: httpx.Response, kinds: tuple[Media, ...]) -> None:
|
||||
assert response.status_code == 400, response.text
|
||||
assert f"does not support parameters: {list(kinds)}" in response.text, response.text
|
||||
|
||||
|
||||
_BOTH: Final[dict[str, JsonValue]] = {"audio": list(_AUDIO), "videos": list(_VIDEOS)}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ("/v1/decisions", "/v1/systemone"))
|
||||
def test_a_deployment_flagged_for_audio_and_video_sends_both_clips_for_a_model_the_cost_map_does_not_list(
|
||||
gateway: Gateway, route: Route
|
||||
) -> None:
|
||||
upstream: Final = _unlisted_openrouter_model()
|
||||
with wire_server(upstream.reply) as wire, gateway.scenario() as scenario:
|
||||
flags: Final[dict[str, JsonValue]] = {"supports_audio_input": True, "supports_video_input": True}
|
||||
flagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags)
|
||||
flagged_dropping: Final = scenario.model(
|
||||
model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags, drop_params=True
|
||||
)
|
||||
audio_only: Final = scenario.model(
|
||||
model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info={"supports_audio_input": True}
|
||||
)
|
||||
unflagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY)
|
||||
|
||||
_assert_refused(_decide(gateway, unflagged, route, _BOTH), ("audio", "videos"))
|
||||
_assert_refused(_decide(gateway, audio_only, route, _BOTH), ("videos",))
|
||||
assert wire.drain() == ()
|
||||
|
||||
answered: Final = _decide(gateway, flagged, route, _BOTH)
|
||||
assert answered.status_code == 200, answered.text
|
||||
(call,) = wire.drain()
|
||||
assert _sent_media(call) == _BOTH
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(_SPEND_QUERY, (answered.headers["x-litellm-call-id"],)),
|
||||
lambda found: len(found) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
assert (rows[0]["status"], rows[0]["model_group"]) == ("success", flagged)
|
||||
|
||||
kept: Final = _decide(gateway, flagged_dropping, route, _BOTH)
|
||||
assert kept.status_code == 200, kept.text
|
||||
(kept_call,) = wire.drain()
|
||||
assert _sent_media(kept_call) == _BOTH
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value",
|
||||
({"supports_audio_input": True, "supports_video_input": True}, 5, [{"supports_audio_input": True}], "x" * 5000),
|
||||
ids=("flags", "int", "list", "5kb"),
|
||||
)
|
||||
def test_a_caller_sent_model_info_never_turns_media_on(gateway: Gateway, value: JsonValue) -> None:
|
||||
upstream: Final = _unlisted_openrouter_model()
|
||||
with wire_server(upstream.reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY)
|
||||
_assert_refused(_decide(gateway, model, "/v1/decisions", {**_BOTH, "model_info": value}), ("audio", "videos"))
|
||||
assert wire.drain() == ()
|
||||
|
|
@ -6,10 +6,11 @@ import copy
|
|||
import json
|
||||
import re
|
||||
import threading
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from concurrent.futures import wait
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
|
@ -23,6 +24,7 @@ import litellm
|
|||
from litellm.constants import AWS_SIGNING_MAX_THREADS
|
||||
from litellm.cost_calculator import get_response_cost_from_hidden_params
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded, refetch_model_cost_map
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
|
||||
from litellm.llms.bedrock.base_aws_llm import AWS_SIGNING_EXECUTOR
|
||||
|
|
@ -3702,6 +3704,387 @@ async def test_a_model_info_object_instead_of_a_dict_still_reaches_the_provider(
|
|||
assert response.answers[0].model_dump(mode="json")["name"] == "is_defect"
|
||||
|
||||
|
||||
_AUDIO_CLIPS: Final = ("data:audio/wav;base64,YQ==",)
|
||||
_VIDEO_CLIPS: Final = ("data:video/mp4;base64,YQ==",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_deployment_that_advertises_media_sends_it_for_a_model_the_cost_map_does_not_list(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
backend: Final = "openrouter/example/omni-preview"
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "hears",
|
||||
"litellm_params": {"model": backend, "api_key": "k"},
|
||||
"model_info": {"id": "media-hears", "supports_audio_input": True},
|
||||
},
|
||||
{
|
||||
"model_name": "hears-and-sees",
|
||||
"litellm_params": {"model": backend, "api_key": "k"},
|
||||
"model_info": {"id": "media-both", "supports_audio_input": True, "supports_video_input": True},
|
||||
},
|
||||
]
|
||||
)
|
||||
send: Final = partial(
|
||||
router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['videos']")):
|
||||
await send(model="hears")
|
||||
assert not route.called
|
||||
|
||||
await send(model="hears-and-sees")
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_media_flags_on_a_deployment_whose_own_mode_is_not_decisions_never_send_clips(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
advertised: Final = {"supports_audio_input": True, "supports_video_input": True}
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": mode,
|
||||
"litellm_params": {"model": f"openrouter/example/omni-{mode}-preview", "api_key": "k"},
|
||||
"model_info": {"id": f"media-{mode}", "mode": mode, **advertised},
|
||||
}
|
||||
for mode in ("chat", "decisions")
|
||||
]
|
||||
)
|
||||
send: Final = partial(
|
||||
router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")):
|
||||
await send(model="chat")
|
||||
assert not route.called
|
||||
|
||||
await send(model="decisions")
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
|
||||
|
||||
|
||||
CatalogLoader: TypeAlias = Callable[[Mapping[str, Mapping[str, object]]], Awaitable[None]]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def load_catalog_with(monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None) -> Iterator[CatalogLoader]:
|
||||
async def load(rows: Mapping[str, Mapping[str, object]]) -> None:
|
||||
catalog: Final = {**litellm.get_model_cost_map(url=""), **rows}
|
||||
transport: Final = httpx.MockTransport(lambda _: httpx.Response(200, json=catalog))
|
||||
monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP")
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
loaded: Final = await refetch_model_cost_map(url="https://catalog.example/prices.json", client=client)
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
assert isinstance(loaded, ModelCostMapReloaded)
|
||||
monkeypatch.setattr(litellm, "model_cost", loaded.model_cost_map)
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
yield load
|
||||
litellm.get_model_cost_map(url="")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_media_flags_on_a_deployment_without_its_own_mode_follow_the_mode_the_catalog_lists(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, load_catalog_with: CatalogLoader
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
await load_catalog_with(
|
||||
{
|
||||
f"openrouter/example/omni-{mode}": {"litellm_provider": "openrouter", "mode": mode}
|
||||
for mode in ("chat", "decisions")
|
||||
}
|
||||
)
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": mode,
|
||||
"litellm_params": {"model": f"openrouter/example/omni-{mode}", "api_key": "k"},
|
||||
"model_info": {"id": f"listed-{mode}", "supports_audio_input": True, "supports_video_input": True},
|
||||
}
|
||||
for mode in ("chat", "decisions")
|
||||
]
|
||||
)
|
||||
send: Final = partial(
|
||||
router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")):
|
||||
await send(model="chat")
|
||||
assert not route.called
|
||||
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
await send(model="chat")
|
||||
dropped: Final = json.loads(route.calls.last.request.content)
|
||||
assert ("audio" in dropped, "videos" in dropped) == (False, False)
|
||||
|
||||
await send(model="decisions")
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sibling_mode", ("responses", "embedding"))
|
||||
async def test_another_deployments_mode_never_switches_off_a_deployments_own_media_flags(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, sibling_mode: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
shared_params: Final = {"model": "openrouter/example/omni-shared", "api_key": "k"}
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "sibling", "litellm_params": shared_params, "model_info": {"id": "s", "mode": sibling_mode}},
|
||||
{
|
||||
"model_name": "flagged",
|
||||
"litellm_params": shared_params,
|
||||
"model_info": {"id": "f", "supports_audio_input": True, "supports_video_input": True},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
await router.adecisions(
|
||||
model="flagged", input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
|
||||
)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
|
||||
|
||||
|
||||
_MEDIA_SURFACES: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType(
|
||||
{
|
||||
"adecisions": {"input": "review", "questions": _PREDICATE_QUESTIONS},
|
||||
"asystemone": {"state": "review", "questions": _QUESTIONS},
|
||||
"systemone": {"state": "review", "questions": _QUESTIONS},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _group_member(
|
||||
group: str, model: str, key: str, weight: int, model_info: Mapping[str, object], **params: object
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"model_name": group,
|
||||
"litellm_params": {"model": model, "api_key": key, "weight": weight, **params},
|
||||
"model_info": {"id": f"{group}-{key}", **model_info},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES))
|
||||
async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_only_when_none_does(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct")
|
||||
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"cloudflare/@cf/cloudflare/clef-omni",
|
||||
{"litellm_provider": "cloudflare", "mode": "evaluation", "supports_audio_input": True},
|
||||
)
|
||||
respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
respx_mock.post("https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-omni").respond(
|
||||
json=_RESPONSE
|
||||
)
|
||||
unlisted: Final = "openrouter/example/omni-preview"
|
||||
listed: Final = "cloudflare/clef-omni"
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
_group_member("unlisted", unlisted, "refuses", 1, {}),
|
||||
_group_member("unlisted", unlisted, "takes", 0, {"supports_audio_input": True}),
|
||||
_group_member("listed", listed, "refuses", 1, {"supports_audio_input": False}),
|
||||
_group_member("listed", listed, "takes", 0, {}),
|
||||
_group_member("none", unlisted, "refuses", 1, {}),
|
||||
_group_member("none", listed, "refuses", 1, {"supports_audio_input": False}),
|
||||
_group_member("drops", unlisted, "drops", 1, {}, drop_params=True),
|
||||
_group_member("drops", listed, "takes", 0, {}),
|
||||
_group_member("unresolvable", "databricks/not a serving endpoint", "unresolvable", 0, {}),
|
||||
_group_member("unresolvable", listed, "takes", 1, {}),
|
||||
]
|
||||
)
|
||||
call: Final = getattr(router, surface)
|
||||
|
||||
async def send(group: str, **media: object) -> object:
|
||||
response: Final = call(model=group, **_MEDIA_SURFACES[surface], **media)
|
||||
return await response if asyncio.iscoroutine(response) else response
|
||||
|
||||
for group in ("unlisted", "listed", "unresolvable"):
|
||||
await send(group, audio=_AUDIO_CLIPS)
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
|
||||
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
|
||||
|
||||
await send("drops", audio=_AUDIO_CLIPS)
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops"
|
||||
assert "audio" not in json.loads(respx_mock.calls.last.request.content)
|
||||
|
||||
await send("unlisted")
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer refuses"
|
||||
|
||||
sent_before_refusal: Final = len(respx_mock.calls)
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")):
|
||||
await send("none", audio=_AUDIO_CLIPS)
|
||||
assert len(respx_mock.calls) == sent_before_refusal
|
||||
|
||||
|
||||
def _answer_unless_key_is(rejected_key: str) -> Callable[[httpx.Request], httpx.Response]:
|
||||
def answer(request: httpx.Request) -> httpx.Response:
|
||||
if request.headers["authorization"] == f"Bearer {rejected_key}":
|
||||
return httpx.Response(401, json={"error": {"message": "invalid key"}})
|
||||
return httpx.Response(200, json=_RESPONSE)
|
||||
|
||||
return answer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clips_to_a_group_whose_only_taker_is_cooling_down_are_refused_rather_than_rate_limited(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").mock(
|
||||
side_effect=_answer_unless_key_is("takes")
|
||||
)
|
||||
unlisted: Final = "openrouter/example/omni-preview"
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
_group_member("g", unlisted, "refuses", 1, {}),
|
||||
_group_member("g", unlisted, "takes", 1, {"supports_audio_input": True}),
|
||||
],
|
||||
num_retries=0,
|
||||
cooldown_time=120,
|
||||
)
|
||||
with pytest.raises(litellm.AuthenticationError):
|
||||
await router.adecisions(model="g-takes", input="review", questions=_PREDICATE_QUESTIONS)
|
||||
sent_before_clips: Final = len(route.calls)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")):
|
||||
await router.adecisions(model="g", input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS)
|
||||
|
||||
assert len(route.calls) == sent_before_clips
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES))
|
||||
async def test_a_router_default_drop_params_overrides_a_deployment_when_choosing_who_takes_clips(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
unlisted: Final = "openrouter/example/omni-preview"
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
_group_member("g", unlisted, "drops", 1, {}, drop_params=True),
|
||||
_group_member("g", unlisted, "takes", 0, {"supports_audio_input": True}),
|
||||
],
|
||||
default_litellm_params={"drop_params": False},
|
||||
)
|
||||
call: Final = getattr(router, surface)
|
||||
|
||||
async def send(**params: object) -> object:
|
||||
response: Final = call(model="g", **_MEDIA_SURFACES[surface], audio=_AUDIO_CLIPS, **params)
|
||||
return await response if asyncio.iscoroutine(response) else response
|
||||
|
||||
await send()
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
|
||||
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
|
||||
|
||||
await send(drop_params=True)
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops"
|
||||
assert "audio" not in json.loads(respx_mock.calls.last.request.content)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_deployment_that_turns_audio_off_is_refused_or_dropped_even_when_the_cost_map_allows_it(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"typesafe/jev-omni",
|
||||
{
|
||||
"litellm_provider": "typesafe",
|
||||
"mode": "decisions",
|
||||
"supports_audio_input": True,
|
||||
"supports_video_input": True,
|
||||
},
|
||||
)
|
||||
route: Final = respx_mock.post("https://example.com/v1/systemone").respond(json=_RESPONSE)
|
||||
send: Final = partial(
|
||||
litellm.asystemone,
|
||||
model="typesafe/jev-omni",
|
||||
state="test",
|
||||
questions=_QUESTIONS,
|
||||
api_base="https://example.com",
|
||||
api_key="test",
|
||||
audio=_AUDIO_CLIPS,
|
||||
videos=_VIDEO_CLIPS,
|
||||
model_info={"supports_audio_input": False},
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio']")):
|
||||
await send()
|
||||
assert not route.called
|
||||
|
||||
await send(drop_params=True)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert "audio" not in sent
|
||||
assert sent["videos"] == list(_VIDEO_CLIPS)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_media_flag_that_is_not_a_boolean_leaves_the_cost_map_in_charge_of_that_kind_only(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://example.com/v1/systemone").respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio']")):
|
||||
await litellm.asystemone(
|
||||
model="typesafe/jev-unlisted",
|
||||
state="test",
|
||||
questions=_QUESTIONS,
|
||||
api_base="https://example.com",
|
||||
api_key="test",
|
||||
audio=_AUDIO_CLIPS,
|
||||
videos=_VIDEO_CLIPS,
|
||||
model_info={"supports_audio_input": "true", "supports_video_input": True},
|
||||
)
|
||||
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_openai_format_deployment_that_advertises_media_is_still_refused_it(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")):
|
||||
await litellm.adecisions(
|
||||
model="openai/gpt-6-luna",
|
||||
input="review",
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
api_key="deployment-key",
|
||||
audio=_AUDIO_CLIPS,
|
||||
videos=_VIDEO_CLIPS,
|
||||
model_info={"supports_audio_input": True, "supports_video_input": True},
|
||||
)
|
||||
|
||||
assert not route.called
|
||||
|
||||
|
||||
def test_dropping_every_question_answers_400_before_http(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE)
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ from litellm.types.decisions import (
|
|||
DecisionsIRScoreAnswer,
|
||||
DecisionsRequestBody,
|
||||
OpenAIDecisionRequestBody,
|
||||
UnsupportedDecisionsRequest,
|
||||
)
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
|
@ -279,7 +278,7 @@ def test_a_malformed_systemone_response_names_the_field_at_fault(
|
|||
assert [item["loc"] for item in error.value.errors()] == [field]
|
||||
|
||||
|
||||
def test_each_media_kind_is_gated_by_its_own_support_flag() -> None:
|
||||
def test_each_media_kind_is_sent_in_its_own_field() -> None:
|
||||
audio: Final = ["data:audio/wav;base64,YQ=="]
|
||||
videos: Final = ["data:video/mp4;base64,YQ=="]
|
||||
audio_only: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "audio": audio}))
|
||||
|
|
@ -287,22 +286,16 @@ def test_each_media_kind_is_gated_by_its_own_support_flag() -> None:
|
|||
_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "videos": videos})
|
||||
)
|
||||
|
||||
assert ir_to_systemone_request("clef-omni", audio_only, supports_audio=True) == {
|
||||
assert ir_to_systemone_request("clef-omni", audio_only) == {
|
||||
"model": "clef-omni",
|
||||
**_SYSTEMONE_REQUEST,
|
||||
"audio": audio,
|
||||
}
|
||||
assert ir_to_systemone_request("clef-omni", video_only, supports_video=True) == {
|
||||
assert ir_to_systemone_request("clef-omni", video_only) == {
|
||||
"model": "clef-omni",
|
||||
**_SYSTEMONE_REQUEST,
|
||||
"videos": videos,
|
||||
}
|
||||
assert isinstance(
|
||||
ir_to_systemone_request("clef-omni", audio_only, supports_video=True), UnsupportedDecisionsRequest
|
||||
)
|
||||
assert isinstance(
|
||||
ir_to_systemone_request("clef-omni", video_only, supports_audio=True), UnsupportedDecisionsRequest
|
||||
)
|
||||
|
||||
|
||||
_ASSET_REQUEST: Final[Mapping[str, object]] = {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
|
||||
import asyncio, importlib, litellm, litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map, pytest
|
||||
import asyncio, importlib, litellm, pytest
|
||||
from typing import Final
|
||||
|
||||
|
||||
|
|
@ -1563,7 +1563,6 @@ def _pr4_bedrock_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
def _use_local_model_cost_map() -> Iterator[None]:
|
||||
with pytest.MonkeyPatch.context() as monkeypatch:
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
importlib.reload(bedrock_govcloud_model_cost_map)
|
||||
importlib.reload(litellm)
|
||||
yield
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue