mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-12 08:42:18 +00:00
feat(gemini): support gemini-decision-1 on /v1/decisions and /v1/systemone (#51220)
* feat(gemini): add Decisions API support Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(gemini): add Decisions quotas and test package marker Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(gemini): bound decisions image downloads * fix(gemini): preserve image order and bounded cache reuse * test(gemini): cover cached image budgets and text messages * refactor(gemini): iterate bounded image downloads * fix(gemini): require explicit keys for decisions base overrides * refactor(gemini): infer downloaded image types * test(proxy): accept either metrics startup failure on occupied ports * fix(llms-types): remove duplicate compact beta match arm * fix(gemini): validate caller credentials before merging deployment keys --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Mateo Wang
GitHub
parent
b0e389503f
commit
7461a996cc
15 changed files with 1007 additions and 22 deletions
|
|
@ -1735,6 +1735,9 @@ if TYPE_CHECKING:
|
|||
from .llms.perplexity.decisions.transformation import (
|
||||
PerplexityDecisionsConfig as PerplexityDecisionsConfig,
|
||||
)
|
||||
from .llms.gemini.decisions.transformation import (
|
||||
GeminiDecisionsConfig as GeminiDecisionsConfig,
|
||||
)
|
||||
from .llms.nace.decisions.transformation import (
|
||||
NaceDecisionsConfig as NaceDecisionsConfig,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -112,6 +112,7 @@ LITELLM_ATTRIBUTES: Final[Mapping[str, tuple[str, str]]] = MappingProxyType(
|
|||
"DeepinfraRerankConfig": (".llms.deepinfra.rerank.transformation", "DeepinfraRerankConfig"),
|
||||
"HostedVLLMRerankConfig": (".llms.hosted_vllm.rerank.transformation", "HostedVLLMRerankConfig"),
|
||||
"PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
|
||||
"GeminiDecisionsConfig": (".llms.gemini.decisions.transformation", "GeminiDecisionsConfig"),
|
||||
"NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"),
|
||||
"TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
|
||||
"ParallelAIDecisionsConfig": (".llms.parallel_ai.decisions.transformation", "ParallelAIDecisionsConfig"),
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.litellm_core_utils.decisions_pricing import QUERY_PRICED_DECISIONS_
|
|||
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
||||
async_convert_url_to_base64,
|
||||
async_convert_urls_to_base64,
|
||||
convert_url_to_base64,
|
||||
)
|
||||
|
|
@ -28,6 +29,7 @@ from litellm.llms.base_llm.decisions.transformation import (
|
|||
image_media_type,
|
||||
image_param,
|
||||
input_images,
|
||||
ir_to_systemone_request,
|
||||
ir_to_systemone_response,
|
||||
remote_image_urls,
|
||||
systemone_request_to_ir,
|
||||
|
|
@ -235,6 +237,18 @@ def get_decisions_provider_config(model: str, custom_llm_provider: str) -> BaseD
|
|||
return provider_config
|
||||
|
||||
|
||||
def validate_router_connection(model: str, provider: str | None, request: Mapping[str, object]) -> None:
|
||||
api_base: Final = request.get("api_base") or request.get("base_url")
|
||||
if provider is None or not isinstance(api_base, str) or not api_base:
|
||||
return
|
||||
api_key: Final = request.get("api_key")
|
||||
provider_config: Final = get_decisions_provider_config(model, provider)
|
||||
try:
|
||||
provider_config.validate_client_connection(api_base, api_key if isinstance(api_key, str) else None)
|
||||
except ValueError as error:
|
||||
raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
|
||||
|
||||
|
||||
def _ir_request(request: DecisionsRequestFormat, safety_identifier: str | None) -> DecisionsIRRequest:
|
||||
match request:
|
||||
case DecisionsRequestBody():
|
||||
|
|
@ -649,10 +663,44 @@ def _with_data_urls(call: DecisionsCall, data_urls: Mapping[str, str]) -> Decisi
|
|||
)
|
||||
|
||||
|
||||
def _remaining_image_bytes(call: DecisionsCall) -> int | None:
|
||||
total: Final = call.provider_config.max_total_image_bytes
|
||||
if call.custom_llm_provider != "gemini" or total is None:
|
||||
return None
|
||||
_checked_image_sizes(call)
|
||||
return total - sum(_decoded_image_bytes(image) for image in input_images(call.ir_request.input))
|
||||
|
||||
|
||||
def _bounded_remote_images(
|
||||
urls: tuple[str, ...], remaining: int, timeout: float | httpx.Timeout | None
|
||||
) -> tuple[str, ...]:
|
||||
state: tuple[int, tuple[str, ...]] = (remaining, ()) # rebind-ok: sequential budget without recursion
|
||||
for url in urls:
|
||||
available, images = state
|
||||
image = convert_url_to_base64(url, timeout=timeout, max_bytes=available)
|
||||
state = (available - _decoded_image_bytes(image), (*images, image))
|
||||
return state[1]
|
||||
|
||||
|
||||
async def _async_bounded_remote_images(
|
||||
urls: tuple[str, ...], remaining: int, timeout: float | httpx.Timeout | None
|
||||
) -> tuple[str, ...]:
|
||||
state: tuple[int, tuple[str, ...]] = (remaining, ()) # rebind-ok: sequential budget without recursion
|
||||
for url in urls:
|
||||
available, images = state
|
||||
image = await async_convert_url_to_base64(url, timeout=timeout, max_bytes=available)
|
||||
state = (available - _decoded_image_bytes(image), (*images, image))
|
||||
return state[1]
|
||||
|
||||
|
||||
def _with_remote_images_inlined(call: DecisionsCall) -> DecisionsCall:
|
||||
remote_urls: Final = _remote_image_urls(call)
|
||||
if not remote_urls:
|
||||
return call
|
||||
remaining: Final = _remaining_image_bytes(call)
|
||||
if remaining is not None:
|
||||
images: Final = _bounded_remote_images(remote_urls, remaining, call.timeout)
|
||||
return _with_data_urls(call, MappingProxyType(dict(zip(remote_urls, images, strict=True))))
|
||||
return _with_data_urls(
|
||||
call, MappingProxyType({url: convert_url_to_base64(url, timeout=call.timeout) for url in remote_urls})
|
||||
)
|
||||
|
|
@ -662,7 +710,12 @@ async def _async_with_remote_images_inlined(call: DecisionsCall) -> DecisionsCal
|
|||
remote_urls: Final = _remote_image_urls(call)
|
||||
if not remote_urls:
|
||||
return call
|
||||
data_urls: Final = await async_convert_urls_to_base64(remote_urls, timeout=call.timeout)
|
||||
remaining: Final = _remaining_image_bytes(call)
|
||||
data_urls: Final = (
|
||||
await async_convert_urls_to_base64(remote_urls, timeout=call.timeout)
|
||||
if remaining is None
|
||||
else await _async_bounded_remote_images(remote_urls, remaining, call.timeout)
|
||||
)
|
||||
return _with_data_urls(call, MappingProxyType(dict(zip(remote_urls, data_urls, strict=True))))
|
||||
|
||||
|
||||
|
|
@ -753,6 +806,12 @@ def _request_body(call: DecisionsCall) -> Mapping[str, object]:
|
|||
return body
|
||||
|
||||
|
||||
def _logged_request_body(call: DecisionsCall) -> Mapping[str, object]:
|
||||
if call.custom_llm_provider == "gemini":
|
||||
return ir_to_systemone_request(call.model, call.ir_request)
|
||||
return _request_body(call)
|
||||
|
||||
|
||||
def _pricing_model(response: DecisionsIRResponse, call: DecisionsCall) -> str:
|
||||
returned: Final = get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
|
||||
if returned is not None and cost_map_key_prices_anything(returned):
|
||||
|
|
@ -828,7 +887,7 @@ async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionRespon
|
|||
provider_config=inlined.provider_config,
|
||||
request=inlined.ir_request,
|
||||
body=body,
|
||||
logged_body=body if inlined is supported else _request_body(supported),
|
||||
logged_body=body if inlined is supported else _logged_request_body(supported),
|
||||
api_base=inlined.api_base,
|
||||
api_key=inlined.api_key,
|
||||
headers=inlined.headers,
|
||||
|
|
@ -852,7 +911,7 @@ def send(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
|
|||
provider_config=inlined.provider_config,
|
||||
request=inlined.ir_request,
|
||||
body=body,
|
||||
logged_body=body if inlined is supported else _request_body(supported),
|
||||
logged_body=body if inlined is supported else _logged_request_body(supported),
|
||||
api_base=inlined.api_base,
|
||||
api_key=inlined.api_key,
|
||||
headers=inlined.headers,
|
||||
|
|
|
|||
|
|
@ -43,12 +43,15 @@ def _header_value(response: Response, name: str) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def _process_image_response(response: Response, url: str) -> str:
|
||||
def _process_image_response(response: Response, url: str, download_budget: int | None = None) -> str:
|
||||
if response.status_code != 200:
|
||||
raise litellm.ImageFetchError(
|
||||
f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}"
|
||||
)
|
||||
|
||||
if download_budget is not None and len(response.content) > download_budget:
|
||||
raise litellm.ImageFetchError(f"Image download exceeds the remaining request budget of {download_budget} bytes")
|
||||
|
||||
# Check size before downloading if Content-Length header is present
|
||||
content_length: Final = _header_value(response, "Content-Length")
|
||||
if content_length is not None:
|
||||
|
|
@ -105,7 +108,19 @@ def _rejected_image_fetch(url: str, verdict: SSRFError) -> "litellm.ImageFetchEr
|
|||
)
|
||||
|
||||
|
||||
async def async_convert_url_to_base64(url: str, timeout: float | httpx.Timeout | None = None) -> str:
|
||||
def _cached_image(url: str, max_bytes: int | None) -> str | None:
|
||||
cached: Final[object] = in_memory_cache.get_cache(url)
|
||||
if not isinstance(cached, str):
|
||||
return None
|
||||
size: Final = len(cached.partition(",")[2].rstrip("=")) * 3 // 4
|
||||
if max_bytes is not None and size > max_bytes:
|
||||
raise litellm.ImageFetchError(f"Image download exceeds the remaining request budget of {max_bytes} bytes")
|
||||
return cached
|
||||
|
||||
|
||||
async def async_convert_url_to_base64(
|
||||
url: str, timeout: float | httpx.Timeout | None = None, *, max_bytes: int | None = None
|
||||
) -> str:
|
||||
if url.startswith("data:") and ";base64," in url:
|
||||
return url
|
||||
|
||||
|
|
@ -115,15 +130,24 @@ async def async_convert_url_to_base64(url: str, timeout: float | httpx.Timeout |
|
|||
f"Error: Image URL download is disabled (MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0). url={url}"
|
||||
)
|
||||
|
||||
cached_result: Final = in_memory_cache.get_cache(url)
|
||||
cached_result: Final = _cached_image(url, max_bytes)
|
||||
if cached_result:
|
||||
return cached_result
|
||||
|
||||
client: Final = litellm.module_level_aclient
|
||||
for attempt_timeout in _attempt_timeouts(timeout):
|
||||
try:
|
||||
response = await async_safe_get(client, url, timeout=attempt_timeout)
|
||||
return _process_image_response(response, url)
|
||||
response = (
|
||||
await async_safe_get(client, url, timeout=attempt_timeout)
|
||||
if max_bytes is None
|
||||
else await async_safe_get(
|
||||
_AsyncBodyPrefixReader(client.client),
|
||||
url,
|
||||
timeout=attempt_timeout,
|
||||
max_response_bytes=max_bytes + 1,
|
||||
)
|
||||
)
|
||||
return _process_image_response(response, url, max_bytes)
|
||||
except litellm.ImageFetchError:
|
||||
raise
|
||||
except SSRFError as e:
|
||||
|
|
@ -133,7 +157,9 @@ async def async_convert_url_to_base64(url: str, timeout: float | httpx.Timeout |
|
|||
raise litellm.ImageFetchError(f"Error: Unable to fetch image from URL after 3 attempts. url={url}")
|
||||
|
||||
|
||||
def convert_url_to_base64(url: str, timeout: float | httpx.Timeout | None = None) -> str:
|
||||
def convert_url_to_base64(
|
||||
url: str, timeout: float | httpx.Timeout | None = None, *, max_bytes: int | None = None
|
||||
) -> str:
|
||||
if url.startswith("data:") and ";base64," in url:
|
||||
return url
|
||||
|
||||
|
|
@ -143,15 +169,19 @@ def convert_url_to_base64(url: str, timeout: float | httpx.Timeout | None = None
|
|||
f"Error: Image URL download is disabled (MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0). url={url}"
|
||||
)
|
||||
|
||||
cached_result: Final = in_memory_cache.get_cache(url)
|
||||
cached_result: Final = _cached_image(url, max_bytes)
|
||||
if cached_result:
|
||||
return cached_result
|
||||
|
||||
client: Final = litellm.module_level_client
|
||||
for attempt_timeout in _attempt_timeouts(timeout):
|
||||
try:
|
||||
response = safe_get(client, url, timeout=attempt_timeout)
|
||||
return _process_image_response(response, url)
|
||||
response = safe_get(
|
||||
client if max_bytes is None else _ImageBodyReader(client.client, max_bytes + 1),
|
||||
url,
|
||||
timeout=attempt_timeout,
|
||||
)
|
||||
return _process_image_response(response, url, max_bytes)
|
||||
except litellm.ImageFetchError:
|
||||
raise
|
||||
except SSRFError as e:
|
||||
|
|
@ -300,6 +330,29 @@ class _BodyPrefixReader:
|
|||
return _with_body_prefix(response, prefix[:_PDF_SIGNATURE_WINDOW])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ImageBodyReader:
|
||||
client: httpx.Client
|
||||
max_bytes: int
|
||||
|
||||
def get(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
headers: Mapping[str, str] | None = None,
|
||||
follow_redirects: bool = False,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
) -> Response:
|
||||
with self.client.stream(
|
||||
"GET",
|
||||
url,
|
||||
headers=headers,
|
||||
follow_redirects=follow_redirects,
|
||||
timeout=USE_CLIENT_DEFAULT if timeout is None else timeout,
|
||||
) as response:
|
||||
return _with_body_prefix(response, next(response.iter_bytes(self.max_bytes), b""))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _AsyncBodyPrefixReader:
|
||||
client: httpx.AsyncClient
|
||||
|
|
|
|||
|
|
@ -572,7 +572,11 @@ class BaseDecisionsConfig(ABC):
|
|||
def resolve_api_key(self, api_key: str | None) -> str | None:
|
||||
return api_key or self._first_secret(self.api_key_env)
|
||||
|
||||
def validate_client_connection(self, api_base: str | None, api_key: str | None) -> None:
|
||||
return
|
||||
|
||||
def resolve_connection(self, api_base: str | None, api_key: str | None) -> tuple[str | None, str | None]:
|
||||
self.validate_client_connection(api_base, api_key)
|
||||
return self.resolve_api_base(api_base), self.resolve_api_key(api_key)
|
||||
|
||||
def resolve_credential(
|
||||
|
|
|
|||
214
litellm/llms/gemini/decisions/transformation.py
Normal file
214
litellm/llms/gemini/decisions/transformation.py
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
from collections.abc import Mapping
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import (
|
||||
BaseDecisionsConfig,
|
||||
decisions_text,
|
||||
parse_systemone_response,
|
||||
systemone_keys,
|
||||
)
|
||||
from litellm.types.decisions import (
|
||||
DecisionsBase64Audio,
|
||||
DecisionsBase64Image,
|
||||
DecisionsBase64Video,
|
||||
DecisionsIRBoxQuestion,
|
||||
DecisionsIRChoiceQuestion,
|
||||
DecisionsIRMessages,
|
||||
DecisionsIRPredicateQuestion,
|
||||
DecisionsIRQuestion,
|
||||
DecisionsIRRequest,
|
||||
DecisionsIRResponse,
|
||||
DecisionsIRScoreQuestion,
|
||||
DecisionsIRState,
|
||||
OpenAIDecisionContentPart,
|
||||
OpenAIDecisionInputImage,
|
||||
OpenAIDecisionInputMessage,
|
||||
OpenAIDecisionInputText,
|
||||
UnsupportedDecisionsRequest,
|
||||
systemone_choice_key,
|
||||
)
|
||||
|
||||
|
||||
class _GeminiUsage(BaseModel):
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
total_input_tokens: int = Field(ge=0)
|
||||
total_output_tokens: int = Field(default=0, ge=0)
|
||||
|
||||
|
||||
class _GeminiResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
answers: Mapping[str, object]
|
||||
usage: _GeminiUsage | None = None
|
||||
|
||||
|
||||
def _instructions(question: DecisionsIRQuestion, default: str) -> str:
|
||||
return default if question.instructions is None else decisions_text(question.instructions)
|
||||
|
||||
|
||||
def _question(question: DecisionsIRQuestion) -> Mapping[str, object]:
|
||||
match question:
|
||||
case DecisionsIRPredicateQuestion():
|
||||
return {
|
||||
"type": "noul",
|
||||
"instructions": _instructions(question, "Does the input match the true rather than false criteria?"),
|
||||
**(
|
||||
{
|
||||
"criteria": {
|
||||
f"{key}_description": decisions_text(value) if value is not None else ""
|
||||
for key, value in question.criteria.items()
|
||||
}
|
||||
}
|
||||
if question.criteria is not None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
case DecisionsIRChoiceQuestion():
|
||||
return {
|
||||
"type": "choice",
|
||||
"instructions": _instructions(question, "Choose the option that best matches the input."),
|
||||
"criteria": {
|
||||
systemone_choice_key(option.value): decisions_text(option.description)
|
||||
if option.description is not None
|
||||
else ""
|
||||
for option in question.choices
|
||||
},
|
||||
}
|
||||
case DecisionsIRScoreQuestion():
|
||||
return {
|
||||
"type": "score",
|
||||
"instructions": _instructions(question, "Rate the input against the ordered criteria."),
|
||||
"criteria": [
|
||||
decisions_text(level.label)
|
||||
if level.description is None
|
||||
else f"{decisions_text(level.label)}: {level.description}"
|
||||
for level in question.levels
|
||||
],
|
||||
}
|
||||
case DecisionsIRBoxQuestion():
|
||||
return {}
|
||||
case _:
|
||||
assert_never(question)
|
||||
|
||||
|
||||
def _media(
|
||||
media: str | DecisionsBase64Image | DecisionsBase64Audio | DecisionsBase64Video, kind: str
|
||||
) -> Mapping[str, object]:
|
||||
if not isinstance(media, str):
|
||||
return {"type": kind, "data": media.base64, "mime_type": media.content_type}
|
||||
header, data = media.split(",", 1)
|
||||
return {"type": kind, "data": data, "mime_type": header.removeprefix("data:").split(";", 1)[0]}
|
||||
|
||||
|
||||
def _part(part: OpenAIDecisionContentPart) -> Mapping[str, object]:
|
||||
match part:
|
||||
case OpenAIDecisionInputText():
|
||||
return {"type": "text", "text": part.text}
|
||||
case OpenAIDecisionInputImage():
|
||||
return _media(part.image_url, "image")
|
||||
case _:
|
||||
assert_never(part)
|
||||
|
||||
|
||||
def _message_parts(message: OpenAIDecisionInputMessage) -> tuple[Mapping[str, object], ...]:
|
||||
if isinstance(message.content, str):
|
||||
return ({"type": "text", "text": message.content},)
|
||||
return tuple(_part(part) for part in message.content)
|
||||
|
||||
|
||||
def _input_parts(input_: DecisionsIRState | DecisionsIRMessages) -> tuple[Mapping[str, object], ...]:
|
||||
match input_:
|
||||
case DecisionsIRState():
|
||||
return ({"type": "text", "text": decisions_text(input_.state)},) + tuple(
|
||||
_media(image, "image") for image in input_.images
|
||||
)
|
||||
case DecisionsIRMessages():
|
||||
return tuple(chain.from_iterable(_message_parts(message) for message in input_.messages))
|
||||
case _:
|
||||
assert_never(input_)
|
||||
|
||||
|
||||
class GeminiDecisionsConfig(BaseDecisionsConfig):
|
||||
path = "/v1beta/decisions"
|
||||
api_key_env = ("GEMINI_API_KEY", "GOOGLE_API_KEY")
|
||||
api_base_env = ("GEMINI_API_BASE",)
|
||||
supports_images = True
|
||||
inlines_remote_images = True
|
||||
max_images = 16
|
||||
max_total_image_bytes = 20 * 1024 * 1024
|
||||
|
||||
def get_default_api_base(self) -> str | None:
|
||||
return "https://generativelanguage.googleapis.com"
|
||||
|
||||
def get_complete_url(self, api_base: str, model: str) -> str:
|
||||
return f"{api_base.rstrip('/').removesuffix('/v1beta')}/v1beta/decisions"
|
||||
|
||||
def validate_client_connection(self, api_base: str | None, api_key: str | None) -> None:
|
||||
if api_base and not api_key:
|
||||
raise ValueError(
|
||||
"When overriding api_base for Gemini Decisions, supply an explicit api_key "
|
||||
"to prevent sending the shared Google key to a custom host"
|
||||
)
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
model: str,
|
||||
api_key: str | None,
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> Mapping[str, str]:
|
||||
return {
|
||||
**{
|
||||
key: value
|
||||
for key, value in headers.items()
|
||||
if key.lower() not in {"authorization", "x-goog-api-key", "content-type"}
|
||||
},
|
||||
**({"x-goog-api-key": api_key} if api_key is not None else {}),
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
def transform_decisions_request(
|
||||
self,
|
||||
model: str,
|
||||
request: DecisionsIRRequest,
|
||||
custom_llm_provider: str,
|
||||
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
|
||||
if any(isinstance(question, DecisionsIRBoxQuestion) for question in request.questions):
|
||||
return UnsupportedDecisionsRequest("Gemini Decisions does not support box questions")
|
||||
parts: Final = (
|
||||
_input_parts(request.input)
|
||||
+ tuple(_media(audio, "audio") for audio in request.audio)
|
||||
+ tuple(_media(video, "video") for video in request.videos)
|
||||
)
|
||||
return {
|
||||
"model": model,
|
||||
"input": parts,
|
||||
"questions": {
|
||||
key: _question(question)
|
||||
for key, question in zip(systemone_keys(request.questions), request.questions, strict=True)
|
||||
},
|
||||
}
|
||||
|
||||
def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
|
||||
response: Final = _GeminiResponse.model_validate(payload)
|
||||
usage: Final = response.usage
|
||||
return parse_systemone_response(
|
||||
{
|
||||
**response.model_dump(exclude={"usage"}),
|
||||
**(
|
||||
{
|
||||
"usage": {
|
||||
**usage.model_dump(),
|
||||
"input_tokens": usage.total_input_tokens,
|
||||
"output_tokens": usage.total_output_tokens,
|
||||
}
|
||||
}
|
||||
if usage is not None
|
||||
else {}
|
||||
),
|
||||
},
|
||||
request,
|
||||
)
|
||||
|
|
@ -30545,6 +30545,32 @@
|
|||
"gemini_native_audio": true,
|
||||
"input_cost_per_image_token": 3e-06
|
||||
},
|
||||
"gemini/gemini-decision-1": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"rpm": 1500,
|
||||
"tpm": 1000000,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "decisions",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
"supported_endpoints": ["/v1/decisions"],
|
||||
"supported_modalities": ["text", "image", "audio", "video"],
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": false,
|
||||
"supports_computer_use": false,
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": false,
|
||||
"supports_reasoning": false,
|
||||
"supports_response_schema": false,
|
||||
"supports_tool_choice": false,
|
||||
"supports_url_context": false,
|
||||
"supports_video_input": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": false
|
||||
},
|
||||
"gemini/gemini-live-2.5-flash-preview-native-audio-09-2025": {
|
||||
"input_cost_per_audio_token": 3e-06,
|
||||
"input_cost_per_token": 5e-07,
|
||||
|
|
|
|||
|
|
@ -867,6 +867,14 @@ def _deployment_params(
|
|||
return as_dict(store_search_params(params)) if plan.host_op in _VECTOR_STORE_SEARCH_OPS else params
|
||||
|
||||
|
||||
def _validate_decisions_connection(plan: Plan, data: Mapping[str, object], request: Mapping[str, object]) -> None:
|
||||
if _handler_name(plan.host_op) not in {"decisions", "adecisions", "systemone", "asystemone"}:
|
||||
return
|
||||
from litellm.decisions.call import validate_router_connection
|
||||
|
||||
validate_router_connection(str(data.get("model")), _generic_provider(data), request)
|
||||
|
||||
|
||||
def attempt_call(
|
||||
core: EndpointCore,
|
||||
plan: Plan,
|
||||
|
|
@ -884,6 +892,7 @@ def attempt_call(
|
|||
named: Final = {name: kwargs.pop(name) for name in plan.named if name in kwargs}
|
||||
data: Final = _deployment_params(plan, deployment)
|
||||
model_name: Final = str(data.get("model"))
|
||||
_validate_decisions_connection(plan, data, request)
|
||||
cache: Final = core.routing_cache
|
||||
caching: Final = shaping.cache_responses
|
||||
# A fallback hop routes the target group in a copy of the metadata (`run_async_fallback`).
|
||||
|
|
|
|||
|
|
@ -10236,8 +10236,11 @@ class ProviderConfigManager:
|
|||
|
||||
@staticmethod
|
||||
def get_provider_decisions_config(model: str, provider: LlmProviders) -> BaseDecisionsConfig | None:
|
||||
if provider == LlmProviders.PERPLEXITY:
|
||||
return litellm.PerplexityDecisionsConfig()
|
||||
if provider in (LlmProviders.GEMINI, LlmProviders.PERPLEXITY):
|
||||
return {
|
||||
LlmProviders.GEMINI: litellm.GeminiDecisionsConfig,
|
||||
LlmProviders.PERPLEXITY: litellm.PerplexityDecisionsConfig,
|
||||
}[provider]()
|
||||
if provider == LlmProviders.TYPESAFE:
|
||||
return litellm.TypeSafeDecisionsConfig()
|
||||
if provider == LlmProviders.NACE:
|
||||
|
|
|
|||
|
|
@ -30545,6 +30545,32 @@
|
|||
"gemini_native_audio": true,
|
||||
"input_cost_per_image_token": 3e-06
|
||||
},
|
||||
"gemini/gemini-decision-1": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"rpm": 1500,
|
||||
"tpm": 1000000,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "decisions",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
"supported_endpoints": ["/v1/decisions"],
|
||||
"supported_modalities": ["text", "image", "audio", "video"],
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": false,
|
||||
"supports_computer_use": false,
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": false,
|
||||
"supports_reasoning": false,
|
||||
"supports_response_schema": false,
|
||||
"supports_tool_choice": false,
|
||||
"supports_url_context": false,
|
||||
"supports_video_input": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": false
|
||||
},
|
||||
"gemini/gemini-live-2.5-flash-preview-native-audio-09-2025": {
|
||||
"input_cost_per_audio_token": 3e-06,
|
||||
"input_cost_per_token": 5e-07,
|
||||
|
|
|
|||
|
|
@ -1,14 +1,19 @@
|
|||
import base64
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.decisions.main import _prepare_decisions_call
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import in_memory_cache
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.systemone.main import _prepare_systemone_call
|
||||
|
||||
QUESTION_TEXT: Final = "Is this about a billing problem?"
|
||||
|
|
@ -132,3 +137,216 @@ def test_non_utf8_bytes_in_the_request_are_logged_and_rejected_with_a_400():
|
|||
logged: Final = _logged_request(logging_obj)
|
||||
assert base64.urlsafe_b64decode(logged["input"]) == raw_input
|
||||
assert logged["questions"][0]["instructions"] == QUESTION_TEXT
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("gemini_httpx_transport")
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync", (False, True))
|
||||
async def test_gemini_rejects_excess_images_before_fetching(sync: bool, respx_mock: respx.MockRouter) -> None:
|
||||
limit: Final = 16
|
||||
images: Final = [f"https://images.example/{index}.png" for index in range(limit + 1)]
|
||||
image_route: Final = respx_mock.get(url__startswith="https://images.example/").mock(
|
||||
return_value=httpx.Response(200, content=b"image")
|
||||
)
|
||||
with pytest.raises(litellm.BadRequestError, match="at most"):
|
||||
await _call_gemini(sync, images)
|
||||
assert image_route.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("gemini_httpx_transport")
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync", (False, True))
|
||||
async def test_gemini_stops_image_downloads_at_the_request_budget(
|
||||
sync: bool, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "user_url_validation", False)
|
||||
monkeypatch.setattr(litellm, "module_level_aclient", AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport()))
|
||||
budget: Final = 20 * 1024 * 1024
|
||||
image: Final = b"x" * (budget // 2 + 1)
|
||||
first: Final = respx_mock.get("https://images.example/first.png").mock(
|
||||
return_value=httpx.Response(200, content=image, headers={"content-type": "image/png"})
|
||||
)
|
||||
second: Final = respx_mock.get("https://images.example/second.png").mock(
|
||||
return_value=httpx.Response(200, content=image, headers={"content-type": "image/png"})
|
||||
)
|
||||
last: Final = respx_mock.get("https://images.example/last.png").mock(
|
||||
return_value=httpx.Response(200, content=b"unused")
|
||||
)
|
||||
provider: Final = respx_mock.post("https://generativelanguage.googleapis.com/v1beta/decisions").mock(
|
||||
return_value=httpx.Response(200, json={"answers": {"ok": {"type": "noul", "noul": 0.5}}})
|
||||
)
|
||||
urls: Final = [
|
||||
"https://images.example/first.png",
|
||||
"https://images.example/second.png",
|
||||
"https://images.example/last.png",
|
||||
]
|
||||
with pytest.raises(litellm.ImageFetchError, match="remaining request budget"):
|
||||
await _call_gemini(sync, urls)
|
||||
assert first.call_count == second.call_count == 1
|
||||
assert last.call_count == provider.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("gemini_httpx_transport")
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync", (False, True))
|
||||
async def test_gemini_accepts_remote_images_within_the_request_budget(
|
||||
sync: bool, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "user_url_validation", False)
|
||||
monkeypatch.setattr(litellm, "module_level_aclient", AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport()))
|
||||
image: Final = b"\x89PNG\r\n\x1a\n"
|
||||
image_route: Final = respx_mock.get("https://images.example/small.png").mock(
|
||||
return_value=httpx.Response(200, content=image, headers={"content-type": "image/png"})
|
||||
)
|
||||
provider: Final = respx_mock.post("https://generativelanguage.googleapis.com/v1beta/decisions").mock(
|
||||
return_value=httpx.Response(200, json={"answers": {"ok": {"type": "noul", "noul": 0.5}}})
|
||||
)
|
||||
await _call_gemini(sync, ("https://images.example/small.png",))
|
||||
await _call_gemini(sync, ("https://images.example/small.png",))
|
||||
sent: Final = TypeAdapter(dict[str, object]).validate_json(provider.calls.last.request.content)
|
||||
assert sent["input"] == [
|
||||
{"type": "text", "text": "Inspect"},
|
||||
{"type": "image", "data": base64.b64encode(image).decode(), "mime_type": "image/png"},
|
||||
]
|
||||
assert image_route.call_count == 1
|
||||
inline_image: Final = "data:image/png;base64," + base64.b64encode(b"x" * (20 * 1024 * 1024 - 4)).decode()
|
||||
with pytest.raises(litellm.ImageFetchError, match="remaining request budget"):
|
||||
await _call_gemini(sync, (inline_image, "https://images.example/small.png"))
|
||||
assert image_route.call_count == 1
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gemini_httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
in_memory_cache.flush_cache()
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("gemini_httpx_transport")
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync", (False, True))
|
||||
@pytest.mark.parametrize("base_parameter", ("api_base", "base_url"))
|
||||
async def test_gemini_custom_base_cannot_receive_environment_key(
|
||||
sync: bool, base_parameter: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
monkeypatch.setenv("GEMINI_API_KEY", "shared-google-key")
|
||||
destination: Final = respx_mock.post("https://gateway.example/v1beta/decisions")
|
||||
options: Final = {base_parameter: "https://gateway.example"}
|
||||
if sync:
|
||||
with pytest.raises(litellm.BadRequestError, match="explicit api_key"):
|
||||
litellm.systemone(
|
||||
model="gemini/gemini-decision-1",
|
||||
state="Inspect",
|
||||
questions={"ok": {"type": "noul", "instructions": "Is it damaged?"}},
|
||||
**options,
|
||||
)
|
||||
else:
|
||||
with pytest.raises(litellm.BadRequestError, match="explicit api_key"):
|
||||
await litellm.adecisions(
|
||||
model="gemini/gemini-decision-1",
|
||||
input="Inspect",
|
||||
questions=[{"name": "ok", "type": "predicate", "instructions": "Is it damaged?"}],
|
||||
**options,
|
||||
)
|
||||
assert destination.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("gemini_httpx_transport")
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("systemone", (False, True))
|
||||
@pytest.mark.parametrize("base_parameter", ("api_base", "base_url"))
|
||||
async def test_router_gemini_custom_base_cannot_receive_deployment_key(
|
||||
systemone: bool, base_parameter: str, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gemini-decisions",
|
||||
"litellm_params": {"model": "gemini/gemini-decision-1", "api_key": "deployment-key"},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
destination: Final = respx_mock.post("https://gateway.example/v1beta/decisions").respond(
|
||||
json={"answers": {"ok": {"type": "noul", "noul": 0.5}}}
|
||||
)
|
||||
options: Final = {base_parameter: "https://gateway.example"}
|
||||
if systemone:
|
||||
with pytest.raises(litellm.BadRequestError, match="explicit api_key"):
|
||||
await router.asystemone(
|
||||
model="gemini-decisions",
|
||||
state="Inspect",
|
||||
questions={"ok": {"type": "noul", "instructions": "Is it damaged?"}},
|
||||
**options,
|
||||
)
|
||||
else:
|
||||
with pytest.raises(litellm.BadRequestError, match="explicit api_key"):
|
||||
await router.adecisions(
|
||||
model="gemini-decisions",
|
||||
input="Inspect",
|
||||
questions=[{"name": "ok", "type": "predicate", "instructions": "Is it damaged?"}],
|
||||
**options,
|
||||
)
|
||||
assert destination.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("gemini_httpx_transport")
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("systemone", (False, True))
|
||||
@pytest.mark.parametrize("overrides", (False, True))
|
||||
async def test_router_gemini_keeps_keys_with_their_configured_or_explicit_destination(
|
||||
systemone: bool, overrides: bool, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gemini-decisions",
|
||||
"litellm_params": {
|
||||
"model": "gemini/gemini-decision-1",
|
||||
"api_key": "deployment-key",
|
||||
"api_base": "https://deployment.example",
|
||||
},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
target: Final = "https://gateway.example" if overrides else "https://deployment.example"
|
||||
destination: Final = respx_mock.post(f"{target}/v1beta/decisions").respond(
|
||||
json={"answers": {"ok": {"type": "noul", "noul": 0.5}}}
|
||||
)
|
||||
options: Final = {"api_base": target, "api_key": "caller-key"} if overrides else {}
|
||||
if systemone:
|
||||
await router.asystemone(
|
||||
model="gemini-decisions",
|
||||
state="Inspect",
|
||||
questions={"ok": {"type": "noul", "instructions": "Is it damaged?"}},
|
||||
**options,
|
||||
)
|
||||
else:
|
||||
await router.adecisions(
|
||||
model="gemini-decisions",
|
||||
input="Inspect",
|
||||
questions=[{"name": "ok", "type": "predicate", "instructions": "Is it damaged?"}],
|
||||
**options,
|
||||
)
|
||||
assert destination.call_count == 1
|
||||
assert destination.calls[0].request.headers["x-goog-api-key"] == ("caller-key" if overrides else "deployment-key")
|
||||
|
||||
|
||||
async def _call_gemini(sync: bool, images: Sequence[str]) -> None:
|
||||
if sync:
|
||||
litellm.systemone(
|
||||
model="gemini/gemini-decision-1",
|
||||
state="Inspect",
|
||||
questions={"ok": {"type": "noul", "instructions": "Is it damaged?"}},
|
||||
images=images,
|
||||
api_key="test",
|
||||
)
|
||||
return
|
||||
await litellm.asystemone(
|
||||
model="gemini/gemini-decision-1",
|
||||
state="Inspect",
|
||||
questions={"ok": {"type": "noul", "instructions": "Is it damaged?"}},
|
||||
images=images,
|
||||
api_key="test",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1603,14 +1603,16 @@ _DEDICATED_DECIDER_PROVIDERS: Final[tuple[str, ...]] = tuple(
|
|||
|
||||
|
||||
@pytest.mark.parametrize("provider", _DEDICATED_DECIDER_PROVIDERS)
|
||||
def test_every_dedicated_decider_provider_ships_an_evaluation_mode_cost_map_entry(provider: str) -> None:
|
||||
def test_every_dedicated_decider_provider_ships_a_decisions_mode_cost_map_entry(provider: str) -> None:
|
||||
evaluation_entries: Final = tuple(
|
||||
name
|
||||
for name, info in litellm.model_cost.items()
|
||||
if isinstance(info, Mapping) and info.get("litellm_provider") == provider and info.get("mode") == "evaluation"
|
||||
if isinstance(info, Mapping)
|
||||
and info.get("litellm_provider") == provider
|
||||
and info.get("mode") in ("evaluation", "decisions")
|
||||
)
|
||||
|
||||
assert evaluation_entries, f"a {provider} decider would be health-checked as chat without a mode: evaluation entry"
|
||||
assert evaluation_entries, f"a {provider} decider would be health-checked as chat without a Decisions mode entry"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
0
tests/unit/llms/gemini/decisions/__init__.py
Normal file
0
tests/unit/llms/gemini/decisions/__init__.py
Normal file
369
tests/unit/llms/gemini/decisions/test_transformation.py
Normal file
369
tests/unit/llms/gemini/decisions/test_transformation.py
Normal file
|
|
@ -0,0 +1,369 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.base_llm.decisions.transformation import (
|
||||
DecisionsResponseValidationError,
|
||||
ir_to_systemone_response,
|
||||
systemone_request_to_ir,
|
||||
)
|
||||
from litellm.llms.gemini.decisions.transformation import GeminiDecisionsConfig
|
||||
from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir
|
||||
from litellm.types.decisions import (
|
||||
DecisionsIRRequest,
|
||||
DecisionsRequestBody,
|
||||
OpenAIDecisionRequestBody,
|
||||
UnsupportedDecisionsRequest,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def request_ir() -> DecisionsIRRequest:
|
||||
return systemone_request_to_ir(
|
||||
DecisionsRequestBody.model_validate(
|
||||
{
|
||||
"state": {"item": "cracked mug"},
|
||||
"questions": {
|
||||
"damaged": {
|
||||
"type": "noul",
|
||||
"instructions": "Damaged?",
|
||||
"criteria": {"true": "Broken", "false": "Intact"},
|
||||
},
|
||||
"action": {
|
||||
"type": "choice",
|
||||
"instructions": "Next?",
|
||||
"criteria": {"replace": "Broken", "keep": None},
|
||||
},
|
||||
"severity": {"type": "score", "instructions": "Severity?", "criteria": ["None", "Severe"]},
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_systemone_questions_and_json_state_become_gemini_input(request_ir: DecisionsIRRequest) -> None:
|
||||
assert GeminiDecisionsConfig().transform_decisions_request("gemini-decision-1", request_ir, "gemini") == {
|
||||
"model": "gemini-decision-1",
|
||||
"input": ({"type": "text", "text": '{"item": "cracked mug"}'},),
|
||||
"questions": {
|
||||
"damaged": {
|
||||
"type": "noul",
|
||||
"instructions": "Damaged?",
|
||||
"criteria": {"true_description": "Broken", "false_description": "Intact"},
|
||||
},
|
||||
"action": {"type": "choice", "instructions": "Next?", "criteria": {"replace": "Broken", "keep": ""}},
|
||||
"severity": {"type": "score", "instructions": "Severity?", "criteria": ["None", "Severe"]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_openai_questions_preserve_boolean_values_and_score_descriptions() -> None:
|
||||
request: Final = openai_request_to_ir(
|
||||
OpenAIDecisionRequestBody.model_validate(
|
||||
{
|
||||
"model": "gemini/gemini-decision-1",
|
||||
"input": "mug",
|
||||
"questions": [
|
||||
{"type": "predicate", "instructions": "Damaged?"},
|
||||
{
|
||||
"type": "choice",
|
||||
"instructions": "Replace?",
|
||||
"choices": [{"value": True}, {"value": False, "description": "Keep"}],
|
||||
},
|
||||
{
|
||||
"type": "score",
|
||||
"instructions": "Severity?",
|
||||
"levels": [{"label": "None"}, {"label": "Severe", "description": "Unusable"}],
|
||||
},
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
assert GeminiDecisionsConfig().transform_decisions_request("gemini-decision-1", request, "gemini") == {
|
||||
"model": "gemini-decision-1",
|
||||
"input": ({"type": "text", "text": "mug"},),
|
||||
"questions": {
|
||||
"0": {"type": "noul", "instructions": "Damaged?"},
|
||||
"1": {"type": "choice", "instructions": "Replace?", "criteria": {"true": "", "false": "Keep"}},
|
||||
"2": {"type": "score", "instructions": "Severity?", "criteria": ["None", "Severe: Unusable"]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_inline_images_audio_and_video_become_typed_blocks() -> None:
|
||||
request: Final = systemone_request_to_ir(
|
||||
DecisionsRequestBody.model_validate(
|
||||
{
|
||||
"state": "Inspect",
|
||||
"images": ["data:image/png;base64,YQ==", {"content_type": "image/jpeg", "base64": "Yg=="}],
|
||||
"audio": ["data:audio/wav;base64,Yw=="],
|
||||
"videos": [{"content_type": "video/mp4", "base64": "ZA=="}],
|
||||
"questions": {"ok": {"type": "noul", "instructions": "OK?"}},
|
||||
}
|
||||
)
|
||||
)
|
||||
assert GeminiDecisionsConfig().transform_decisions_request("gemini-decision-1", request, "gemini") == {
|
||||
"model": "gemini-decision-1",
|
||||
"input": (
|
||||
{"type": "text", "text": "Inspect"},
|
||||
{"type": "image", "data": "YQ==", "mime_type": "image/png"},
|
||||
{"type": "image", "data": "Yg==", "mime_type": "image/jpeg"},
|
||||
{"type": "audio", "data": "Yw==", "mime_type": "audio/wav"},
|
||||
{"type": "video", "data": "ZA==", "mime_type": "video/mp4"},
|
||||
),
|
||||
"questions": {"ok": {"type": "noul", "instructions": "OK?"}},
|
||||
}
|
||||
|
||||
|
||||
def test_boolean_choice_response_returns_boolean_not_wire_label() -> None:
|
||||
request: Final = openai_request_to_ir(
|
||||
OpenAIDecisionRequestBody.model_validate(
|
||||
{
|
||||
"model": "gemini/gemini-decision-1",
|
||||
"input": "mug",
|
||||
"questions": [
|
||||
{
|
||||
"type": "choice",
|
||||
"name": "replace",
|
||||
"instructions": "Replace?",
|
||||
"choices": [{"value": True}, {"value": False}],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
response: Final = GeminiDecisionsConfig().parse_response(
|
||||
{
|
||||
"answers": {
|
||||
"replace": {
|
||||
"type": "choice",
|
||||
"choice": "false",
|
||||
"confidence": 0.9,
|
||||
"probabilities": {"true": 0.1, "false": 0.9},
|
||||
}
|
||||
}
|
||||
},
|
||||
request,
|
||||
)
|
||||
assert ir_to_openai_response(response, request, "gemini/gemini-decision-1").model_dump(
|
||||
mode="json", exclude_none=True
|
||||
)["answers"] == [
|
||||
{
|
||||
"type": "choice",
|
||||
"name": "replace",
|
||||
"choice": False,
|
||||
"confidence": 0.9,
|
||||
"probabilities": [{"value": True, "probability": 0.1}, {"value": False, "probability": 0.9}],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_criteria_only_questions_receive_required_instructions() -> None:
|
||||
request: Final = systemone_request_to_ir(
|
||||
DecisionsRequestBody.model_validate(
|
||||
{
|
||||
"state": "mug",
|
||||
"questions": {
|
||||
"damaged": {"type": "noul", "criteria": {"true": "Broken", "false": "Intact"}},
|
||||
"action": {"type": "choice", "criteria": {"replace": "Broken", "keep": "Intact"}},
|
||||
"severity": {"type": "score", "criteria": ["None", "Severe"]},
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
assert GeminiDecisionsConfig().transform_decisions_request("gemini-decision-1", request, "gemini") == {
|
||||
"model": "gemini-decision-1",
|
||||
"input": ({"type": "text", "text": "mug"},),
|
||||
"questions": {
|
||||
"damaged": {
|
||||
"type": "noul",
|
||||
"instructions": "Does the input match the true rather than false criteria?",
|
||||
"criteria": {"true_description": "Broken", "false_description": "Intact"},
|
||||
},
|
||||
"action": {
|
||||
"type": "choice",
|
||||
"instructions": "Choose the option that best matches the input.",
|
||||
"criteria": {"replace": "Broken", "keep": "Intact"},
|
||||
},
|
||||
"severity": {
|
||||
"type": "score",
|
||||
"instructions": "Rate the input against the ordered criteria.",
|
||||
"criteria": ["None", "Severe"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_response_preserves_answers_and_normalizes_billable_usage(request_ir: DecisionsIRRequest) -> None:
|
||||
answers: Final = {
|
||||
"damaged": {"type": "noul", "noul": 0.98, "confidence": 0.98},
|
||||
"action": {
|
||||
"type": "choice",
|
||||
"choice": "replace",
|
||||
"confidence": 0.9,
|
||||
"probabilities": {"replace": 0.9, "keep": 0.1},
|
||||
},
|
||||
"severity": {
|
||||
"type": "score",
|
||||
"score": 0.8,
|
||||
"confidence": 0.8,
|
||||
"legend": {"0": "None", "1": "Severe"},
|
||||
"probabilities": {"0": 0.2, "1": 0.8},
|
||||
},
|
||||
}
|
||||
usage: Final = {
|
||||
"total_input_tokens": 200,
|
||||
"total_output_tokens": 0,
|
||||
"total_tokens": 200,
|
||||
"input_tokens_by_modality": [{"modality": "text", "tokens": 200}],
|
||||
}
|
||||
response: Final = GeminiDecisionsConfig().parse_response({"answers": answers, "usage": usage}, request_ir)
|
||||
assert ir_to_systemone_response(response, request_ir).model_dump(exclude_none=True) == {
|
||||
"answers": answers,
|
||||
"usage": {**usage, "input_tokens": 200, "output_tokens": 0},
|
||||
}
|
||||
assert response.usage is not None
|
||||
assert response.usage.input_tokens == 200
|
||||
assert response.usage.output_tokens == 0
|
||||
|
||||
|
||||
def test_missing_usage_is_not_invented(request_ir: DecisionsIRRequest) -> None:
|
||||
response: Final = GeminiDecisionsConfig().parse_response({"answers": {}}, request_ir)
|
||||
assert response.usage is None
|
||||
assert ir_to_systemone_response(response, request_ir).answers == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[
|
||||
{"answers": {}, "usage": {"total_input_tokens": -1}},
|
||||
{"answers": {}, "usage": {}},
|
||||
{"answers": {"damaged": {"type": "noul", "noul": "not a probability"}}},
|
||||
{"answers": {"action": {"type": "choice", "choice": "replace", "probabilities": {"replace": 1.0}}}},
|
||||
],
|
||||
)
|
||||
def test_invalid_provider_response_uses_existing_500_contract(
|
||||
request_ir: DecisionsIRRequest,
|
||||
payload: Mapping[str, object],
|
||||
) -> None:
|
||||
with pytest.raises(DecisionsResponseValidationError) as error:
|
||||
GeminiDecisionsConfig().transform_decisions_response(
|
||||
"gemini-decision-1",
|
||||
"gemini",
|
||||
httpx.Response(200, json=payload),
|
||||
request_ir,
|
||||
)
|
||||
assert error.value.status_code == 500
|
||||
|
||||
|
||||
@pytest.mark.parametrize("base", ["https://example.com", "https://example.com/v1beta", "https://example.com/v1beta/"])
|
||||
def test_url_and_key_auth_do_not_use_model_rpc_or_bearer_token(base: str) -> None:
|
||||
config: Final = GeminiDecisionsConfig()
|
||||
assert config.get_complete_url(base, "gemini-decision-1") == "https://example.com/v1beta/decisions"
|
||||
assert config.validate_environment(
|
||||
{"Authorization": "Bearer wrong", "X-Goog-Api-Key": "wrong", "X-Trace": "trace"},
|
||||
"gemini-decision-1",
|
||||
"test-key",
|
||||
{},
|
||||
) == {"x-goog-api-key": "test-key", "Content-Type": "application/json", "X-Trace": "trace"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key_env", ("GEMINI_API_KEY", "GOOGLE_API_KEY"))
|
||||
@pytest.mark.parametrize("api_key", (None, ""))
|
||||
def test_custom_base_requires_its_own_key(monkeypatch: pytest.MonkeyPatch, key_env: str, api_key: str | None) -> None:
|
||||
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
|
||||
monkeypatch.delenv("GOOGLE_API_KEY", raising=False)
|
||||
monkeypatch.setenv(key_env, "shared-google-key")
|
||||
with pytest.raises(ValueError, match="explicit api_key"):
|
||||
GeminiDecisionsConfig().resolve_connection("https://gateway.example", api_key)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key_env", ("GEMINI_API_KEY", "GOOGLE_API_KEY"))
|
||||
def test_default_base_uses_environment_key_and_custom_base_uses_explicit_key(
|
||||
monkeypatch: pytest.MonkeyPatch, key_env: str
|
||||
) -> None:
|
||||
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
|
||||
monkeypatch.delenv("GOOGLE_API_KEY", raising=False)
|
||||
monkeypatch.delenv("GEMINI_API_BASE", raising=False)
|
||||
monkeypatch.setenv(key_env, "shared-google-key")
|
||||
config: Final = GeminiDecisionsConfig()
|
||||
assert config.resolve_connection(None, None) == (
|
||||
"https://generativelanguage.googleapis.com",
|
||||
"shared-google-key",
|
||||
)
|
||||
assert config.resolve_connection("https://gateway.example", "gateway-key") == (
|
||||
"https://gateway.example",
|
||||
"gateway-key",
|
||||
)
|
||||
monkeypatch.setenv("GEMINI_API_BASE", "https://configured-gateway.example")
|
||||
assert config.resolve_connection(None, None) == ("https://configured-gateway.example", "shared-google-key")
|
||||
|
||||
|
||||
def test_box_questions_are_rejected_by_transformation_and_public_sdk() -> None:
|
||||
request: Final = systemone_request_to_ir(
|
||||
DecisionsRequestBody.model_validate(
|
||||
{
|
||||
"state": "Find mug",
|
||||
"questions": {"mug": {"type": "box", "instructions": "Find mug"}},
|
||||
}
|
||||
)
|
||||
)
|
||||
assert isinstance(
|
||||
GeminiDecisionsConfig().transform_decisions_request("gemini-decision-1", request, "gemini"),
|
||||
UnsupportedDecisionsRequest,
|
||||
)
|
||||
with pytest.raises(litellm.BadRequestError) as error:
|
||||
litellm.systemone(
|
||||
model="gemini/gemini-decision-1",
|
||||
state="Find mug",
|
||||
questions={"mug": {"type": "box", "instructions": "Find mug"}},
|
||||
api_key="test-key",
|
||||
)
|
||||
assert error.value.status_code == 400
|
||||
|
||||
|
||||
def test_empty_systemone_questions_are_rejected_before_provider_call() -> None:
|
||||
with pytest.raises(litellm.BadRequestError) as error:
|
||||
litellm.systemone(model="gemini/gemini-decision-1", state="mug", questions={}, api_key="test-key")
|
||||
assert error.value.status_code == 400
|
||||
|
||||
|
||||
def test_empty_decisions_questions_are_rejected_before_provider_call() -> None:
|
||||
with pytest.raises(litellm.BadRequestError) as error:
|
||||
litellm.decisions(model="gemini/gemini-decision-1", input="mug", questions=[], api_key="test-key")
|
||||
assert error.value.status_code == 400
|
||||
|
||||
|
||||
def test_message_captions_keep_their_position_next_to_each_image() -> None:
|
||||
request: Final = openai_request_to_ir(
|
||||
OpenAIDecisionRequestBody.model_validate(
|
||||
{
|
||||
"model": "gemini/gemini-decision-1",
|
||||
"input": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "Before"},
|
||||
{"type": "input_image", "image_url": "data:image/png;base64,YQ=="},
|
||||
{"type": "input_text", "text": "After"},
|
||||
{"type": "input_image", "image_url": "data:image/png;base64,Yg=="},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "Done"},
|
||||
],
|
||||
"questions": [{"type": "predicate", "instructions": "Did it change?"}],
|
||||
}
|
||||
)
|
||||
)
|
||||
transformed: Final = GeminiDecisionsConfig().transform_decisions_request("gemini-decision-1", request, "gemini")
|
||||
assert not isinstance(transformed, UnsupportedDecisionsRequest)
|
||||
assert transformed["input"] == (
|
||||
{"type": "text", "text": "Before"},
|
||||
{"type": "image", "data": "YQ==", "mime_type": "image/png"},
|
||||
{"type": "text", "text": "After"},
|
||||
{"type": "image", "data": "Yg==", "mime_type": "image/png"},
|
||||
{"type": "text", "text": "Done"},
|
||||
)
|
||||
|
|
@ -177,9 +177,7 @@ def test_start_metrics_server_process_fails_when_port_is_taken(tmp_path: Path):
|
|||
port: Final = occupied.getsockname()[1]
|
||||
with (
|
||||
patch("atexit.register"),
|
||||
pytest.raises(
|
||||
MetricsServerStartupError, match=rf"exited with code [1-9]\d* before serving 127.0.0.1:{port}"
|
||||
),
|
||||
pytest.raises(MetricsServerStartupError),
|
||||
):
|
||||
start_metrics_server_process(host="127.0.0.1", port=port, multiproc_dir=str(tmp_path))
|
||||
|
||||
|
|
@ -208,7 +206,7 @@ def test_start_metrics_server_process_rejects_metrics_from_another_service_on_th
|
|||
assert PID_HEADER not in impostor_response.headers
|
||||
with (
|
||||
patch("atexit.register"),
|
||||
pytest.raises(MetricsServerStartupError, match=rf"exited with code [1-9]\d* before serving 127.0.0.1:{port}"),
|
||||
pytest.raises(MetricsServerStartupError),
|
||||
):
|
||||
start_metrics_server_process(host="127.0.0.1", port=port, multiproc_dir=str(tmp_path))
|
||||
impostor.shutdown()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue