mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 1c9304a718 into 100529391a
This commit is contained in:
commit
eed07b47e8
11 changed files with 371 additions and 52 deletions
|
|
@ -81,8 +81,14 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
|
|||
def is_model_gemini_audio_model(self, model: str) -> bool:
|
||||
return "tts" in model
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> str:
|
||||
return "gemini"
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
sampling_params: Final = ("temperature", "top_p") if self.supports_sampling_params(model) else ()
|
||||
sampling_params: Final = (
|
||||
("temperature", "top_p") if self.supports_sampling_params(model, self.custom_llm_provider) else ()
|
||||
)
|
||||
supported_params: Final = [
|
||||
*sampling_params,
|
||||
"max_tokens",
|
||||
|
|
|
|||
|
|
@ -684,9 +684,30 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows(
|
|||
)
|
||||
|
||||
|
||||
_GOOGLE_MODEL_NAMESPACES: Final = ("vertex_ai/", "gemini/")
|
||||
|
||||
|
||||
def _configured_deployment_model(litellm_params: Mapping[str, object]) -> str | None:
|
||||
configured_model: Final = litellm_params.get("model")
|
||||
return configured_model if isinstance(configured_model, str) else None
|
||||
|
||||
|
||||
def _batch_row_model(deployment_model: str | None, row_model: object) -> str:
|
||||
"""The Gemini id a batch row is shaped and gated for: the deployment's model, which the stored GCS path
|
||||
binds the job to, else the row's own id for a direct SDK upload that names no deployment. The Google
|
||||
namespace comes off either way, as ``get_llm_provider`` does for a live call, so a ``gemini/`` id copied
|
||||
from an AI Studio call lands on its Vertex row instead of past every row."""
|
||||
chosen: Final = deployment_model if deployment_model else row_model
|
||||
if not isinstance(chosen, str):
|
||||
raise TypeError("`body.model` on batch requests must be a string, but was missing or not a string")
|
||||
namespace: Final = next((ns for ns in _GOOGLE_MODEL_NAMESPACES if chosen.startswith(ns)), "")
|
||||
return chosen.removeprefix(namespace)
|
||||
|
||||
|
||||
def _openai_batch_jsonl_entry_to_vertex_rows(
|
||||
openai_entry: dict[str, Any],
|
||||
map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
|
||||
deployment_model: str | None = None,
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
"""
|
||||
Transforms a single OpenAI JSONL batch entry into the Vertex rows it maps to.
|
||||
|
|
@ -706,10 +727,11 @@ def _openai_batch_jsonl_entry_to_vertex_rows(
|
|||
if _is_responses_batch_entry(openai_entry)
|
||||
else openai_request_body
|
||||
)
|
||||
row_body: Final = {**chat_request_body, "model": _batch_row_model(deployment_model, chat_request_body.get("model"))}
|
||||
vertex_request_body: Final = transform_request_body(
|
||||
messages=map_developer_role_to_system_role(chat_request_body.get("messages", [])),
|
||||
model=chat_request_body.get("model", ""),
|
||||
optional_params=map_openai_to_vertex_params(chat_request_body),
|
||||
messages=map_developer_role_to_system_role(row_body.get("messages", [])),
|
||||
model=row_body["model"],
|
||||
optional_params=map_openai_to_vertex_params(row_body),
|
||||
custom_llm_provider="vertex_ai",
|
||||
litellm_params={},
|
||||
cached_content=None,
|
||||
|
|
@ -859,15 +881,19 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream):
|
|||
self,
|
||||
openai_file_content: FileTypes,
|
||||
map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
|
||||
deployment_model: str | None = None,
|
||||
) -> None:
|
||||
self._openai_file_content = openai_file_content
|
||||
self._map_openai_to_vertex_params = map_openai_to_vertex_params
|
||||
self._deployment_model = deployment_model
|
||||
|
||||
def _iter_vertex_jsonl_chunks(self) -> Iterator[bytes]:
|
||||
first = True
|
||||
for lineno, entry in _iter_openai_jsonl_entries(self._openai_file_content):
|
||||
try:
|
||||
vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(entry, self._map_openai_to_vertex_params)
|
||||
vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(
|
||||
entry, self._map_openai_to_vertex_params, self._deployment_model
|
||||
)
|
||||
except Exception as e:
|
||||
raise VertexAIError(
|
||||
status_code=400,
|
||||
|
|
@ -1053,7 +1079,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
api_key: str | None,
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
litellm_params: dict[str, object],
|
||||
data: CreateFileRequest,
|
||||
) -> str:
|
||||
"""
|
||||
|
|
@ -1077,11 +1103,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
raise ValueError("file is required")
|
||||
if purpose is None:
|
||||
raise ValueError("purpose is required")
|
||||
configured_model: Final = litellm_params.get("model")
|
||||
object_name = self.get_object_name(
|
||||
file_data,
|
||||
purpose,
|
||||
deployment_model=configured_model if isinstance(configured_model, str) else None,
|
||||
deployment_model=_configured_deployment_model(litellm_params),
|
||||
passthrough=is_passthrough_batch_upload(data, litellm_params),
|
||||
)
|
||||
if object_prefix:
|
||||
|
|
@ -1120,15 +1145,16 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
sampling_params: Final = tuple(
|
||||
p for p in ("temperature", "top_p", "top_k") if openai_request_body.get(p) is not None
|
||||
)
|
||||
if sampling_params and not VertexGeminiConfig.supports_sampling_params(model):
|
||||
drop_params: Final = drops_unsupported_params(openai_request_body.get("drop_params", deployment_drop_params))
|
||||
if sampling_params and not VertexGeminiConfig.supports_sampling_params(model, "vertex_ai"):
|
||||
VertexGeminiConfig.reject_unsupported_sampling_param(
|
||||
model=model, param=", ".join(sampling_params), drop_params=False
|
||||
model=model, param=", ".join(sampling_params), drop_params=drop_params
|
||||
)
|
||||
vertex_params: Final = config.map_openai_params(
|
||||
model=model,
|
||||
non_default_params=openai_request_body,
|
||||
optional_params={},
|
||||
drop_params=drops_unsupported_params(openai_request_body.get("drop_params", deployment_drop_params)),
|
||||
drop_params=drop_params,
|
||||
allow_integer_enums=False,
|
||||
)
|
||||
return vertex_params
|
||||
|
|
@ -1138,7 +1164,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
model: str,
|
||||
create_file_data: CreateFileRequest,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
litellm_params: dict[str, object],
|
||||
) -> bytes | str | dict:
|
||||
"""
|
||||
2 Cases:
|
||||
|
|
@ -1170,6 +1196,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
body_stream=_OpenAIToVertexBatchUploadStream(
|
||||
file_data,
|
||||
partial(self._map_openai_to_vertex_params, deployment_drop_params=deployment_drop_params),
|
||||
_configured_deployment_model(litellm_params),
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1200,7 +1200,9 @@ def transform_request_body(
|
|||
|
||||
from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig
|
||||
|
||||
drops_top_k: Final = "top_k" in optional_params and not VertexGeminiConfig.supports_sampling_params(model)
|
||||
drops_top_k: Final = "top_k" in optional_params and not VertexGeminiConfig.supports_sampling_params(
|
||||
model, custom_llm_provider
|
||||
)
|
||||
if drops_top_k:
|
||||
request_params: Final = cast(Mapping[str, object], litellm_params) # cast-ok: untyped shared signature
|
||||
VertexGeminiConfig.reject_unsupported_sampling_param(
|
||||
|
|
|
|||
|
|
@ -702,23 +702,23 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
return False
|
||||
return True
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> str:
|
||||
return "vertex_ai"
|
||||
|
||||
@staticmethod
|
||||
def supports_sampling_params(model: str) -> bool:
|
||||
def supports_sampling_params(model: str, custom_llm_provider: str | None) -> bool:
|
||||
"""Whether Google still honors ``temperature`` / ``top_p`` / ``top_k`` for ``model``.
|
||||
|
||||
Driven by ``supports_sampling_params`` in the model map: Gemini 3.6 Flash, 3.5 Flash-Lite
|
||||
and later accept the fields but ignore them, and a later generation will 400 on them.
|
||||
A model the map says nothing about keeps forwarding them. ``temperature`` and ``top_p``
|
||||
leave ``get_supported_openai_params`` when the flag is off, so the shared unsupported-param
|
||||
check raises or drops them and ``allowed_openai_params`` still forwards them verbatim;
|
||||
``top_k`` is a provider-specific kwarg that bypasses that check and is gated while the
|
||||
request body is built."""
|
||||
rows: Final = cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: JSON rows by model id
|
||||
bare: Final = model.split("/")[-1]
|
||||
for candidate in (model, bare, f"vertex_ai/{bare}", f"gemini/{bare}"):
|
||||
if isinstance(flag := rows.get(candidate, {}).get("supports_sampling_params"), bool):
|
||||
return flag
|
||||
return True
|
||||
Read from the model map through the provider-aware lookup: an explicit ``supports_sampling_params``
|
||||
on the row wins, a chat, Responses or batch row without one and an id the map has never seen are
|
||||
answered by the ``gemini-sampling-params-ignored`` fallback generalization (Gemini 3.6 Flash, 3.5
|
||||
Flash-Lite and every later text model), and a row for any other mode keeps forwarding.
|
||||
``temperature`` and ``top_p`` leave ``get_supported_openai_params`` when the flag is off, so the
|
||||
shared unsupported-param check raises or drops them and ``allowed_openai_params`` still forwards
|
||||
them verbatim; ``top_k`` is a provider-specific kwarg that bypasses that check and is gated while
|
||||
the request body is built."""
|
||||
return not is_explicitly_disabled_factory(model, custom_llm_provider, "supports_sampling_params")
|
||||
|
||||
@staticmethod
|
||||
def reject_unsupported_sampling_param(model: str, param: str, drop_params: bool) -> None:
|
||||
|
|
@ -728,15 +728,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
return
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} does not support {param}: the model map flags it `supports_sampling_params: false` "
|
||||
"because Google ignores sampling params on this model. "
|
||||
"To drop unsupported params, set `litellm.drop_params = True`."
|
||||
f"{model} does not support {param}: Google ignores sampling params on this model, so LiteLLM "
|
||||
"treats it as `supports_sampling_params: false`. To drop unsupported params, set "
|
||||
"`litellm.drop_params = True`, or set `supports_sampling_params: true` in the model's "
|
||||
"model_info to forward them anyway."
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
sampling_params: Final = ("temperature", "top_p", "top_k") if self.supports_sampling_params(model) else ()
|
||||
sampling_params: Final = (
|
||||
("temperature", "top_p", "top_k") if self.supports_sampling_params(model, self.custom_llm_provider) else ()
|
||||
)
|
||||
supported_params: Final = [
|
||||
*sampling_params,
|
||||
"max_tokens",
|
||||
|
|
@ -1616,7 +1619,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
allow_integer_enums: bool = True,
|
||||
) -> dict:
|
||||
self._apply_include_server_side_tool_invocations(non_default_params, optional_params)
|
||||
supports_sampling_params: Final = self.supports_sampling_params(model)
|
||||
supports_sampling_params: Final = self.supports_sampling_params(model, self.custom_llm_provider)
|
||||
for param, value in non_default_params.items():
|
||||
if param in ("temperature", "top_p", "top_k") and not supports_sampling_params:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -61322,6 +61322,15 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_web_search": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "gemini-sampling-params-ignored",
|
||||
"pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:3[.-](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[4-9]|[1-9]\\d)(?:[.-]\\d{1,2})?(?!\\d)|3[.-]5-flash-lite)(?:-|$)",
|
||||
"fill_missing_for_providers": ["gemini", "vertex_ai", "vertex_ai-language-models"],
|
||||
"description": "Any Gemini text id at or past the version where Google stopped honoring temperature, top_p and top_k: 3.6 and every later 3.x minor, every major from 4 up with or without a minor, and 3.5 Flash-Lite by name, under any namespace (bare, gemini/, vertex_ai/, databricks-gemini-<major>-<minor>) and with any trailing variant tag, so gemini-4-argon and gemini-3.6-flash-preview match while gemini-3.5-flash, gemini-3.1-pro and the 2.x line do not. The same lookahead as gemini-chat-baseline keeps the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines out, since Google's notice covers the text models. Google announced with Gemini 3.6 Flash and 3.5 Flash-Lite that these models accept the sampling fields but ignore them and that a later generation will reject them (reported in #47015, 2026-10), so an unmapped id past the cut resolves to supports_sampling_params: false and a mapped Google row the map has not flagged yet is filled the same way; the Gemini and Vertex transformations turn that into a drop under drop_params or a 400 that names it. The fill is limited to Google's own providers because resellers route through transformations that do not read the flag. An explicit supports_sampling_params on a row, including one a deployment's model_info registers, always wins.",
|
||||
"model_info": {
|
||||
"supports_sampling_params": false
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
|
|||
|
|
@ -277,7 +277,7 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
_CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes}
|
||||
_BACKFILL_MODES: Final = frozenset({"chat", "responses"})
|
||||
_BACKFILL_MODES: Final = frozenset({"chat", "responses", "batch"})
|
||||
|
||||
# +-----------------------------------------------+
|
||||
# | |
|
||||
|
|
@ -6697,7 +6697,9 @@ def get_model_info_helper(
|
|||
_model_info: dict[str, Any] | None = lookup[1]
|
||||
|
||||
if _model_info is not None and key is not None and _model_info.get("mode", "chat") in _BACKFILL_MODES:
|
||||
fill_missing: Final = match_fill_missing_generalizations(key, _model_info.get("litellm_provider", ""))
|
||||
fill_missing: Final = match_fill_missing_generalizations(
|
||||
key, _model_info.get("litellm_provider") or model_cost_custom_llm_provider or ""
|
||||
)
|
||||
if fill_missing is not None:
|
||||
_model_info = {
|
||||
**{k: v for k, v in fill_missing.items() if k not in _model_info},
|
||||
|
|
@ -6987,6 +6989,7 @@ def get_model_info_helper(
|
|||
supports_web_search=_model_info.get("supports_web_search", None),
|
||||
supports_url_context=_model_info.get("supports_url_context", None),
|
||||
supports_reasoning=_model_info.get("supports_reasoning", None),
|
||||
supports_sampling_params=_model_info.get("supports_sampling_params", None),
|
||||
supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None),
|
||||
supports_legacy_thinking=_model_info.get("supports_legacy_thinking", None),
|
||||
thinking_always_on=_model_info.get("thinking_always_on", None),
|
||||
|
|
|
|||
|
|
@ -61322,6 +61322,15 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_web_search": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "gemini-sampling-params-ignored",
|
||||
"pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:3[.-](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[4-9]|[1-9]\\d)(?:[.-]\\d{1,2})?(?!\\d)|3[.-]5-flash-lite)(?:-|$)",
|
||||
"fill_missing_for_providers": ["gemini", "vertex_ai", "vertex_ai-language-models"],
|
||||
"description": "Any Gemini text id at or past the version where Google stopped honoring temperature, top_p and top_k: 3.6 and every later 3.x minor, every major from 4 up with or without a minor, and 3.5 Flash-Lite by name, under any namespace (bare, gemini/, vertex_ai/, databricks-gemini-<major>-<minor>) and with any trailing variant tag, so gemini-4-argon and gemini-3.6-flash-preview match while gemini-3.5-flash, gemini-3.1-pro and the 2.x line do not. The same lookahead as gemini-chat-baseline keeps the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines out, since Google's notice covers the text models. Google announced with Gemini 3.6 Flash and 3.5 Flash-Lite that these models accept the sampling fields but ignore them and that a later generation will reject them (reported in #47015, 2026-10), so an unmapped id past the cut resolves to supports_sampling_params: false and a mapped Google row the map has not flagged yet is filled the same way; the Gemini and Vertex transformations turn that into a drop under drop_params or a 400 that names it. The fill is limited to Google's own providers because resellers route through transformations that do not read the flag. An explicit supports_sampling_params on a row, including one a deployment's model_info registers, always wins.",
|
||||
"model_info": {
|
||||
"supports_sampling_params": false
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
|
|||
|
|
@ -390,6 +390,17 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp
|
|||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
},
|
||||
"acme-batch": {
|
||||
"input_cost_per_token": 9e-6,
|
||||
"output_cost_per_token": 1e-5,
|
||||
"litellm_provider": "openai",
|
||||
"mode": "batch",
|
||||
},
|
||||
"acme-unowned": {
|
||||
"input_cost_per_token": 2e-6,
|
||||
"output_cost_per_token": 3e-6,
|
||||
"mode": "chat",
|
||||
},
|
||||
},
|
||||
)
|
||||
restore_generalizations(
|
||||
|
|
@ -422,6 +433,14 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp
|
|||
image = litellm.get_model_info("acme-image", custom_llm_provider="openai")
|
||||
assert image.get("supports_reasoning") is None
|
||||
|
||||
batch = litellm.get_model_info("acme-batch", custom_llm_provider="openai")
|
||||
assert batch["supports_reasoning"] is True
|
||||
assert batch["max_tokens"] == 5
|
||||
|
||||
unowned = litellm.get_model_info("acme-unowned", custom_llm_provider="openai")
|
||||
assert unowned["supports_reasoning"] is True
|
||||
assert litellm.get_model_info("acme-unowned", custom_llm_provider="openrouter").get("supports_reasoning") is None
|
||||
|
||||
restore_generalizations(
|
||||
[{"name": "acme-backfill", "pattern": r"^acme-", "model_info": {"supports_reasoning": True, "max_tokens": 5}}]
|
||||
)
|
||||
|
|
@ -506,7 +525,7 @@ def test_shipped_gemini_chat_baseline_skips_non_chat_and_pre_2_5_ids(shipped_cos
|
|||
"databricks-gemini-3-1-flash-image",
|
||||
"openrouter/google/gemini-2.0-flash-001",
|
||||
):
|
||||
assert match_capability_generalizations(model) is None, model
|
||||
assert "mode" not in (match_capability_generalizations(model) or {}), model
|
||||
|
||||
|
||||
def test_shipped_gemini_chat_baseline_keeps_reasoning_effort_on_unmapped_model(shipped_cost_map):
|
||||
|
|
@ -985,3 +1004,60 @@ def test_shipped_tool_search_rule_fills_mapped_claude_entries_without_flag(shipp
|
|||
assert match_fill_missing_generalizations("claude-opus-5", "bedrock")["supports_tool_search"] is True
|
||||
assert match_fill_missing_generalizations("claude-opus-5", "azure_ai")["supports_tool_search"] is True
|
||||
assert match_fill_missing_generalizations("claude-opus-5", "perplexity") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,provider,sampling",
|
||||
[
|
||||
("gemini-3.6-flash-zeta", "gemini", False),
|
||||
("gemini-3.5-flash-lite-zeta", "vertex_ai", False),
|
||||
("gemini-3.10-pro-zeta", "gemini", False),
|
||||
("gemini-4-zeta", "gemini", False),
|
||||
("gemini-4.1-flash-zeta", "vertex_ai", False),
|
||||
("gemini-10-zeta", "gemini", False),
|
||||
("vertex_ai/gemini-4-zeta", None, False),
|
||||
("gemini-3.5-flash-zeta", "gemini", None),
|
||||
("gemini-3.5-pro-zeta", "vertex_ai", None),
|
||||
("gemini-3.1-pro-zeta", "gemini", None),
|
||||
("gemini-2.5-flash-zeta", "gemini", None),
|
||||
],
|
||||
)
|
||||
def test_shipped_gemini_sampling_rule_version_boundaries(shipped_cost_map, model, provider, sampling):
|
||||
"""The gemini-sampling-params-ignored rule marks Gemini 3.6 and every later 3.x minor, every 4+ major
|
||||
and 3.5 Flash-Lite as not honoring temperature, top_p and top_k, and leaves 3.5 Flash, 3.5 Pro, 3.1 and
|
||||
2.5 ids without an opinion"""
|
||||
assert model not in litellm.model_cost
|
||||
info = litellm.get_model_info(model, custom_llm_provider=provider)
|
||||
assert info.get("supports_sampling_params") is sampling, model
|
||||
|
||||
|
||||
def test_shipped_gemini_sampling_rule_skips_non_text_ids(shipped_cost_map):
|
||||
for model in (
|
||||
"gemini/gemini-4-flash-image",
|
||||
"gemini/gemini-3.8-flash-tts",
|
||||
"gemini/gemini-4-flash-live-preview",
|
||||
"gemini/gemini-3.6-flash-native-audio",
|
||||
"gemini/gemini-embedding-4",
|
||||
"gemini/gemini-4-flash-transcribe",
|
||||
"gemini/gemini-3.6-flash-live-translate-preview",
|
||||
"gemini/gemini-4-computer-use-preview",
|
||||
"gemini/gemini-zeta-9",
|
||||
):
|
||||
assert (match_capability_generalizations(model) or {}).get("supports_sampling_params") is None, model
|
||||
|
||||
|
||||
def test_shipped_gemini_sampling_rule_fills_mapped_google_rows_without_the_flag(shipped_cost_map):
|
||||
"""A mapped Gemini row past Google's cut reads as supports_sampling_params: false on AI Studio and Vertex
|
||||
whether or not someone flagged it by hand, and a reseller copy of the same model is left alone"""
|
||||
for key, model, provider in (
|
||||
("gemini/gemini-4-argon", "gemini-4-argon", "gemini"),
|
||||
("vertex_ai/gemini-4-argon", "gemini-4-argon", "vertex_ai"),
|
||||
("gemini/gemini-3.6-flash", "gemini-3.6-flash", "gemini"),
|
||||
):
|
||||
assert key in litellm.model_cost
|
||||
assert litellm.get_model_info(model, custom_llm_provider=provider)["supports_sampling_params"] is False, key
|
||||
|
||||
assert match_fill_missing_generalizations("gemini-4-argon", "vertex_ai-language-models") == {
|
||||
"supports_sampling_params": False
|
||||
}
|
||||
assert match_fill_missing_generalizations("gemini-4-argon", "github_copilot") is None
|
||||
|
|
|
|||
|
|
@ -5,25 +5,26 @@ Includes tests for Vertex AI batch output transformation to OpenAI format.
|
|||
|
||||
import json
|
||||
import urllib.parse
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from collections.abc import Iterable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from openai.types.file_deleted import FileDeleted
|
||||
from pydantic import TypeAdapter
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import litellm
|
||||
from litellm.llms.vertex_ai.common_utils import VertexAIError
|
||||
from litellm.llms.vertex_ai.files.transformation import (
|
||||
VertexAIFilesConfig,
|
||||
_get_litellm_batch_custom_id_from_labels,
|
||||
_openai_batch_jsonl_entry_to_vertex_rows,
|
||||
_sanitize_gcp_label_value,
|
||||
)
|
||||
from litellm.types.llms.openai import OpenAIFileObject, HttpxBinaryResponseContent
|
||||
from openai.types.file_deleted import FileDeleted
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIFileObject
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -171,9 +172,7 @@ class TestBatchObjectNaming:
|
|||
def test_should_store_fine_tuned_endpoint_under_endpoints_path(self, config):
|
||||
"""A numeric endpoint id must not be filed under publishers/google/models/gemini/<id>,
|
||||
which the batch transformation later mangles into a nonexistent publisher model (LIT-6899)."""
|
||||
object_name = config._get_gcs_object_name_from_batch_jsonl(
|
||||
[{"body": {"model": "gemini/7768560373388541952"}}]
|
||||
)
|
||||
object_name = config._get_gcs_object_name_from_batch_jsonl([{"body": {"model": "gemini/7768560373388541952"}}])
|
||||
assert object_name.startswith("litellm-vertex-files/endpoints/7768560373388541952/")
|
||||
assert "publishers" not in object_name
|
||||
|
||||
|
|
@ -865,7 +864,6 @@ class TestVertexBatchOutputTransformation:
|
|||
assert config._try_transform_vertex_batch_output_to_openai(binary) == binary
|
||||
|
||||
|
||||
|
||||
class TestTryTransformDoesNotMutateCallerLoggingObj:
|
||||
"""Regression tests: _try_transform_vertex_batch_output_to_openai must not mutate
|
||||
the caller's logging_obj (model, start_time, optional_params)."""
|
||||
|
|
@ -2045,3 +2043,110 @@ def test_batch_row_for_an_unflagged_gemini_model_keeps_the_knobs(monkeypatch: py
|
|||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
(row,) = _wrap_entries([_chat_entry("gemini-2.5-flash", _SAMPLING_KNOBS)])
|
||||
assert _generation_config(row) == {"max_output_tokens": 5, **_SAMPLING_KNOBS}
|
||||
|
||||
|
||||
def _stamp_batch_deployment_row(monkeypatch: pytest.MonkeyPatch, model: str) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost, model, {"mode": "batch", "litellm_provider": "vertex_ai"})
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
def test_batch_row_for_a_gemini_4_batch_deployment_raises_a_400_naming_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The proxy stamps a batch deployment's `mode: batch` onto the backend model's map row, so a Gemini
|
||||
model whose name is past Google's cut must still be filled by the shipped sampling rule and not pass as a
|
||||
non-chat row"""
|
||||
_stamp_batch_deployment_row(monkeypatch, "vertex_ai/gemini-4-zeta")
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=r"temperature, top_p, top_k.*drop_params"):
|
||||
_wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)])
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
def test_batch_row_for_a_gemini_4_batch_deployment_drops_the_knobs_under_global_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_stamp_batch_deployment_row(monkeypatch, "vertex_ai/gemini-4-zeta")
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
(row,) = _wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)])
|
||||
assert _generation_config(row) == {"max_output_tokens": 5}
|
||||
|
||||
|
||||
def _google_rows_flagged_only_on_vertex(monkeypatch: pytest.MonkeyPatch, model_name: str) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost, f"gemini/{model_name}", {"litellm_provider": "gemini", "mode": "chat"})
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
f"vertex_ai/{model_name}",
|
||||
{"litellm_provider": "vertex_ai", "mode": "chat", "supports_sampling_params": False},
|
||||
)
|
||||
|
||||
|
||||
def _upload_rows(
|
||||
entries: Iterable[Mapping[str, object]], deployment_model: str | None, deployment_drop_params: bool | None = None
|
||||
) -> list[object]:
|
||||
"""The Vertex rows a jsonl upload stores, built through the create-file transform the way the proxy's
|
||||
`?model=<deployment>` branch runs it, with the deployment's model and drop_params in litellm_params"""
|
||||
jsonl: Final = "\n".join(json.dumps(dict(entry)) for entry in entries).encode()
|
||||
litellm_params: Final[dict[str, object]] = {
|
||||
**({"model": deployment_model} if deployment_model else {}),
|
||||
**({"drop_params": deployment_drop_params} if deployment_drop_params is not None else {}),
|
||||
}
|
||||
request: Final = VertexAIFilesConfig().transform_create_file_request(
|
||||
model="",
|
||||
create_file_data={"file": ("batch.jsonl", jsonl, "application/jsonl"), "purpose": "batch"},
|
||||
optional_params={},
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
assert isinstance(request, dict)
|
||||
body: Final = b"".join(request["streaming_media_upload"]["body_stream"].iter_bytes())
|
||||
return [json.loads(line) for line in body.decode().splitlines()]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deployment_model", ["vertex_ai/gemini-zeta-9", None], ids=["proxy_deployment", "sdk_upload"])
|
||||
def test_batch_row_keeping_a_gemini_prefix_is_judged_by_its_vertex_row(
|
||||
deployment_model: str | None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""A row copied from an AI Studio call keeps its `gemini/` prefix but runs on Vertex, so the Vertex row's flag
|
||||
decides. Looked up under that prefix on the Vertex route it found no row and fell open, so the knobs were
|
||||
stored even though the model ignores them"""
|
||||
_google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9")
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
with pytest.raises(VertexAIError, match=r"temperature, top_p, top_k.*drop_params"):
|
||||
_upload_rows([_chat_entry("gemini/gemini-zeta-9", _SAMPLING_KNOBS)], deployment_model)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deployment_model", ["vertex_ai/gemini-zeta-9", None], ids=["proxy_deployment", "sdk_upload"])
|
||||
def test_batch_row_keeping_a_gemini_prefix_drops_the_knobs_under_global_drop_params(
|
||||
deployment_model: str | None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9")
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
(row,) = _upload_rows([_chat_entry("gemini/gemini-zeta-9", _SAMPLING_KNOBS)], deployment_model)
|
||||
assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5}
|
||||
|
||||
|
||||
def test_batch_row_for_a_flagged_gemini_model_drops_the_knobs_under_the_deployment_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A batch deployment with `drop_params: true` in its litellm_params gets the knobs dropped the way a
|
||||
real-time call on that deployment does, instead of a 400 that only the global switch could avoid"""
|
||||
_google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9")
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
(row,) = _upload_rows(
|
||||
[_chat_entry("gemini-zeta-9", _SAMPLING_KNOBS)], "vertex_ai/gemini-zeta-9", deployment_drop_params=True
|
||||
)
|
||||
assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5}
|
||||
|
||||
|
||||
def test_batch_row_is_judged_by_the_deployment_model_it_runs_on_not_the_id_it_was_written_with(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The stored GCS path binds the batch job to the deployment's model, so a row written for an older id that
|
||||
still takes the knobs is gated for the model that actually executes it"""
|
||||
_google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9")
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost, "vertex_ai/gemini-zeta-1", {"litellm_provider": "vertex_ai", "mode": "chat"}
|
||||
)
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
with pytest.raises(VertexAIError, match=r"gemini-zeta-9 does not support temperature, top_p, top_k"):
|
||||
_upload_rows([_chat_entry("gemini-zeta-1", _SAMPLING_KNOBS)], "vertex_ai/gemini-zeta-9")
|
||||
|
|
|
|||
|
|
@ -4116,12 +4116,13 @@ def _gemini_request_body(
|
|||
custom_llm_provider: Literal["vertex_ai", "gemini"],
|
||||
optional_params: dict[str, object],
|
||||
litellm_params: dict[str, object],
|
||||
model: str = "gemini-zeta-9",
|
||||
) -> RequestBody:
|
||||
from litellm.llms.vertex_ai.gemini.transformation import transform_request_body
|
||||
|
||||
return transform_request_body(
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
model="gemini-zeta-9",
|
||||
model=model,
|
||||
optional_params=optional_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
|
|
@ -4217,13 +4218,21 @@ def test_gemini_keeps_forwarding_sampling_params_unless_the_model_map_flags_them
|
|||
assert body["generationConfig"] == _SAMPLING_PARAMS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_key", ["gemini-zeta-9", "vertex_ai/gemini-zeta-9", "gemini/gemini-zeta-9"])
|
||||
def test_gemini_sampling_flag_is_read_from_whichever_provider_row_carries_it(
|
||||
model_key: str, monkeypatch: pytest.MonkeyPatch
|
||||
@pytest.mark.parametrize(
|
||||
("config_cls", "model_key"),
|
||||
[
|
||||
("vertex", "gemini-zeta-9"),
|
||||
("vertex", "vertex_ai/gemini-zeta-9"),
|
||||
("studio", "gemini-zeta-9"),
|
||||
("studio", "gemini/gemini-zeta-9"),
|
||||
],
|
||||
)
|
||||
def test_gemini_sampling_flag_is_read_from_the_row_of_the_route_serving_the_call(
|
||||
config_cls: Literal["vertex", "studio"], model_key: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost, model_key, {"supports_sampling_params": False})
|
||||
|
||||
result: Final = VertexGeminiConfig().map_openai_params(
|
||||
result: Final = _gemini_config(config_cls).map_openai_params(
|
||||
non_default_params={"temperature": 0.3},
|
||||
optional_params={},
|
||||
model="gemini-zeta-9",
|
||||
|
|
@ -4268,6 +4277,76 @@ def test_get_optional_params_drops_sampling_params_for_a_flagged_gemini_model_un
|
|||
assert optional_params["max_output_tokens"] == 10
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize("custom_llm_provider", ["vertex_ai", "gemini"])
|
||||
@pytest.mark.parametrize(
|
||||
("model", "forwarded"),
|
||||
[
|
||||
("gemini-3.6-flash-zeta", False),
|
||||
("gemini-3.5-flash-lite-zeta", False),
|
||||
("gemini-3.7-pro-zeta", False),
|
||||
("gemini-4-zeta", False),
|
||||
("gemini-4.1-flash-zeta", False),
|
||||
("gemini-10-zeta", False),
|
||||
("gemini-3.5-flash-zeta", True),
|
||||
("gemini-3.5-pro-zeta", True),
|
||||
("gemini-3.1-pro-zeta", True),
|
||||
("gemini-2.5-flash-zeta", True),
|
||||
("gemini-zeta-9", True),
|
||||
],
|
||||
)
|
||||
def test_gemini_model_unknown_to_the_map_is_judged_by_the_shipped_sampling_rule(
|
||||
custom_llm_provider: Literal["vertex_ai", "gemini"], model: str, forwarded: bool
|
||||
) -> None:
|
||||
"""Google ignores the knobs on Gemini 3.6 Flash, 3.5 Flash-Lite and everything later, so a model newer
|
||||
than the map gets the same drop-or-400 contract from the gemini-sampling-params-ignored fallback
|
||||
generalization instead of silently forwarding (#47015 follow-up)"""
|
||||
from litellm.utils import get_optional_params
|
||||
|
||||
optional_params: Final = get_optional_params(
|
||||
model=model, custom_llm_provider=custom_llm_provider, max_tokens=10, drop_params=True, **_SAMPLING_PARAMS
|
||||
)
|
||||
body: Final = _gemini_request_body(custom_llm_provider, dict(optional_params), {"drop_params": True}, model=model)
|
||||
|
||||
knobs: Final = _SAMPLING_PARAMS if forwarded else {}
|
||||
assert body["generationConfig"] == {**knobs, "max_output_tokens": 10}, model
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize(
|
||||
("row", "forwarded"),
|
||||
[
|
||||
({"mode": "chat"}, False),
|
||||
({}, False),
|
||||
({"mode": "batch"}, False),
|
||||
({"mode": "audio_speech"}, True),
|
||||
({"mode": "chat", "supports_sampling_params": True}, True),
|
||||
],
|
||||
ids=[
|
||||
"chat_row_without_flag",
|
||||
"row_without_mode",
|
||||
"batch_deployment_row_without_flag",
|
||||
"non_chat_row_without_flag",
|
||||
"explicit_true",
|
||||
],
|
||||
)
|
||||
def test_map_row_for_a_gemini_4_model_decides_before_the_shipped_sampling_rule(
|
||||
row: dict[str, object], forwarded: bool, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""An explicit flag always wins, a row silent on it or stamped `mode: batch` by a proxy batch deployment
|
||||
is filled by the shipped rule, and a row for another mode keeps forwarding because Google's notice covers
|
||||
the text models"""
|
||||
from litellm.utils import get_optional_params
|
||||
|
||||
monkeypatch.setitem(litellm.model_cost, "gemini-4-zeta", row)
|
||||
|
||||
optional_params: Final = get_optional_params(
|
||||
model="gemini-4-zeta", custom_llm_provider="gemini", temperature=0.3, drop_params=True
|
||||
)
|
||||
|
||||
assert ("temperature" in optional_params) is forwarded
|
||||
|
||||
|
||||
def test_media_resolution_from_detail_parameter():
|
||||
"""Test that OpenAI's detail parameter is correctly mapped to media_resolution"""
|
||||
from litellm.llms.vertex_ai.gemini.transformation import (
|
||||
|
|
@ -7308,7 +7387,7 @@ def test_gemini_3_sampling_deprecation_is_logged_once_when_first_calls_race(capl
|
|||
def test_gemini_3_sampling_deprecation_for_unmapped_names_does_not_hide_mapped_models(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
unmapped: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(100))
|
||||
unmapped: Final = tuple(f"gemini-3.5-flash-{uuid.uuid4().hex}" for _ in range(100))
|
||||
mapped: Final = _mapped_gemini_3_models()[0]
|
||||
config: Final = VertexGeminiConfig()
|
||||
|
||||
|
|
@ -7351,7 +7430,7 @@ def test_gemini_3_sampling_deprecation_names_models_cataloged_only_under_vertex_
|
|||
def test_gemini_3_sampling_deprecation_for_names_priced_at_runtime_shares_one_notice(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
priced: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(20))
|
||||
priced: Final = tuple(f"gemini-3.5-flash-{uuid.uuid4().hex}" for _ in range(20))
|
||||
for name in priced:
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost, # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed
|
||||
|
|
|
|||
|
|
@ -585,7 +585,7 @@ def test_should_not_pollute_shared_key_with_custom_nonzero_pricing():
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag", ["supports_regex_lookaround", "supports_assistant_prefill", "supports_sampling_params"])
|
||||
@pytest.mark.parametrize("flag", ["supports_regex_lookaround", "supports_assistant_prefill"])
|
||||
def test_capability_flag_stays_on_the_deployment_that_set_it(flag: str) -> None:
|
||||
"""A deployment's capability override must not land on the shared
|
||||
``{provider}/{model}`` key, or every sibling deployment of that model would inherit it."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue