mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(vertex_ai): honor a batch deployment's drop_params on flagged sampling knobs
A batch deployment with `litellm_params: drop_params: true` still rejected rows that carried temperature, top_p or top_k for a Gemini model flagged `supports_sampling_params: false`, because the batch mapper passed `drop_params=False` to the rejection check while the live chat path reads the deployment's drop_params. Both gates now read the same flag, so the knobs are dropped instead of the upload failing with a 400 The upload stream's `deployment_model` argument defaults to None again, so the two-argument callers (direct SDK uploads and the existing batch tests) keep working, and a row whose body.model is missing or not a string is still reported as a 400 naming the line when the deployment names no model
This commit is contained in:
parent
b2dddd4012
commit
62ff5d2bd9
2 changed files with 29 additions and 7 deletions
|
|
@ -697,7 +697,9 @@ def _batch_row_model(deployment_model: str | None, row_model: object) -> str:
|
|||
binds the job to, else the row's own id for a direct SDK upload that names no deployment. The Google
|
||||
namespace comes off either way, as ``get_llm_provider`` does for a live call, so a ``gemini/`` id copied
|
||||
from an AI Studio call lands on its Vertex row instead of past every row."""
|
||||
chosen: Final = deployment_model if deployment_model else row_model if isinstance(row_model, str) else ""
|
||||
chosen: Final = deployment_model if deployment_model else row_model
|
||||
if not isinstance(chosen, str):
|
||||
raise TypeError("`body.model` on batch requests must be a string, but was missing or not a string")
|
||||
namespace: Final = next((ns for ns in _GOOGLE_MODEL_NAMESPACES if chosen.startswith(ns)), "")
|
||||
return chosen.removeprefix(namespace)
|
||||
|
||||
|
|
@ -879,7 +881,7 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream):
|
|||
self,
|
||||
openai_file_content: FileTypes,
|
||||
map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
|
||||
deployment_model: str | None,
|
||||
deployment_model: str | None = None,
|
||||
) -> None:
|
||||
self._openai_file_content = openai_file_content
|
||||
self._map_openai_to_vertex_params = map_openai_to_vertex_params
|
||||
|
|
@ -1143,15 +1145,16 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
sampling_params: Final = tuple(
|
||||
p for p in ("temperature", "top_p", "top_k") if openai_request_body.get(p) is not None
|
||||
)
|
||||
drop_params: Final = drops_unsupported_params(openai_request_body.get("drop_params", deployment_drop_params))
|
||||
if sampling_params and not VertexGeminiConfig.supports_sampling_params(model, "vertex_ai"):
|
||||
VertexGeminiConfig.reject_unsupported_sampling_param(
|
||||
model=model, param=", ".join(sampling_params), drop_params=False
|
||||
model=model, param=", ".join(sampling_params), drop_params=drop_params
|
||||
)
|
||||
vertex_params: Final = config.map_openai_params(
|
||||
model=model,
|
||||
non_default_params=openai_request_body,
|
||||
optional_params={},
|
||||
drop_params=drops_unsupported_params(openai_request_body.get("drop_params", deployment_drop_params)),
|
||||
drop_params=drop_params,
|
||||
allow_integer_enums=False,
|
||||
)
|
||||
return vertex_params
|
||||
|
|
|
|||
|
|
@ -2081,15 +2081,21 @@ def _google_rows_flagged_only_on_vertex(monkeypatch: pytest.MonkeyPatch, model_n
|
|||
)
|
||||
|
||||
|
||||
def _upload_rows(entries: Iterable[Mapping[str, object]], deployment_model: str | None) -> list[object]:
|
||||
def _upload_rows(
|
||||
entries: Iterable[Mapping[str, object]], deployment_model: str | None, deployment_drop_params: bool | None = None
|
||||
) -> list[object]:
|
||||
"""The Vertex rows a jsonl upload stores, built through the create-file transform the way the proxy's
|
||||
`?model=<deployment>` branch runs it, with the deployment's model in litellm_params"""
|
||||
`?model=<deployment>` branch runs it, with the deployment's model and drop_params in litellm_params"""
|
||||
jsonl: Final = "\n".join(json.dumps(dict(entry)) for entry in entries).encode()
|
||||
litellm_params: Final[dict[str, object]] = {
|
||||
**({"model": deployment_model} if deployment_model else {}),
|
||||
**({"drop_params": deployment_drop_params} if deployment_drop_params is not None else {}),
|
||||
}
|
||||
request: Final = VertexAIFilesConfig().transform_create_file_request(
|
||||
model="",
|
||||
create_file_data={"file": ("batch.jsonl", jsonl, "application/jsonl"), "purpose": "batch"},
|
||||
optional_params={},
|
||||
litellm_params={"model": deployment_model} if deployment_model else {},
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
assert isinstance(request, dict)
|
||||
body: Final = b"".join(request["streaming_media_upload"]["body_stream"].iter_bytes())
|
||||
|
|
@ -2119,6 +2125,19 @@ def test_batch_row_keeping_a_gemini_prefix_drops_the_knobs_under_global_drop_par
|
|||
assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5}
|
||||
|
||||
|
||||
def test_batch_row_for_a_flagged_gemini_model_drops_the_knobs_under_the_deployment_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A batch deployment with `drop_params: true` in its litellm_params gets the knobs dropped the way a
|
||||
real-time call on that deployment does, instead of a 400 that only the global switch could avoid"""
|
||||
_google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9")
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
(row,) = _upload_rows(
|
||||
[_chat_entry("gemini-zeta-9", _SAMPLING_KNOBS)], "vertex_ai/gemini-zeta-9", deployment_drop_params=True
|
||||
)
|
||||
assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5}
|
||||
|
||||
|
||||
def test_batch_row_is_judged_by_the_deployment_model_it_runs_on_not_the_id_it_was_written_with(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue