mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(router): merge default metadata under the request's instead of replacing it
This commit is contained in:
parent
b91fc92b6d
commit
f4015d9f51
4 changed files with 197 additions and 2 deletions
|
|
@ -118,6 +118,7 @@ from litellm.litellm_core_utils.batch_utils import (
|
|||
)
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
as_str_mapping,
|
||||
coerce_token_limit,
|
||||
get_litellm_metadata_from_kwargs,
|
||||
get_metadata_variable_name_from_kwargs,
|
||||
|
|
@ -264,6 +265,7 @@ from litellm.router_utils.common_utils import (
|
|||
format_no_fallback_group_message,
|
||||
get_request_team_id,
|
||||
is_proxy_admin_request,
|
||||
metadata_defaults_under_request,
|
||||
provider_for_generic_call,
|
||||
request_user_api_key_auth,
|
||||
resolve_model_group_alias,
|
||||
|
|
@ -677,6 +679,7 @@ _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None)
|
|||
_ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
|
||||
_FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_REQUEST_METADATA_ADAPTER: Final = TypeAdapter(Mapping[object, object])
|
||||
_DEPLOYMENT_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | GenericLiteLLMParams]] = TypeAdapter(
|
||||
Mapping[str, object] | GenericLiteLLMParams
|
||||
)
|
||||
|
|
@ -5037,7 +5040,7 @@ class Router(DeprecatedRouterMembers):
|
|||
"""
|
||||
# 1) copy your defaults and pull out metadata
|
||||
defaults: Final = dict(without_retrieve_stream_defaults(self._core_default_litellm_params, kwargs))
|
||||
metadata_defaults: Final = defaults.pop("metadata", {}) or {}
|
||||
metadata_defaults: Final = as_str_mapping(defaults.pop("metadata", None)) or {}
|
||||
|
||||
# 2) add any non-metadata defaults that aren't already in kwargs
|
||||
for key, value in defaults.items():
|
||||
|
|
@ -5046,7 +5049,12 @@ class Router(DeprecatedRouterMembers):
|
|||
kwargs.setdefault(key, value)
|
||||
|
||||
# 3) merge in metadata, this handles inserting this as either "metadata" or "litellm_metadata"
|
||||
kwargs.setdefault(metadata_variable_name, {}).update(metadata_defaults)
|
||||
request_metadata: Final = _REQUEST_METADATA_ADAPTER.validate_python(
|
||||
kwargs.setdefault(metadata_variable_name, {})
|
||||
)
|
||||
kwargs[metadata_variable_name].update(
|
||||
metadata_defaults_under_request(defaults=metadata_defaults, request_metadata=request_metadata)
|
||||
)
|
||||
|
||||
def _handle_clientside_credential(
|
||||
self, deployment: dict, kwargs: dict, function_name: str | None = None
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import litellm
|
|||
from litellm._logging import verbose_logger, verbose_router_logger
|
||||
from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure
|
||||
|
|
@ -61,6 +62,32 @@ def without_router_only_kwargs(kwargs: Mapping[str, _V]) -> dict[str, _V]:
|
|||
return {key: value for key, value in kwargs.items() if key not in ROUTER_ONLY_CALL_KWARGS}
|
||||
|
||||
|
||||
_ROUTER_OWNED_METADATA_KEYS: Final = frozenset({"caching_groups"})
|
||||
|
||||
|
||||
def metadata_defaults_under_request(
|
||||
defaults: Mapping[str, object], request_metadata: Mapping[object, object]
|
||||
) -> Mapping[str, object]:
|
||||
return {
|
||||
key: (
|
||||
default
|
||||
if key in _ROUTER_OWNED_METADATA_KEYS
|
||||
else _request_value_over_default(default=default, requested=request_metadata.get(key))
|
||||
)
|
||||
for key, default in defaults.items()
|
||||
}
|
||||
|
||||
|
||||
def _request_value_over_default(default: object, requested: object) -> object:
|
||||
if requested is None:
|
||||
return default
|
||||
default_mapping: Final = as_str_mapping(default)
|
||||
requested_mapping: Final = as_str_mapping(requested)
|
||||
if default_mapping is None or requested_mapping is None:
|
||||
return requested
|
||||
return {**default_mapping, **requested_mapping}
|
||||
|
||||
|
||||
def as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]:
|
||||
if not isinstance(value, tuple):
|
||||
return ()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,85 @@
|
|||
import json
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy_process
|
||||
from integration._support.vertex import service_account_json
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
|
||||
|
||||
_MODEL_NAME: Final = "labelled-gemini"
|
||||
_PROJECT: Final = "scripted-project"
|
||||
_LOCATION: Final = "us-central1"
|
||||
_ROUTER_LABELS: Final = {"env": "prod"}
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_REPLY: Final = json.dumps(
|
||||
{
|
||||
"candidates": [{"content": {"role": "model", "parts": [{"text": "OK"}]}, "finishReason": "STOP"}],
|
||||
"usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 1, "totalTokenCount": 4},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def _gemini_peer(request: Request) -> Reply:
|
||||
if request.target.endswith(":generateContent"):
|
||||
return Reply(body=_REPLY)
|
||||
return Reply(status=404, body=json.dumps({"error": request.target}).encode())
|
||||
|
||||
|
||||
def _config(directory: Path, gateway: Gateway, api_base: str) -> Path:
|
||||
config: Final = directory / "router-default-requester-metadata.yaml"
|
||||
config.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": _MODEL_NAME,
|
||||
"litellm_params": {
|
||||
"model": "vertex_ai/gemini-3.8-flash",
|
||||
"api_base": api_base,
|
||||
"vertex_project": _PROJECT,
|
||||
"vertex_location": _LOCATION,
|
||||
"vertex_credentials": service_account_json(_PROJECT, gateway.upstream_url.rstrip("/")),
|
||||
},
|
||||
}
|
||||
],
|
||||
"router_settings": {"default_litellm_params": {"metadata": {"requester_metadata": _ROUTER_LABELS}}},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def _labels_sent(proxy: Gateway, wire: Wire, metadata: Mapping[str, JsonValue] | None) -> JsonValue:
|
||||
response: Final = proxy.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": _MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Return exactly OK."}],
|
||||
**({"metadata": dict(metadata)} if metadata is not None else {}),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
generations: Final = tuple(request for request in wire.drain() if request.target.endswith(":generateContent"))
|
||||
assert len(generations) == 1, generations
|
||||
return _JSON_OBJECT.validate_json(generations[0].body).get("labels")
|
||||
|
||||
|
||||
def test_router_default_requester_metadata_merges_under_each_callers_vertex_labels(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with (
|
||||
wire_server(_gemini_peer) as wire,
|
||||
owned_proxy_process(gateway, tmp_path, {}, config=_config(tmp_path, gateway, wire.url)) as owned,
|
||||
):
|
||||
proxy: Final = Gateway(owned.gateway.client, owned.gateway.key, gateway.upstream_url)
|
||||
assert _labels_sent(proxy, wire, {"team": "ai-platform"}) == {**_ROUTER_LABELS, "team": "ai-platform"}
|
||||
assert _labels_sent(proxy, wire, {"env": "staging"}) == {"env": "staging"}
|
||||
assert _labels_sent(proxy, wire, None) == _ROUTER_LABELS
|
||||
|
|
@ -20,6 +20,7 @@ from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels
|
||||
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
|
||||
from litellm.types.litellm_params import RoutingStrategyName
|
||||
from litellm.types.router import AlertingConfig, Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo
|
||||
|
|
@ -396,6 +397,80 @@ def test_update_kwargs_with_default_litellm_params(model_list):
|
|||
assert kwargs["metadata"]["key2"] == "value2"
|
||||
|
||||
|
||||
def _vertex_labels_router(metadata_defaults: Mapping[str, object]) -> Router:
|
||||
return Router(
|
||||
model_list=[{"model_name": "g", "litellm_params": {"model": "vertex_ai/gemini-3.8-flash"}}],
|
||||
default_litellm_params={"metadata": dict(metadata_defaults)},
|
||||
)
|
||||
|
||||
|
||||
def _vertex_labels_after_defaults(router: Router, request_metadata: dict, metadata_variable_name: str):
|
||||
kwargs: Final = {metadata_variable_name: request_metadata}
|
||||
router._update_kwargs_with_default_litellm_params(kwargs=kwargs, metadata_variable_name=metadata_variable_name)
|
||||
return pop_vertex_request_labels(optional_params={}, litellm_params=kwargs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("metadata_variable_name", ["metadata", "litellm_metadata"])
|
||||
@pytest.mark.parametrize(
|
||||
"request_labels, expected_labels",
|
||||
[
|
||||
({"litellm-team": "ai-platform"}, {"env": "prod", "litellm-team": "ai-platform"}),
|
||||
({"env": "dev", "litellm-team": "ai-platform"}, {"env": "dev", "litellm-team": "ai-platform"}),
|
||||
(None, {"env": "prod"}),
|
||||
({}, {"env": "prod"}),
|
||||
],
|
||||
)
|
||||
def test_default_requester_metadata_merges_under_request_labels(
|
||||
metadata_variable_name: str, request_labels: dict | None, expected_labels: dict
|
||||
):
|
||||
router: Final = _vertex_labels_router({"requester_metadata": {"env": "prod"}})
|
||||
request_metadata: Final = {} if request_labels is None else {"requester_metadata": request_labels}
|
||||
|
||||
labels: Final = _vertex_labels_after_defaults(router, request_metadata, metadata_variable_name)
|
||||
|
||||
assert labels == expected_labels
|
||||
|
||||
|
||||
def test_default_requester_metadata_does_not_carry_labels_between_requests():
|
||||
router: Final = _vertex_labels_router({"requester_metadata": {"env": "prod"}})
|
||||
|
||||
first: Final = _vertex_labels_after_defaults(router, {"requester_metadata": {"litellm-team": "a"}}, "metadata")
|
||||
second: Final = _vertex_labels_after_defaults(router, {}, "metadata")
|
||||
|
||||
assert first == {"env": "prod", "litellm-team": "a"}
|
||||
assert second == {"env": "prod"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_metadata, expected_tags",
|
||||
[
|
||||
({"tags": ["caller"]}, ["caller"]),
|
||||
({}, ["proxy-default"]),
|
||||
({"tags": None}, ["proxy-default"]),
|
||||
],
|
||||
)
|
||||
def test_default_metadata_list_value_applies_only_when_request_sets_none(request_metadata: dict, expected_tags: list):
|
||||
router: Final = _vertex_labels_router({"tags": ["proxy-default"]})
|
||||
kwargs: Final = {"metadata": request_metadata}
|
||||
|
||||
router._update_kwargs_with_default_litellm_params(kwargs=kwargs)
|
||||
|
||||
assert kwargs["metadata"]["tags"] == expected_tags
|
||||
|
||||
|
||||
@pytest.mark.parametrize("router_caching_groups", [[("g", "other")], None])
|
||||
def test_router_caching_groups_win_over_request_metadata(router_caching_groups: list | None):
|
||||
router: Final = Router(
|
||||
model_list=[{"model_name": "g", "litellm_params": {"model": "vertex_ai/gemini-3.8-flash"}}],
|
||||
caching_groups=router_caching_groups,
|
||||
)
|
||||
kwargs: Final = {"metadata": {"caching_groups": [("g", "premium")]}}
|
||||
|
||||
router._update_kwargs_with_default_litellm_params(kwargs=kwargs)
|
||||
|
||||
assert kwargs["metadata"]["caching_groups"] == router_caching_groups
|
||||
|
||||
|
||||
def test_get_timeout(model_list):
|
||||
"""Test if the '_get_timeout' function is working correctly"""
|
||||
router = Router(model_list=model_list)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue