fix(router): merge default metadata under the request's instead of replacing it

This commit is contained in:
mateo-berri 2026-10-29 22:28:18 -07:00
parent b91fc92b6d
commit f4015d9f51
4 changed files with 197 additions and 2 deletions

View file

@ -118,6 +118,7 @@ from litellm.litellm_core_utils.batch_utils import (
)
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
coerce_token_limit,
get_litellm_metadata_from_kwargs,
get_metadata_variable_name_from_kwargs,
@ -264,6 +265,7 @@ from litellm.router_utils.common_utils import (
format_no_fallback_group_message,
get_request_team_id,
is_proxy_admin_request,
metadata_defaults_under_request,
provider_for_generic_call,
request_user_api_key_auth,
resolve_model_group_alias,
@ -677,6 +679,7 @@ _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None)
_ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
_FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_REQUEST_METADATA_ADAPTER: Final = TypeAdapter(Mapping[object, object])
_DEPLOYMENT_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | GenericLiteLLMParams]] = TypeAdapter(
Mapping[str, object] | GenericLiteLLMParams
)
@ -5037,7 +5040,7 @@ class Router(DeprecatedRouterMembers):
"""
# 1) copy your defaults and pull out metadata
defaults: Final = dict(without_retrieve_stream_defaults(self._core_default_litellm_params, kwargs))
metadata_defaults: Final = defaults.pop("metadata", {}) or {}
metadata_defaults: Final = as_str_mapping(defaults.pop("metadata", None)) or {}
# 2) add any non-metadata defaults that aren't already in kwargs
for key, value in defaults.items():
@ -5046,7 +5049,12 @@ class Router(DeprecatedRouterMembers):
kwargs.setdefault(key, value)
# 3) merge in metadata, this handles inserting this as either "metadata" or "litellm_metadata"
kwargs.setdefault(metadata_variable_name, {}).update(metadata_defaults)
request_metadata: Final = _REQUEST_METADATA_ADAPTER.validate_python(
kwargs.setdefault(metadata_variable_name, {})
)
kwargs[metadata_variable_name].update(
metadata_defaults_under_request(defaults=metadata_defaults, request_metadata=request_metadata)
)
def _handle_clientside_credential(
self, deployment: dict, kwargs: dict, function_name: str | None = None

View file

@ -13,6 +13,7 @@ import litellm
from litellm._logging import verbose_logger, verbose_router_logger
from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure
@ -61,6 +62,32 @@ def without_router_only_kwargs(kwargs: Mapping[str, _V]) -> dict[str, _V]:
return {key: value for key, value in kwargs.items() if key not in ROUTER_ONLY_CALL_KWARGS}
_ROUTER_OWNED_METADATA_KEYS: Final = frozenset({"caching_groups"})
def metadata_defaults_under_request(
defaults: Mapping[str, object], request_metadata: Mapping[object, object]
) -> Mapping[str, object]:
return {
key: (
default
if key in _ROUTER_OWNED_METADATA_KEYS
else _request_value_over_default(default=default, requested=request_metadata.get(key))
)
for key, default in defaults.items()
}
def _request_value_over_default(default: object, requested: object) -> object:
if requested is None:
return default
default_mapping: Final = as_str_mapping(default)
requested_mapping: Final = as_str_mapping(requested)
if default_mapping is None or requested_mapping is None:
return requested
return {**default_mapping, **requested_mapping}
def as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]:
if not isinstance(value, tuple):
return ()

View file

@ -0,0 +1,85 @@
import json
from collections.abc import Mapping
from pathlib import Path
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.process import graceful_stop_seconds, owned_proxy_process
from integration._support.vertex import service_account_json
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
_MODEL_NAME: Final = "labelled-gemini"
_PROJECT: Final = "scripted-project"
_LOCATION: Final = "us-central1"
_ROUTER_LABELS: Final = {"env": "prod"}
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_REPLY: Final = json.dumps(
{
"candidates": [{"content": {"role": "model", "parts": [{"text": "OK"}]}, "finishReason": "STOP"}],
"usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 1, "totalTokenCount": 4},
}
).encode()
def _gemini_peer(request: Request) -> Reply:
if request.target.endswith(":generateContent"):
return Reply(body=_REPLY)
return Reply(status=404, body=json.dumps({"error": request.target}).encode())
def _config(directory: Path, gateway: Gateway, api_base: str) -> Path:
config: Final = directory / "router-default-requester-metadata.yaml"
config.write_text(
json.dumps(
{
"model_list": [
{
"model_name": _MODEL_NAME,
"litellm_params": {
"model": "vertex_ai/gemini-3.8-flash",
"api_base": api_base,
"vertex_project": _PROJECT,
"vertex_location": _LOCATION,
"vertex_credentials": service_account_json(_PROJECT, gateway.upstream_url.rstrip("/")),
},
}
],
"router_settings": {"default_litellm_params": {"metadata": {"requester_metadata": _ROUTER_LABELS}}},
}
),
encoding="utf-8",
)
return config
def _labels_sent(proxy: Gateway, wire: Wire, metadata: Mapping[str, JsonValue] | None) -> JsonValue:
response: Final = proxy.request(
"POST",
"/v1/chat/completions",
{
"model": _MODEL_NAME,
"messages": [{"role": "user", "content": "Return exactly OK."}],
**({"metadata": dict(metadata)} if metadata is not None else {}),
},
)
assert response.status_code == 200, response.text
generations: Final = tuple(request for request in wire.drain() if request.target.endswith(":generateContent"))
assert len(generations) == 1, generations
return _JSON_OBJECT.validate_json(generations[0].body).get("labels")
def test_router_default_requester_metadata_merges_under_each_callers_vertex_labels(
gateway: Gateway, tmp_path: Path
) -> None:
with (
wire_server(_gemini_peer) as wire,
owned_proxy_process(gateway, tmp_path, {}, config=_config(tmp_path, gateway, wire.url)) as owned,
):
proxy: Final = Gateway(owned.gateway.client, owned.gateway.key, gateway.upstream_url)
assert _labels_sent(proxy, wire, {"team": "ai-platform"}) == {**_ROUTER_LABELS, "team": "ai-platform"}
assert _labels_sent(proxy, wire, {"env": "staging"}) == {"env": "staging"}
assert _labels_sent(proxy, wire, None) == _ROUTER_LABELS

View file

@ -20,6 +20,7 @@ from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
from litellm.types.litellm_params import RoutingStrategyName
from litellm.types.router import AlertingConfig, Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo
@ -396,6 +397,80 @@ def test_update_kwargs_with_default_litellm_params(model_list):
assert kwargs["metadata"]["key2"] == "value2"
def _vertex_labels_router(metadata_defaults: Mapping[str, object]) -> Router:
return Router(
model_list=[{"model_name": "g", "litellm_params": {"model": "vertex_ai/gemini-3.8-flash"}}],
default_litellm_params={"metadata": dict(metadata_defaults)},
)
def _vertex_labels_after_defaults(router: Router, request_metadata: dict, metadata_variable_name: str):
kwargs: Final = {metadata_variable_name: request_metadata}
router._update_kwargs_with_default_litellm_params(kwargs=kwargs, metadata_variable_name=metadata_variable_name)
return pop_vertex_request_labels(optional_params={}, litellm_params=kwargs)
@pytest.mark.parametrize("metadata_variable_name", ["metadata", "litellm_metadata"])
@pytest.mark.parametrize(
"request_labels, expected_labels",
[
({"litellm-team": "ai-platform"}, {"env": "prod", "litellm-team": "ai-platform"}),
({"env": "dev", "litellm-team": "ai-platform"}, {"env": "dev", "litellm-team": "ai-platform"}),
(None, {"env": "prod"}),
({}, {"env": "prod"}),
],
)
def test_default_requester_metadata_merges_under_request_labels(
metadata_variable_name: str, request_labels: dict | None, expected_labels: dict
):
router: Final = _vertex_labels_router({"requester_metadata": {"env": "prod"}})
request_metadata: Final = {} if request_labels is None else {"requester_metadata": request_labels}
labels: Final = _vertex_labels_after_defaults(router, request_metadata, metadata_variable_name)
assert labels == expected_labels
def test_default_requester_metadata_does_not_carry_labels_between_requests():
router: Final = _vertex_labels_router({"requester_metadata": {"env": "prod"}})
first: Final = _vertex_labels_after_defaults(router, {"requester_metadata": {"litellm-team": "a"}}, "metadata")
second: Final = _vertex_labels_after_defaults(router, {}, "metadata")
assert first == {"env": "prod", "litellm-team": "a"}
assert second == {"env": "prod"}
@pytest.mark.parametrize(
"request_metadata, expected_tags",
[
({"tags": ["caller"]}, ["caller"]),
({}, ["proxy-default"]),
({"tags": None}, ["proxy-default"]),
],
)
def test_default_metadata_list_value_applies_only_when_request_sets_none(request_metadata: dict, expected_tags: list):
router: Final = _vertex_labels_router({"tags": ["proxy-default"]})
kwargs: Final = {"metadata": request_metadata}
router._update_kwargs_with_default_litellm_params(kwargs=kwargs)
assert kwargs["metadata"]["tags"] == expected_tags
@pytest.mark.parametrize("router_caching_groups", [[("g", "other")], None])
def test_router_caching_groups_win_over_request_metadata(router_caching_groups: list | None):
router: Final = Router(
model_list=[{"model_name": "g", "litellm_params": {"model": "vertex_ai/gemini-3.8-flash"}}],
caching_groups=router_caching_groups,
)
kwargs: Final = {"metadata": {"caching_groups": [("g", "premium")]}}
router._update_kwargs_with_default_litellm_params(kwargs=kwargs)
assert kwargs["metadata"]["caching_groups"] == router_caching_groups
def test_get_timeout(model_list):
"""Test if the '_get_timeout' function is working correctly"""
router = Router(model_list=model_list)