scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

feat(proxy): add credential overrides per team/project via model_config metadata

Browse source
This commit is contained in:
michelligabriele 2026-03-23 21:22:33 +01:00
parent 63425b4cb4
commit 8e9857c5fc
No known key found for this signature in database
2 changed files with 494 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

120
litellm/proxy/litellm_pre_call_utils.py
View file

@ -8,6 +8,7 @@ from starlette.datastructures import Headers
import litellm
from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm._service_logger import ServiceLogging
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.proxy._types import (
@ -1251,6 +1252,12 @@ async def add_litellm_data_to_request( # noqa: PLR0915
user_api_key_dict=user_api_key_dict,
)
# Team/Project credential overrides from model_config
_apply_credential_overrides_from_model_config(
data=data,
user_api_key_dict=user_api_key_dict,
)
verbose_proxy_logger.debug(
"[PROXY] returned data from litellm_pre_call_utils: %s", data
)
@ -1333,6 +1340,119 @@ def _update_model_if_key_alias_exists(
return
def _apply_credential_overrides_from_model_config(
data: dict,
user_api_key_dict: UserAPIKeyAuth,
) -> None:
"""
Walk the model_config precedence chain in team/project metadata.
If a matching credential is found, set api_base/api_key/api_version on data
so they override deployment defaults in the router.
Precedence (highest to lowest):
1. Clientside credentials (already in data — skip if present)
2. Project model-specific override
3. Project default override (defaultconfig)
4. Team model-specific override
5. Team default override (defaultconfig)
6. Deployment default (no action needed)
"""
# Respect clientside credentials — highest precedence
if data.get("api_base") or data.get("api_key"):
return
model_name = data.get("model")
if not model_name:
return
project_metadata = user_api_key_dict.project_metadata or {}
team_metadata = user_api_key_dict.team_metadata or {}
project_model_config = project_metadata.get("model_config")
team_model_config = team_metadata.get("model_config")
if not project_model_config and not team_model_config:
return
credential_name = _resolve_credential_from_model_config(
model_name=model_name,
project_model_config=project_model_config,
team_model_config=team_model_config,
)
if not credential_name:
return
credential_values = CredentialAccessor.get_credential_values(credential_name)
if not credential_values:
verbose_proxy_logger.warning(
"model_config references credential '%s' but it was not found",
credential_name,
)
return
# Apply credential overrides to request data
for key in ("api_base", "api_key", "api_version"):
if key in credential_values:
data[key] = credential_values[key]
verbose_proxy_logger.debug(
"Applied credential override '%s' for model '%s'",
credential_name,
model_name,
)
def _resolve_credential_from_model_config(
model_name: str,
project_model_config: Optional[dict],
team_model_config: Optional[dict],
) -> Optional[str]:
"""
Walk the precedence chain and return the first matching credential name.
Checks (in order):
1. project_model_config[model_name][provider] — project model-specific
2. project_model_config["defaultconfig"][provider] — project default
3. team_model_config[model_name][provider] — team model-specific
4. team_model_config["defaultconfig"][provider] — team default
"""
for model_config in (project_model_config, team_model_config):
if not model_config:
continue
# Model-specific check
model_entry = model_config.get(model_name)
if model_entry:
credential_name = _extract_credential_from_entry(model_entry)
if credential_name:
return credential_name
# Default check
default_entry = model_config.get("defaultconfig")
if default_entry:
credential_name = _extract_credential_from_entry(default_entry)
if credential_name:
return credential_name
return None
def _extract_credential_from_entry(entry: dict) -> Optional[str]:
"""
Extract litellm_credentials from a model_config entry.
Entry structure: {"azure": {"litellm_credentials": "name"}, ...}
Returns the first credential name found across all provider keys.
"""
for provider_config in entry.values():
if isinstance(provider_config, dict):
credential_name = provider_config.get("litellm_credentials")
if credential_name:
return credential_name
return None
def _get_enforced_params(
general_settings: Optional[dict], user_api_key_dict: UserAPIKeyAuth
) -> Optional[list]:

374
tests/test_litellm/proxy/test_litellm_pre_call_utils.py
View file

@ -13,14 +13,18 @@ from litellm.proxy._types import TeamCallbackMetadata, UserAPIKeyAuth
from litellm.proxy.litellm_pre_call_utils import (
KeyAndTeamLoggingSettings,
LiteLLMProxyRequestSetup,
_apply_credential_overrides_from_model_config,
_extract_credential_from_entry,
_get_dynamic_logging_metadata,
_get_enforced_params,
_get_metadata_variable_name,
_resolve_credential_from_model_config,
_update_model_if_key_alias_exists,
add_guardrails_from_policy_engine,
add_litellm_data_to_request,
check_if_token_is_service_account,
)
from litellm.types.utils import CredentialItem
sys.path.insert(
0, os.path.abspath("../../..")
@ -1817,3 +1821,373 @@ async def test_bearer_token_not_in_debug_logs():
f"Bearer token leaked in debug logs. "
f"Found token in log output:\n{log_output[:500]}"
)
# ============================================================================
# Tests for credential overrides from model_config (team/project metadata)
# ============================================================================
@pytest.fixture()
def setup_test_credentials():
"""Populate litellm.credential_list with test credentials, clean up after."""
original = litellm.credential_list[:]
litellm.credential_list.extend(
[
CredentialItem(
credential_name="hotel-azure-eastus",
credential_info={},
credential_values={
"api_base": "https://hotel-eastus.openai.azure.com/",
"api_key": "key-hotel-eastus",
},
),
CredentialItem(
credential_name="hotel-azure-westus",
credential_info={},
credential_values={
"api_base": "https://hotel-westus.openai.azure.com/",
"api_key": "key-hotel-westus",
},
),
CredentialItem(
credential_name="hotel-rec-azure",
credential_info={},
credential_values={
"api_base": "https://hotel-rec-app.openai.azure.com/",
"api_key": "key-hotel-rec",
},
),
CredentialItem(
credential_name="hotel-rec-vision",
credential_info={},
credential_values={
"api_base": "https://hotel-rec-vision.openai.azure.com/",
"api_key": "key-hotel-rec-vision",
"api_version": "2024-06-01",
},
),
CredentialItem(
credential_name="flight-azure-centralus",
credential_info={},
credential_values={
"api_base": "https://flight-centralus.openai.azure.com/",
"api_key": "key-flight-centralus",
},
),
]
)
yield
litellm.credential_list[:] = original
# --- Unit tests for _extract_credential_from_entry ---
def test_extract_credential_from_entry_azure():
entry = {"azure": {"litellm_credentials": "my-cred"}}
assert _extract_credential_from_entry(entry) == "my-cred"
def test_extract_credential_from_entry_no_credential():
entry = {"azure": {"some_other_key": "value"}}
assert _extract_credential_from_entry(entry) is None
def test_extract_credential_from_entry_empty():
assert _extract_credential_from_entry({}) is None
def test_extract_credential_from_entry_non_dict_value():
entry = {"azure": "not-a-dict"}
assert _extract_credential_from_entry(entry) is None
# --- Unit tests for _resolve_credential_from_model_config ---
def test_resolve_project_model_specific_wins():
project_config = {
"gpt-4": {"azure": {"litellm_credentials": "proj-gpt4"}},
"defaultconfig": {"azure": {"litellm_credentials": "proj-default"}},
}
team_config = {
"gpt-4": {"azure": {"litellm_credentials": "team-gpt4"}},
"defaultconfig": {"azure": {"litellm_credentials": "team-default"}},
}
result = _resolve_credential_from_model_config(
"gpt-4", project_config, team_config
)
assert result == "proj-gpt4"
def test_resolve_project_default_wins_over_team():
project_config = {
"defaultconfig": {"azure": {"litellm_credentials": "proj-default"}},
}
team_config = {
"gpt-4": {"azure": {"litellm_credentials": "team-gpt4"}},
"defaultconfig": {"azure": {"litellm_credentials": "team-default"}},
}
result = _resolve_credential_from_model_config(
"gpt-4", project_config, team_config
)
assert result == "proj-default"
def test_resolve_team_model_specific_wins_over_team_default():
team_config = {
"gpt-4": {"azure": {"litellm_credentials": "team-gpt4"}},
"defaultconfig": {"azure": {"litellm_credentials": "team-default"}},
}
result = _resolve_credential_from_model_config("gpt-4", None, team_config)
assert result == "team-gpt4"
def test_resolve_team_default_used_as_fallback():
team_config = {
"defaultconfig": {"azure": {"litellm_credentials": "team-default"}},
}
result = _resolve_credential_from_model_config("gpt-3.5", None, team_config)
assert result == "team-default"
def test_resolve_no_match_returns_none():
result = _resolve_credential_from_model_config("gpt-4", None, None)
assert result is None
def test_resolve_empty_configs_returns_none():
result = _resolve_credential_from_model_config("gpt-4", {}, {})
assert result is None
def test_resolve_model_not_in_any_config():
project_config = {"gpt-4": {"azure": {"litellm_credentials": "x"}}}
result = _resolve_credential_from_model_config("gpt-3.5", project_config, None)
assert result is None
# --- Integration tests for _apply_credential_overrides_from_model_config ---
def test_apply_overrides_project_model_specific(setup_test_credentials):
"""Scenario 2: Hotel Rec App -> gpt-4-vision -> project model-specific."""
data = {"model": "gpt-4-vision"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "hotel-azure-eastus"}
},
"gpt-4": {"azure": {"litellm_credentials": "hotel-azure-westus"}},
}
},
project_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "hotel-rec-azure"}
},
"gpt-4-vision": {
"azure": {"litellm_credentials": "hotel-rec-vision"}
},
}
},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert data["api_base"] == "https://hotel-rec-vision.openai.azure.com/"
assert data["api_key"] == "key-hotel-rec-vision"
assert data["api_version"] == "2024-06-01"
def test_apply_overrides_project_default(setup_test_credentials):
"""Scenario 1: Hotel Rec App -> gpt-4 -> project default."""
data = {"model": "gpt-4"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "hotel-azure-eastus"}
},
"gpt-4": {"azure": {"litellm_credentials": "hotel-azure-westus"}},
}
},
project_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "hotel-rec-azure"}
},
"gpt-4-vision": {
"azure": {"litellm_credentials": "hotel-rec-vision"}
},
}
},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert data["api_base"] == "https://hotel-rec-app.openai.azure.com/"
assert data["api_key"] == "key-hotel-rec"
def test_apply_overrides_team_model_specific(setup_test_credentials):
"""Scenario 4: Hotel Review App -> gpt-4 -> team model-specific."""
data = {"model": "gpt-4"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "hotel-azure-eastus"}
},
"gpt-4": {"azure": {"litellm_credentials": "hotel-azure-westus"}},
}
},
project_metadata={},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert data["api_base"] == "https://hotel-westus.openai.azure.com/"
assert data["api_key"] == "key-hotel-westus"
def test_apply_overrides_team_default(setup_test_credentials):
"""Scenario 3: Hotel Review App -> gpt-3.5 -> team default."""
data = {"model": "gpt-3.5"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "hotel-azure-eastus"}
},
"gpt-4": {"azure": {"litellm_credentials": "hotel-azure-westus"}},
}
},
project_metadata={},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert data["api_base"] == "https://hotel-eastus.openai.azure.com/"
assert data["api_key"] == "key-hotel-eastus"
def test_apply_overrides_no_config(setup_test_credentials):
"""Scenario 6: No model_config anywhere -> data unchanged."""
data = {"model": "gpt-4"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={},
project_metadata={},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert "api_base" not in data
assert "api_key" not in data
def test_apply_overrides_clientside_credentials_take_precedence(
setup_test_credentials,
):
"""Clientside api_base/api_key in data should block model_config override."""
data = {
"model": "gpt-4",
"api_base": "https://my-custom-endpoint.openai.azure.com/",
"api_key": "my-custom-key",
}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "hotel-azure-eastus"}
}
}
},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert data["api_base"] == "https://my-custom-endpoint.openai.azure.com/"
assert data["api_key"] == "my-custom-key"
def test_apply_overrides_missing_credential_name(setup_test_credentials):
"""model_config references a credential that doesn't exist -> no override."""
data = {"model": "gpt-4"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"gpt-4": {
"azure": {"litellm_credentials": "nonexistent-credential"}
}
}
},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert "api_base" not in data
assert "api_key" not in data
def test_apply_overrides_api_version_only_if_present(setup_test_credentials):
"""api_version should only be set if the credential contains it."""
data = {"model": "gpt-3.5"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "hotel-azure-eastus"}
}
}
},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert data["api_base"] == "https://hotel-eastus.openai.azure.com/"
assert data["api_key"] == "key-hotel-eastus"
assert "api_version" not in data
def test_apply_overrides_no_model_in_data():
"""No model in request data -> skip override."""
data = {"messages": [{"role": "user", "content": "hello"}]}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "some-cred"}
}
}
},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert "api_base" not in data
def test_apply_overrides_none_metadata():
"""None metadata on both team and project -> skip override."""
data = {"model": "gpt-4"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata=None,
project_metadata=None,
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert "api_base" not in data