From 609acd5d9ccf54caf2061c883ea923347fd05b9b Mon Sep 17 00:00:00 2001 From: shrey Date: Sun, 15 Nov 2026 19:11:53 -0800 Subject: [PATCH] fix(proxy): stop team model writes from pulling in proxy-level settings (#51874) --- .../model_management_endpoints.py | 82 ++++++- .../test_model_management_endpoints.py | 217 ++++++++++++++++++ 2 files changed, 298 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 6823edb767a..bc29b7903e1 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -222,6 +222,57 @@ class KubernetesPodsResponse(BaseModel): CLEARABLE_LITELLM_PARAMS: Final = frozenset({"cache_control_injection_points", "litellm_credential_name"}) NULL_CLEARABLE_LITELLM_PARAMS: Final = frozenset((*SPECIAL_MODEL_INFO_PARAMS, *CLEARABLE_LITELLM_PARAMS)) _STR_OBJECT_DICT: Final = TypeAdapter(dict[str, object]) +_OBJECT_LIST: Final = TypeAdapter(list[object]) + + +_Node: TypeAlias = tuple[tuple[object, ...], object] + + +def _child_nodes(node: _Node) -> tuple[_Node, ...]: + path, value = node + if is_str_keyed_mapping(value): + return tuple(((*path, key), child) for key, child in value.items()) + if isinstance(value, list): + return tuple(((*path, index), child) for index, child in enumerate(_OBJECT_LIST.validate_python(value))) + return () + + +def _next_level(level: tuple[_Node, ...], _: None) -> tuple[_Node, ...]: + return tuple(itertools.chain.from_iterable(map(_child_nodes, level))) + + +def _string_leaves(value: object) -> Mapping[tuple[object, ...], str]: + levels: Final = itertools.takewhile( + bool, itertools.accumulate(itertools.repeat(None), _next_level, initial=(((), value),)) + ) + return MappingProxyType( + {path: leaf for path, leaf in itertools.chain.from_iterable(levels) if isinstance(leaf, str)} + ) + + +def _adds_environment_reference(value: object, stored: object) -> bool: + kept: Final = _string_leaves(stored) + return any( + leaf.startswith("os.environ/") and kept.get(path) != leaf for path, leaf in _string_leaves(value).items() + ) + + +@dataclass(frozen=True, slots=True) +class _EnvironmentReferenceDenied: + field: str + + +def _raise_environment_reference_denied(denial: _EnvironmentReferenceDenied | None) -> None: + match denial: + case None: + return + case _EnvironmentReferenceDenied(field=field): + raise ProxyException( + message=f"Only proxy admins can set an os.environ/ reference on a model ({field!r}).", + type=ProxyErrorTypes.auth_error.value, + code=status.HTTP_403_FORBIDDEN, + param=field, + ) async def update_team(*args, **kwargs): @@ -2845,6 +2896,27 @@ class ModelManagementAuthChecks: param="api_key", ) + @staticmethod + def _non_admin_environment_reference( + *, + incoming_params: Mapping[str, object] | None, + stored_params: GenericLiteLLMParams | None, + user_api_key_dict: UserAPIKeyAuth, + ) -> _EnvironmentReferenceDenied | None: + if incoming_params is None or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN: + return None + stored: Final[Mapping[str, object]] = ( + MappingProxyType({}) if stored_params is None else _decrypted_litellm_params(stored_params) + ) + return next( + ( + _EnvironmentReferenceDenied(name) + for name, value in incoming_params.items() + if _adds_environment_reference(value, stored.get(name)) + ), + None, + ) + @staticmethod async def can_user_make_model_call( model_params: Deployment, @@ -2926,12 +2998,20 @@ class ModelManagementAuthChecks: llm_router=llm_router, ) - return ModelManagementAuthChecks.can_user_make_team_model_call( + team_write: Final = ModelManagementAuthChecks.can_user_make_team_model_call( team_id=model_params.model_info.team_id, user_api_key_dict=user_api_key_dict, team_obj=team_obj, premium_user=premium_user, ) + _raise_environment_reference_denied( + ModelManagementAuthChecks._non_admin_environment_reference( + incoming_params=incoming_params, + stored_params=model_params.litellm_params if member_operation == "update" else None, + user_api_key_dict=user_api_key_dict, + ) + ) + return team_write ## Check non-team model auth elif user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: raise HTTPException( diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index be72e0f1fae..7fb6afafb31 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -569,6 +569,34 @@ class TestModelManagementAuthChecks: await save() assert (getattr(exc_info.value, "param", None), getattr(exc_info.value, "code", None)) == ("api_key", "400") + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("model_info", "team_admin", "refusal"), + [ + ({}, True, "User does not have permission to make this model call"), + ({"team_id": "test_team"}, False, "Team ID=test_team does not match the API key's team"), + ], + ids=["proxy-level model", "team model, caller not team admin"], + ) + async def test_a_refused_write_carrying_a_reference_keeps_its_original_refusal( + self, model_info: dict, team_admin: bool, refusal: str + ) -> None: + model_params: Final = Deployment( + model_name="test_model", + litellm_params=LiteLLM_Params(model="openai/gpt-4o-mini", api_key="os.environ/OTHER_VALUE"), + model_info=model_info, + ) + + with pytest.raises(HTTPException, match=refusal) as refused: + await ModelManagementAuthChecks.can_user_make_model_call( + model_params=model_params, + user_api_key_dict=self.normal_user, + prisma_client=MockPrismaClient(team_exists=True, user_admin=team_admin), + premium_user=True, + incoming_params={"model": "openai/gpt-4o-mini", "api_key": "os.environ/OTHER_VALUE"}, + ) + assert refused.value.status_code == 403 + @pytest.mark.asyncio async def test_can_user_make_model_call_normal_user_fails(self): """Test that normal users cannot make model calls""" @@ -9107,6 +9135,195 @@ class TestTeamMemberAutoRouterWrites: self._classifier_config({**connection, "timeout_ms": 8100}, False) if change_config else stored_config ) + @staticmethod + def _plain_row(stored: Mapping[str, object]) -> LiteLLM_ProxyModelTable: + return TestTeamMemberAutoRouterWrites._row().model_copy( + update={"litellm_params": {"model": "openai/gpt-4o-mini", **stored}} + ) + + @staticmethod + def _team_admin_setup(row: LiteLLM_ProxyModelTable) -> tuple[MagicMock, UserAPIKeyAuth]: + team: Final = TestTeamMemberAutoRouterWrites._team().model_copy( + update={"members_with_roles": [Member(user_id="owner", role="admin")]} + ) + return ( + TestTeamMemberAutoRouterWrites._database(team, row), + UserAPIKeyAuth(user_id="owner", user_role=LitellmUserRoles.INTERNAL_USER), + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize("endpoint", ["create", "patch", "legacy"]) + @pytest.mark.parametrize("field", ["api_key", "api_base"]) + async def test_team_admin_model_writes_reject_new_environment_references(self, endpoint: str, field: str) -> None: + from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model + + row: Final = self._plain_row({"api_key": "sk-stored", "api_base": "https://provider.test/v1"}) + database, actor = self._team_admin_setup(row) + written: Final = {"model": "openai/gpt-4o-mini", field: "os.environ/OTHER_VALUE"} + request: Final = updateDeployment( + litellm_params=updateLiteLLMParams.model_validate(written), + model_info=ModelInfo(id=row.model_id, team_id="member-team"), + ) + operation: Final = ( + add_new_model( + Deployment( + model_name="team-model", + litellm_params=LiteLLM_Params(**written), + model_info=ModelInfo(id=row.model_id, team_id="member-team"), + ), + actor, + ) + if endpoint == "create" + else patch_model(row.model_id, request, actor) if endpoint == "patch" else update_model(request, actor) + ) + with self._environment(database, row), pytest.raises(ProxyException) as denied: + await operation + + assert (denied.value.code, denied.value.param) == ("403", field) + database.db.litellm_proxymodeltable.create.assert_not_awaited() + database.db.litellm_proxymodeltable.update.assert_not_awaited() + + @pytest.mark.asyncio + @pytest.mark.parametrize("endpoint", ["patch", "legacy"]) + async def test_team_admin_edits_resend_a_stored_environment_reference(self, endpoint: str) -> None: + from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper + + stored: Final = {"api_key": "os.environ/TEAM_VALUE", "api_base": "os.environ/TEAM_BASE"} + row: Final = self._plain_row(stored) + database, actor = self._team_admin_setup(row) + request: Final = updateDeployment( + litellm_params=updateLiteLLMParams.model_validate({**stored, "rpm": 10}), + model_info=ModelInfo(id=row.model_id), + ) + with self._environment(database, row): + await (patch_model(row.model_id, request, actor) if endpoint == "patch" else update_model(request, actor)) + + written: Final = json.loads(database.db.litellm_proxymodeltable.update.await_args.kwargs["data"]["litellm_params"]) + saved: Final = {name: decrypt_value_helper(value=written[name], key=name) for name in stored} + assert (saved, written["rpm"]) == (stored, 10) + + @pytest.mark.asyncio + async def test_team_admin_edit_keeps_a_nested_reference_beside_a_masked_value(self) -> None: + stored: Final = { + "dataSources": [ + { + "type": "azure_search", + "parameters": {"endpoint": "os.environ/SEARCH_ENDPOINT", "key": "os.environ/SEARCH_KEY"}, + } + ] + } + row: Final = self._plain_row(stored) + database, actor = self._team_admin_setup(row) + resent: Final = { + "dataSources": [ + {"type": "azure_search", "parameters": {"endpoint": "os.environ/SEARCH_ENDPOINT", "key": "os.e****_KEY"}} + ], + "tpm": 100, + } + request: Final = updateDeployment( + litellm_params=updateLiteLLMParams.model_validate(resent), model_info=ModelInfo(id=row.model_id) + ) + with self._environment(database, row): + await patch_model(row.model_id, request, actor) + + written: Final = json.loads(database.db.litellm_proxymodeltable.update.await_args.kwargs["data"]["litellm_params"]) + assert (written["dataSources"][0]["parameters"]["endpoint"], written["tpm"]) == ("os.environ/SEARCH_ENDPOINT", 100) + + @pytest.mark.asyncio + async def test_team_admin_cannot_copy_a_stored_reference_into_another_data_source(self) -> None: + row: Final = self._plain_row( + {"dataSources": [{"parameters": {"key": "sk-plain"}}, {"parameters": {"key": "os.environ/SEARCH_KEY"}}]} + ) + database, actor = self._team_admin_setup(row) + copied: Final = {"parameters": {"key": "os.environ/SEARCH_KEY"}} + request: Final = updateDeployment( + litellm_params=updateLiteLLMParams.model_validate({"dataSources": [copied, copied]}), + model_info=ModelInfo(id=row.model_id), + ) + with self._environment(database, row), pytest.raises(ProxyException) as denied: + await patch_model(row.model_id, request, actor) + + assert (denied.value.code, denied.value.param) == ("403", "dataSources") + database.db.litellm_proxymodeltable.update.assert_not_awaited() + + @pytest.mark.asyncio + async def test_team_admin_can_save_a_value_that_only_mentions_the_reference_prefix(self) -> None: + from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper + + row: Final = self._plain_row({"api_key": "sk-stored"}) + database, actor = self._team_admin_setup(row) + request: Final = updateDeployment( + litellm_params=updateLiteLLMParams(api_key="sk-os.environ/text"), model_info=ModelInfo(id=row.model_id) + ) + with self._environment(database, row): + await patch_model(row.model_id, request, actor) + + written: Final = json.loads(database.db.litellm_proxymodeltable.update.await_args.kwargs["data"]["litellm_params"]) + assert decrypt_value_helper(value=written["api_key"], key="api_key") == "sk-os.environ/text" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("written", "field"), + [ + ({"api_key": "os.environ/OTHER_VALUE"}, "api_key"), + ({"api_base": "os.environ/TEAM_VALUE"}, "api_base"), + ({"dataSources": [{"type": "azure_search", "parameters": {"key": "os.environ/OTHER_VALUE"}}]}, "dataSources"), + ], + ids=["another reference", "stored reference in another field", "nested reference"], + ) + async def test_team_admin_cannot_write_a_reference_other_than_the_stored_one( + self, written: Mapping[str, object], field: str + ) -> None: + row: Final = self._plain_row({"api_key": "os.environ/TEAM_VALUE"}) + database, actor = self._team_admin_setup(row) + request: Final = updateDeployment( + litellm_params=updateLiteLLMParams.model_validate(written), model_info=ModelInfo(id=row.model_id) + ) + with self._environment(database, row), pytest.raises(ProxyException) as denied: + await patch_model(row.model_id, request, actor) + + assert (denied.value.code, denied.value.param) == ("403", field) + database.db.litellm_proxymodeltable.update.assert_not_awaited() + + @pytest.mark.asyncio + @pytest.mark.parametrize("endpoint", ["create", "patch", "legacy"]) + async def test_proxy_admin_model_writes_keep_environment_references(self, endpoint: str) -> None: + from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper + from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model + + row: Final = self._plain_row({"api_key": "sk-stored"}) + database: Final = self._database(self._team(), row) + actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + written: Final = {"model": "openai/gpt-4o-mini", "api_key": "os.environ/TEAM_VALUE"} + request: Final = updateDeployment( + litellm_params=updateLiteLLMParams.model_validate(written), + model_info=ModelInfo(id=row.model_id, team_id="member-team"), + ) + with ( + self._environment(database, row), + patch("litellm.proxy.proxy_server.proxy_config.add_deployment", new=AsyncMock(return_value=ReconcileOutcome( # test-quality-ok: [TQ008] model reload I/O boundary + still_desired=frozenset((row.model_id,)), live_after=frozenset((row.model_id,)) + ))), + patch("litellm.proxy.management_endpoints.model_management_endpoints.append_team_models", new=AsyncMock()), # test-quality-ok: [TQ008] team allowlist persistence boundary + ): + await ( + add_new_model( + Deployment( + model_name="team-model", + litellm_params=LiteLLM_Params(**written), + model_info=ModelInfo(id=row.model_id, team_id="member-team"), + ), + actor, + ) + if endpoint == "create" + else patch_model(row.model_id, request, actor) if endpoint == "patch" else update_model(request, actor) + ) + + table: Final = database.db.litellm_proxymodeltable + write: Final = table.create if endpoint == "create" else table.update + saved: Final = json.loads(write.await_args.kwargs["data"]["litellm_params"]) + assert decrypt_value_helper(value=saved["api_key"], key="api_key") == "os.environ/TEAM_VALUE" + @pytest.mark.asyncio @pytest.mark.parametrize( "stored_params,removed,denied_param",