scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_/mongodb-vector-store-e4ff63

Browse source
This commit is contained in:
Yuneng Jiang 2026-09-04 14:20:31 -07:00
commit 38cd1bff7b
No known key found for this signature in database
3 changed files with 203 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
litellm/proxy/health_endpoints/_health_endpoints.py
View file

@ -115,6 +115,29 @@ _CONFIG_CONNECTION_FIELDS: Final[frozenset[str]] = frozenset(
)
def _request_inherits_config_credentials(
config_params: Mapping[str, object],
request_params: Mapping[str, object],
allow_client_side_credentials: bool,
) -> bool:
"""Whether the configuration's credentials are this request's to be probed with.
The configuration reached here by matching the request's model string, which
also matches wildcard routes and unrelated deployments that merely serve the
same model, so a request naming a stored credential of its own has already
said where its credentials come from and does not borrow that one's. A blank
name is no name: ``load_credentials_from_list`` resolves nothing from it, so
it must not cost the request the credentials it would otherwise be probed
with.
"""
requested_credential: Final = request_params.get("litellm_credential_name")
if requested_credential and requested_credential != config_params.get("litellm_credential_name"):
return False
if allow_client_side_credentials:
return True
return not any(param in request_params for param in _BANNED_REQUEST_BODY_PARAMS)
def _config_base_for_health_check(
config_params: Mapping[str, object],
request_params: Mapping[str, object],
@ -122,25 +145,19 @@ def _config_base_for_health_check(
) -> dict[str, object]:
"""Return the configured parameters to merge under a connection-test request.
A request that sets its own connection fields describes a connection of its
own, so the configuration's credentials are not carried into it: they belong
to the endpoint the configuration names. Anything the request does not set
still comes from the configuration, which is what lets a request name a
configured model and test it as configured.
A request that sets its own connection fields, or names its own stored
credential, describes a connection of its own, so the configuration's
credentials are not carried into it: they belong to the endpoint the
configuration names. Anything the request does not set still comes from the
configuration, which is what lets a request name a configured model and test
it as configured.
``litellm_credential_name`` is dropped alongside the literal credential
fields: it names a stored credential that ``load_credentials_from_list``
resolves into the same secrets further down the call, so leaving it in place
would reintroduce them by reference.
``general_settings.allow_client_side_credentials`` is the existing proxy-wide
opt-in for callers supplying their own connection parameters. Where an admin
has enabled it, a request may pair its own endpoint with the configured
credentials, as it could before.
"""
if allow_client_side_credentials:
return dict(config_params)
if not any(param in request_params for param in _BANNED_REQUEST_BODY_PARAMS):
if _request_inherits_config_credentials(config_params, request_params, allow_client_side_credentials):
return dict(config_params)
return {key: value for key, value in config_params.items() if key not in _CONFIG_CONNECTION_FIELDS}
@ -1959,6 +1976,9 @@ async def test_model_connection(
Note:
- If the model is configured in proxy_config.yaml, credentials (api_key, api_base, etc.)
will be automatically loaded from the config (with resolved environment variables).
- A request naming a stored credential (`litellm_credential_name`) that the configuration
does not name is probed with that credential instead, and inherits no credentials
from the configuration its model string happened to match.
- You can override specific params by including them in the request.
- You can use `os.environ/VARIABLE_NAME` syntax to reference environment variables,
which will be resolved automatically (same as in proxy_config.yaml).

167
tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py
View file

@ -15,6 +15,7 @@ from prisma.errors import ClientNotConnectedError, HTTPClientClosedError, Prisma
import litellm
import litellm.proxy.health_endpoints._health_endpoints as _health_endpoints_module
from litellm.litellm_core_utils.health_check_helpers import TEST_IMAGE_BASE64
from litellm.models.credentials import CredentialItem
from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.health_endpoints._health_endpoints import (
@ -2675,6 +2676,172 @@ class TestConfigBaseForHealthCheck:
assert base["litellm_credential_name"] == "OpenAI-prod"
assert base["api_key"] == "sk-configured"
def test_request_naming_another_credential_does_not_inherit_config_credentials(self):
base = self._base(self.CONFIG, {"model": "openai/gpt-4o", "litellm_credential_name": "Another-cred"})
assert "api_key" not in base
assert "api_base" not in base
assert "vertex_credentials" not in base
assert base["rpm"] == 100
def test_blank_credential_name_names_no_credential(self):
base = self._base(self.CONFIG, {"model": "openai/gpt-4o", "litellm_credential_name": ""})
assert base["api_key"] == "sk-configured"
def test_opt_in_does_not_put_config_credentials_over_a_named_credential(self):
base = self._base(
self.CONFIG,
{"model": "openai/gpt-4o", "litellm_credential_name": "Another-cred"},
allow_client_side_credentials=True,
)
assert "api_key" not in base
class TestTestConnectionUsesTheNamedCredential:
CREDENTIAL_KEY = "sk-credential-key"
OTHER_DEPLOYMENT_KEY = "sk-other-deployment-key"
OTHER_DEPLOYMENT_BASE = "https://other-deployment.example/v1"
REQUEST = {
"model": "xai/grok-4",
"custom_llm_provider": "xai",
"litellm_credential_name": "my-xai-cred",
}
COMPLETION = {
"id": "chatcmpl-test",
"object": "chat.completion",
"created": 1700000000,
"model": "grok-4",
"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "ok"}}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
@staticmethod
def _credential(**values: str) -> CredentialItem:
return CredentialItem(credential_name="my-xai-cred", credential_info={}, credential_values=values)
@staticmethod
def _wildcard_deployment(**litellm_params: str) -> dict:
return {
"model_name": "xai/*",
"litellm_params": {"model": "xai/*", **litellm_params},
"model_info": {"id": "unrelated-wildcard-deployment"},
}
def _probe(
self,
monkeypatch,
deployment: dict,
request_litellm_params: dict,
deployment_by_id: object | None = None,
request_model_info: dict | None = None,
) -> httpx.Request:
"""Run /health/test_connection and hand back the upstream request it made."""
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
app = FastAPI()
app.include_router(_health_endpoints_module.router)
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)
router = MagicMock()
router.get_model_list.return_value = [deployment]
router.get_deployment.return_value = deployment_by_id
with (
patch( # test-quality-ok: the endpoint reads the proxy-global DB client and 500s when it is None; it has no injection seam
"litellm.proxy.proxy_server.prisma_client", MagicMock()
),
patch( # test-quality-ok: the deployment the probe is matched against is a proxy global; it has no injection seam
"litellm.proxy.proxy_server.llm_router", router
),
respx.mock(assert_all_called=True) as respx_mock,
):
respx_mock.post(path__regex=r".*/chat/completions").respond(json=self.COMPLETION)
response = TestClient(app).post(
"/health/test_connection",
json={
"mode": "chat",
"litellm_params": request_litellm_params,
"model_info": request_model_info or {"mode": "chat"},
},
)
probe = respx_mock.calls.last.request
assert response.status_code == 200, response.text
assert response.json()["status"] == "success", response.text
return probe
def test_named_credentials_key_is_sent_not_the_matched_deployments_key(self, monkeypatch):
monkeypatch.setattr(litellm, "credential_list", [self._credential(api_key=self.CREDENTIAL_KEY)])
probe = self._probe(
monkeypatch,
self._wildcard_deployment(api_key=self.OTHER_DEPLOYMENT_KEY),
self.REQUEST,
)
assert probe.headers["authorization"] == f"Bearer {self.CREDENTIAL_KEY}"
def test_named_credentials_api_base_is_used_not_the_matched_deployments(self, monkeypatch):
monkeypatch.setattr(
litellm,
"credential_list",
[self._credential(api_key=self.CREDENTIAL_KEY, api_base="https://credential.example/v1")],
)
probe = self._probe(
monkeypatch,
self._wildcard_deployment(api_base=self.OTHER_DEPLOYMENT_BASE),
self.REQUEST,
)
assert probe.url.host == "credential.example"
def test_named_credential_without_an_api_base_leaves_the_provider_default(self, monkeypatch):
monkeypatch.setattr(litellm, "credential_list", [self._credential(api_key=self.CREDENTIAL_KEY)])
probe = self._probe(
monkeypatch,
self._wildcard_deployment(api_base=self.OTHER_DEPLOYMENT_BASE),
self.REQUEST,
)
assert probe.url.host == "api.x.ai"
def test_configured_model_named_without_a_credential_still_inherits_its_config(self, monkeypatch):
probe = self._probe(
monkeypatch,
self._wildcard_deployment(api_key=self.OTHER_DEPLOYMENT_KEY, api_base=self.OTHER_DEPLOYMENT_BASE),
{"model": "xai/grok-4", "custom_llm_provider": "xai"},
)
assert probe.headers["authorization"] == f"Bearer {self.OTHER_DEPLOYMENT_KEY}"
assert probe.url.host == "other-deployment.example"
def test_deployment_probed_by_id_keeps_the_endpoint_it_is_configured_with(self, monkeypatch):
"""The model detail page always echoes back the credential the deployment already uses."""
from litellm.types.router import Deployment, LiteLLM_Params
monkeypatch.setattr(litellm, "credential_list", [self._credential(api_key=self.CREDENTIAL_KEY)])
probe = self._probe(
monkeypatch,
self._wildcard_deployment(api_key=self.OTHER_DEPLOYMENT_KEY, api_base=self.OTHER_DEPLOYMENT_BASE),
self.REQUEST,
deployment_by_id=Deployment(
model_name="grok-4",
litellm_params=LiteLLM_Params(
model="xai/grok-4",
api_base="https://configured.example/v1",
litellm_credential_name="my-xai-cred",
),
model_info={"id": "configured-deployment"},
),
request_model_info={"id": "configured-deployment", "mode": "chat"},
)
assert probe.url.host == "configured.example"
assert probe.headers["authorization"] == f"Bearer {self.CREDENTIAL_KEY}"
class TestNoRedisWarning:
"""`show_no_redis_warning` drives the Admin UI's default-on "no Redis" banner."""

3
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -7037,6 +7037,9 @@ export interface paths {
* Note:
* - If the model is configured in proxy_config.yaml, credentials (api_key, api_base, etc.)
* will be automatically loaded from the config (with resolved environment variables).
* - A request naming a stored credential (`litellm_credential_name`) that the configuration
* does not name is probed with that credential instead, and inherits no credentials
* from the configuration its model string happened to match.
* - You can override specific params by including them in the request.
* - You can use `os.environ/VARIABLE_NAME` syntax to reference environment variables,
* which will be resolved automatically (same as in proxy_config.yaml).