diff --git a/litellm/router.py b/litellm/router.py index 2c20e810839..49c9ffe3288 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3774,7 +3774,16 @@ class Router: self, deployment: dict, kwargs: dict, function_name: str | None = None ) -> Deployment: """ - Handle clientside credential + Build a per-request Deployment carrying the caller-supplied api_key/api_base, + with its own stable id for cooldown, logging, and cost-map identity. + + This deployment is deliberately never registered with the router (no + upsert_deployment/add_deployment call): doing so used to add it to + self.model_list under the shared model_name, which made a request-scoped, + caller-supplied provider credential a permanent, load-balanced deployment + that every other caller of that model group could be routed onto. Its + pricing is still registered directly, so a custom price configured on the + underlying deployment still applies to this call. """ model_info: Final = deployment.get("model_info", {}).copy() litellm_params: Final = deployment["litellm_params"].copy() @@ -3793,7 +3802,7 @@ class Router: litellm_params=LiteLLM_Params(**dynamic_litellm_params), model_info=model_info, ) - self.upsert_deployment(deployment=deployment_pydantic_obj) # add new deployment to router + Router._register_deployment_pricing(deployment=deployment_pydantic_obj) return deployment_pydantic_obj @staticmethod @@ -9701,40 +9710,7 @@ class Router: # initialize client self._add_deployment(deployment=deployment) - _model_info_dict: Final[dict] = deployment.model_info.model_dump(exclude_none=True) - for field in CustomPricingLiteLLMParams.model_fields: - field_value = deployment.litellm_params.get(field) - if field_value is not None: - _model_info_dict[field] = field_value - - Router._inherit_builtin_base_rates_for_off_peak( - model_info=_model_info_dict, - backend_model=deployment.litellm_params.model, - custom_llm_provider=deployment.litellm_params.custom_llm_provider, - ) - if _model_info_dict.get("input_cost_per_token") is not None: - Router._inherit_builtin_cache_pricing( - model_info=_model_info_dict, - backend_model=deployment.litellm_params.model, - custom_llm_provider=deployment.litellm_params.custom_llm_provider, - ) - Router._inherit_builtin_tiered_output_rate( - model_info=_model_info_dict, - backend_model=deployment.litellm_params.model, - custom_llm_provider=deployment.litellm_params.custom_llm_provider, - ) - - # Register custom pricing in litellm.model_cost. - # Mirrors _create_deployment() logic to ensure dynamically-added deployments - # (e.g., loaded from DB) also have their custom pricing registered. - # Without this, _is_model_cost_zero() cannot detect explicitly-configured - # zero-cost models, causing budget checks to block free models. - Router._register_deployment_in_model_cost( - model_id=deployment.model_info.id, - model_info=_model_info_dict, - model=deployment.litellm_params.model, - custom_llm_provider=deployment.litellm_params.custom_llm_provider, - ) + Router._register_deployment_pricing(deployment=deployment) # add to model names self._add_model_to_list_and_index_map(model=_deployment, model_id=deployment.model_info.id) @@ -9996,6 +9972,21 @@ class Router: ) return model_info + @staticmethod + def _register_deployment_pricing(deployment: Deployment) -> None: + """Register a deployment's custom/inherited pricing in ``litellm.model_cost``. + + Takes only a ``Deployment``, so it registers pricing for a deployment that + is never added to ``self.model_list`` (a per-request client-side-credential + deployment) just as readily as one that is. + """ + Router._register_deployment_in_model_cost( + model_id=deployment.model_info.id, + model_info=Router._deployment_model_cost_payload(deployment), + model=deployment.litellm_params.model, + custom_llm_provider=deployment.litellm_params.custom_llm_provider, + ) + @staticmethod def _register_deployment_in_model_cost( *, diff --git a/tests/local_testing/test_router_utils.py b/tests/local_testing/test_router_utils.py index 45fe42f4cd3..1b3e361bb1f 100644 --- a/tests/local_testing/test_router_utils.py +++ b/tests/local_testing/test_router_utils.py @@ -3,6 +3,7 @@ import sys, os, time import traceback, asyncio +import httpx import pytest import litellm @@ -402,6 +403,10 @@ def test_router_redis_cache(): def test_router_handle_clientside_credential(): + """A caller-supplied credential must stay scoped to the current call: it must + never be registered as a router deployment, or a later caller with no override + of their own can be load-balanced onto it and reach the provider with someone + else's credential (see LIT-7811).""" deployment = { "model_name": "gemini/*", "litellm_params": {"model": "gemini/*"}, @@ -421,7 +426,67 @@ def test_router_handle_clientside_credential(): ) assert new_deployment.litellm_params.api_key == "123" - assert len(router.get_model_list()) == 2 + assert len(router.get_model_list()) == 1 + assert router.get_deployment(model_id=new_deployment.model_info.id) is None + + +async def test_router_clientside_credential_not_reused_by_other_callers( + respx_mock, monkeypatch: pytest.MonkeyPatch +): + """End-to-end regression test for LIT-7811. + + One caller's request-scoped api_key must never leak into a later, unrelated + caller's request. Before the fix, the router registered the caller-supplied + credential as a second, permanent deployment for the shared model group, so + plain follow-up calls with no override of their own could be load-balanced + onto it and reach the provider with the first caller's key. + """ + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + route = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 0, + "model": "gpt-4o", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + ) + router = Router( + model_list=[ + { + "model_name": "shared-model", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "configured-key"}, + "model_info": {"id": "configured-deployment"}, + } + ] + ) + + await router.acompletion( + model="shared-model", + messages=[{"role": "user", "content": "hi"}], + api_key="alternate-tenant-key", + ) + assert route.calls[-1].request.headers["authorization"] == "Bearer alternate-tenant-key" + + # The forwarded credential must never become a routable deployment for the + # model group other callers share. + assert [d["model_info"]["id"] for d in router.get_model_list(model_name="shared-model")] == [ + "configured-deployment" + ] + + for _ in range(20): + await router.acompletion( + model="shared-model", + messages=[{"role": "user", "content": "hi"}], + ) + + used_auth_headers = {call.request.headers["authorization"] for call in route.calls[1:]} + assert used_auth_headers == {"Bearer configured-key"} def test_router_get_async_openai_model_client(): diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index b18bf9351c8..14d86743557 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -2099,8 +2099,12 @@ def test_handle_clientside_credential_metadata_loading( assert result_deployment.model_info.id != "original-id-123" assert result_deployment.model_info.original_model_id == "original-id-123" - # Verify the deployment was added to the router - assert len(router.model_list) == len(model_list) + 1 + # The caller-supplied credential must stay scoped to this call: it must never be + # registered as a router deployment, or a later caller with no override of their + # own could be load-balanced onto it and reach the provider with this credential + # (see LIT-7811). + assert len(router.model_list) == len(model_list) + assert router.get_deployment(model_id=result_deployment.model_info.id) is None # Test that the function correctly uses the right metadata key # For acompletion, it should use "metadata" @@ -2260,14 +2264,63 @@ def test_handle_clientside_credential_with_responses_function(model_list): assert result_deployment.model_info.id != "original-id-responses" assert result_deployment.model_info.original_model_id == "original-id-responses" - # Verify the deployment was added to the router - assert len(router.model_list) == len(model_list) + 1 + # The caller-supplied credential must stay scoped to this call: it must never be + # registered as a router deployment (see LIT-7811). + assert len(router.model_list) == len(model_list) + assert router.get_deployment(model_id=result_deployment.model_info.id) is None print( "✓ Success with _ageneric_api_call_with_fallbacks function name and litellm_metadata" ) +def test_handle_clientside_credential_still_registers_custom_pricing(model_list): + """A clientside-credential call must still price against the deployment's own + custom rate, even though the call's ephemeral deployment is never added to the + router (see LIT-7811): losing that registration would silently fall back to + public catalog pricing for every clientside-credential call on a deployment + with a custom rate configured.""" + router = Router(model_list=model_list) + deployment = { + "model_name": "gpt-4.1", + "litellm_params": { + "model": "gpt-4.1", + "api_key": "test_key", + "input_cost_per_token": 0.0001234, + "output_cost_per_token": 0.0005678, + }, + "model_info": {"id": "original-id-pricing"}, + } + kwargs = {"api_key": "client_side_key", "metadata": {"model_group": "gpt-4.1"}} + + result_deployment = router._handle_clientside_credential( + deployment=deployment, kwargs=kwargs, function_name="acompletion" + ) + + registered = litellm.model_cost.get(result_deployment.model_info.id) + assert registered is not None + assert registered["input_cost_per_token"] == 0.0001234 + assert registered["output_cost_per_token"] == 0.0005678 + + +def test_register_deployment_pricing_direct_call(): + """Direct-call unit test for the pricing-registration helper `_handle_clientside_credential` + relies on, so it prices a deployment that is deliberately never added to `self.model_list`.""" + deployment = Deployment( + model_name="gpt-4.1", + litellm_params=LiteLLM_Params( + model="gpt-4.1", + api_key="test_key", + input_cost_per_token=0.0009999, + ), + model_info=ModelInfo(id="direct-call-pricing-id"), + ) + + Router._register_deployment_pricing(deployment=deployment) + + assert litellm.model_cost["direct-call-pricing-id"]["input_cost_per_token"] == 0.0009999 + + def test_get_metadata_variable_name_from_kwargs(model_list): """ Test _get_metadata_variable_name_from_kwargs method returns correct metadata variable name based on kwargs content.