From fc0d21794dc14d9cf1ff105ce2b50efdaefd0c16 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 14 Mar 2025 10:14:21 -0700 Subject: [PATCH 1/6] style: cleanup credential from leftnav - now in models tab --- litellm/proxy/_experimental/out/onboarding.html | 1 - ui/litellm-dashboard/src/components/leftnav.tsx | 1 - 2 files changed, 2 deletions(-) delete mode 100644 litellm/proxy/_experimental/out/onboarding.html diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html deleted file mode 100644 index 95f6319f7ac..00000000000 --- a/litellm/proxy/_experimental/out/onboarding.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 46540401a94..786e94bb2a5 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -69,7 +69,6 @@ const menuItems: MenuItem[] = [ { key: "10", page: "budgets", label: "Budgets", icon: , roles: all_admin_roles }, { key: "11", page: "guardrails", label: "Guardrails", icon: , roles: all_admin_roles }, { key: "18", page: "transform-request", label: "Playground", icon: , roles: all_admin_roles }, - { key: "19", page: "credentials", label: "Credentials", icon: , roles: all_admin_roles }, ] }, { From 6629354329294f1889e093a38d97d58c1721d82b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 14 Mar 2025 10:42:17 -0700 Subject: [PATCH 2/6] fix(endpoints.py): update credentials should update before storing --- .../proxy/credential_endpoints/endpoints.py | 48 ++++++++++++++++++- .../src/components/networking.tsx | 2 +- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 84b35e3c522..75d89502b5b 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -164,7 +164,45 @@ async def delete_credential( return handle_exception_on_proxy(e) -@router.put( +def update_db_credential( + db_credential: CredentialItem, updated_patch: CredentialItem +) -> CredentialItem: + """ + Update a credential in the DB. + """ + merged_credential = CredentialItem( + credential_name=db_credential.credential_name, + credential_info=db_credential.credential_info, + credential_values=db_credential.credential_values, + ) + + encrypted_credential = CredentialHelperUtils.encrypt_credential_values( + updated_patch + ) + # update model name + if encrypted_credential.credential_name: + merged_credential.credential_name = encrypted_credential.credential_name + + # update litellm params + if encrypted_credential.credential_values: + # Encrypt any sensitive values + encrypted_params = { + k: v for k, v in encrypted_credential.credential_values.items() + } + + merged_credential.credential_values.update(encrypted_params) + + # update model info + if encrypted_credential.credential_info: + """Update credential info""" + if "credential_info" not in merged_credential.credential_info: + merged_credential.credential_info = {} + merged_credential.credential_info.update(encrypted_credential.credential_info) + + return merged_credential + + +@router.patch( "/credentials/{credential_name}", dependencies=[Depends(user_api_key_auth)], tags=["credential management"], @@ -187,7 +225,13 @@ async def update_credential( status_code=500, detail={"error": CommonProxyErrors.db_not_connected_error.value}, ) - credential_object_jsonified = jsonify_object(credential.model_dump()) + db_credential = await prisma_client.db.litellm_credentialstable.find_unique( + where={"credential_name": credential_name}, + ) + if db_credential is None: + raise HTTPException(status_code=404, detail="Credential not found in DB.") + merged_credential = update_db_credential(db_credential, credential) + credential_object_jsonified = jsonify_object(merged_credential.model_dump()) await prisma_client.db.litellm_credentialstable.update( where={"credential_name": credential_name}, data={ diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 19589400a0e..2cd122aba80 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2698,7 +2698,7 @@ export const credentialUpdateCall = async ( const url = proxyBaseUrl ? `${proxyBaseUrl}/credentials/${credentialName}` : `/credentials/${credentialName}`; const response = await fetch(url, { - method: "PUT", + method: "PATCH", headers: { [globalLitellmHeaderName]: `Bearer ${accessToken}`, "Content-Type": "application/json", From 605a4d1121f2270657c10d4a798854a0500ec2fb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 14 Mar 2025 12:02:50 -0700 Subject: [PATCH 3/6] feat(endpoints.py): enable retrieving existing credentials by model name Enables reusing existing credentials --- litellm/litellm_core_utils/litellm_logging.py | 74 ++++++++++++++----- .../proxy/credential_endpoints/endpoints.py | 72 +++++++++++++++--- litellm/types/router.py | 43 ++++++----- 3 files changed, 142 insertions(+), 47 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 18af6399180..f0b5afa67fc 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -719,25 +719,9 @@ class Logging(LiteLLMLoggingBaseClass): Masks the headers of the request sent from LiteLLM """ - sensitive_keywords = [ - "authorization", - "token", - "key", - "secret", - ] - return { - k: ( - (v[:-44] + "*" * 44) - if (isinstance(v, str) and len(v) > 44) - else "*****" - ) - for k, v in headers.items() - if not ignore_sensitive_headers - or not any( - sensitive_keyword in k.lower() - for sensitive_keyword in sensitive_keywords - ) - } + return _get_masked_values( + headers, ignore_sensitive_values=ignore_sensitive_headers + ) def post_call( self, original_response, input=None, api_key=None, additional_args={} @@ -2413,6 +2397,58 @@ class Logging(LiteLLMLoggingBaseClass): return result +def _get_masked_values( + sensitive_object: dict, + ignore_sensitive_values: bool = False, + mask_all_values: bool = False, + unmasked_length: int = 44, + number_of_asterisks: Optional[int] = None, +) -> dict: + """ + Internal debugging helper function + + Masks the headers of the request sent from LiteLLM + + Args: + masked_length: Optional length for the masked portion (number of *). If set, will use exactly this many * + regardless of original string length. The total length will be unmasked_length + masked_length. + """ + sensitive_keywords = [ + "authorization", + "token", + "key", + "secret", + ] + return { + k: ( + ( + v[: unmasked_length // 2] + + "*" * number_of_asterisks + + v[-unmasked_length // 2 :] + ) + if ( + isinstance(v, str) + and len(v) > unmasked_length + and number_of_asterisks is not None + ) + else ( + ( + v[: unmasked_length // 2] + + "*" * (len(v) - unmasked_length) + + v[-unmasked_length // 2 :] + ) + if (isinstance(v, str) and len(v) > unmasked_length) + else "*****" + ) + ) + for k, v in sensitive_object.items() + if not ignore_sensitive_values + or not any( + sensitive_keyword in k.lower() for sensitive_keyword in sensitive_keywords + ) + } + + def set_callbacks(callback_list, function_id=None): # noqa: PLR0915 """ Globally sets the callback client diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 75d89502b5b..466766fc2db 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -2,15 +2,19 @@ CRUD endpoints for storing reusable credentials. """ +from typing import Optional + from fastapi import APIRouter, Depends, HTTPException, Request, Response import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.litellm_logging import _get_masked_values from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.utils import handle_exception_on_proxy, jsonify_object +from litellm.types.router import CredentialLiteLLMParams from litellm.types.utils import CredentialItem router = APIRouter() @@ -101,30 +105,76 @@ async def get_credentials( @router.get( - "/credentials/{credential_name}", + "/credentials/by_name/{credential_name}", dependencies=[Depends(user_api_key_auth)], tags=["credential management"], + response_model=CredentialItem, +) +@router.get( + "/credentials/by_model/{model_id}", + dependencies=[Depends(user_api_key_auth)], + tags=["credential management"], + response_model=CredentialItem, ) async def get_credential( request: Request, fastapi_response: Response, - credential_name: str, + credential_name: Optional[str] = None, + model_id: Optional[str] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ [BETA] endpoint. This might change unexpectedly. """ + from litellm.proxy.proxy_server import llm_router + try: - for credential in litellm.credential_list: - if credential.credential_name == credential_name: - masked_credential = { - "credential_name": credential.credential_name, - "credential_values": credential.credential_values, - } - return {"success": True, "credential": masked_credential} - return {"success": False, "message": "Credential not found"} + if model_id: + if llm_router is None: + raise HTTPException(status_code=500, detail="LLM router not found") + # get model from router + model = llm_router.get_deployment(model_id) + if model is None: + raise HTTPException( + status_code=404, detail="Model not found. Got model ID: " + model_id + ) + # get credential object from model + credential_values = _get_masked_values( + CredentialLiteLLMParams(**model.litellm_params.model_dump()).model_dump( + exclude_none=True + ), + unmasked_length=4, + number_of_asterisks=4, + ) + credential = CredentialItem( + credential_name="{}-credential-{}".format(model.model_name, model_id), + credential_values=credential_values, + credential_info={}, + ) + # return credential object + return credential + elif credential_name: + for credential in litellm.credential_list: + if credential.credential_name == credential_name: + masked_credential = CredentialItem( + credential_name=credential.credential_name, + credential_values=_get_masked_values( + credential.credential_values + ), + credential_info=credential.credential_info, + ) + return masked_credential + raise HTTPException( + status_code=404, + detail="Credential not found. Got credential name: " + credential_name, + ) + else: + raise HTTPException( + status_code=404, detail="Credential name or model ID required" + ) except Exception as e: - return handle_exception_on_proxy(e) + verbose_proxy_logger.exception(e) + raise handle_exception_on_proxy(e) @router.delete( diff --git a/litellm/types/router.py b/litellm/types/router.py index 9a5fb168dac..e34366aa229 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -144,7 +144,26 @@ class ModelInfo(BaseModel): setattr(self, key, value) -class GenericLiteLLMParams(BaseModel): +class CredentialLiteLLMParams(BaseModel): + api_key: Optional[str] = None + api_base: Optional[str] = None + api_version: Optional[str] = None + ## VERTEX AI ## + vertex_project: Optional[str] = None + vertex_location: Optional[str] = None + vertex_credentials: Optional[Union[str, dict]] = None + ## UNIFIED PROJECT/REGION ## + region_name: Optional[str] = None + + ## AWS BEDROCK / SAGEMAKER ## + aws_access_key_id: Optional[str] = None + aws_secret_access_key: Optional[str] = None + aws_region_name: Optional[str] = None + ## IBM WATSONX ## + watsonx_region_name: Optional[str] = None + + +class GenericLiteLLMParams(CredentialLiteLLMParams): """ LiteLLM Params without 'model' arg (used across completion / assistants api) """ @@ -152,9 +171,6 @@ class GenericLiteLLMParams(BaseModel): custom_llm_provider: Optional[str] = None tpm: Optional[int] = None rpm: Optional[int] = None - api_key: Optional[str] = None - api_base: Optional[str] = None - api_version: Optional[str] = None timeout: Optional[Union[float, str, httpx.Timeout]] = ( None # if str, pass in as os.environ/ ) @@ -167,18 +183,7 @@ class GenericLiteLLMParams(BaseModel): ## LOGGING PARAMS ## litellm_trace_id: Optional[str] = None - ## UNIFIED PROJECT/REGION ## - region_name: Optional[str] = None - ## VERTEX AI ## - vertex_project: Optional[str] = None - vertex_location: Optional[str] = None - vertex_credentials: Optional[Union[str, dict]] = None - ## AWS BEDROCK / SAGEMAKER ## - aws_access_key_id: Optional[str] = None - aws_secret_access_key: Optional[str] = None - aws_region_name: Optional[str] = None - ## IBM WATSONX ## - watsonx_region_name: Optional[str] = None + ## CUSTOM PRICING ## input_cost_per_token: Optional[float] = None output_cost_per_token: Optional[float] = None @@ -245,7 +250,11 @@ class GenericLiteLLMParams(BaseModel): args.pop("__class__", None) if max_retries is not None and isinstance(max_retries, str): max_retries = int(max_retries) # cast to int - super().__init__(max_retries=max_retries, **args, **params) + # We need to keep max_retries in args since it's a parameter of GenericLiteLLMParams + args["max_retries"] = ( + max_retries # Put max_retries back in args after popping it + ) + super().__init__(**args, **params) def __contains__(self, key): # Define custom behavior for the 'in' operator From f089b1e23f7170aa8105d6e137bac31b0c10b2fe Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 14 Mar 2025 12:32:32 -0700 Subject: [PATCH 4/6] feat(endpoints.py): support adding credentials by model id Allows user to reuse existing model credentials --- litellm/litellm_core_utils/litellm_logging.py | 4 +- .../proxy/credential_endpoints/endpoints.py | 61 +++++++++++++------ litellm/router.py | 58 +++++++++++------- litellm/types/completion.py | 2 +- litellm/types/utils.py | 21 ++++++- 5 files changed, 99 insertions(+), 47 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f0b5afa67fc..a369b7f3e36 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2401,8 +2401,8 @@ def _get_masked_values( sensitive_object: dict, ignore_sensitive_values: bool = False, mask_all_values: bool = False, - unmasked_length: int = 44, - number_of_asterisks: Optional[int] = None, + unmasked_length: int = 4, + number_of_asterisks: Optional[int] = 4, ) -> dict: """ Internal debugging helper function diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 466766fc2db..5a82f44e803 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -2,7 +2,7 @@ CRUD endpoints for storing reusable credentials. """ -from typing import Optional +from typing import Optional, Union from fastapi import APIRouter, Depends, HTTPException, Request, Response @@ -14,8 +14,7 @@ from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.utils import handle_exception_on_proxy, jsonify_object -from litellm.types.router import CredentialLiteLLMParams -from litellm.types.utils import CredentialItem +from litellm.types.utils import CreateCredentialItem, CredentialItem router = APIRouter() @@ -39,7 +38,7 @@ class CredentialHelperUtils: async def create_credential( request: Request, fastapi_response: Response, - credential: CredentialItem, + credential: CreateCredentialItem, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -47,7 +46,7 @@ async def create_credential( Stores credential in DB. Reloads credentials in memory. """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import llm_router, prisma_client try: if prisma_client is None: @@ -55,9 +54,35 @@ async def create_credential( status_code=500, detail={"error": CommonProxyErrors.db_not_connected_error.value}, ) + if credential.model_id: + if llm_router is None: + raise HTTPException( + status_code=500, + detail="LLM router not found. Please ensure you have a valid router instance.", + ) + # get model from router + model = llm_router.get_deployment(credential.model_id) + if model is None: + raise HTTPException(status_code=404, detail="Model not found") + credential_values = llm_router.get_deployment_credentials( + credential.model_id + ) + if credential_values is None: + raise HTTPException(status_code=404, detail="Model not found") + credential.credential_values = credential_values + if credential.credential_values is None: + raise HTTPException( + status_code=400, + detail="Credential values are required. Unable to infer credential values from model ID.", + ) + processed_credential = CredentialItem( + credential_name=credential.credential_name, + credential_values=credential.credential_values, + credential_info=credential.credential_info, + ) encrypted_credential = CredentialHelperUtils.encrypt_credential_values( - credential + processed_credential ) credentials_dict = encrypted_credential.model_dump() credentials_dict_jsonified = jsonify_object(credentials_dict) @@ -70,7 +95,7 @@ async def create_credential( ) ## ADD TO LITELLM ## - CredentialAccessor.upsert_credentials([credential]) + CredentialAccessor.upsert_credentials([processed_credential]) return {"success": True, "message": "Credential created successfully"} except Exception as e: @@ -95,6 +120,7 @@ async def get_credentials( masked_credentials = [ { "credential_name": credential.credential_name, + "credential_values": _get_masked_values(credential.credential_values), "credential_info": credential.credential_info, } for credential in litellm.credential_list @@ -132,23 +158,20 @@ async def get_credential( if model_id: if llm_router is None: raise HTTPException(status_code=500, detail="LLM router not found") - # get model from router model = llm_router.get_deployment(model_id) if model is None: - raise HTTPException( - status_code=404, detail="Model not found. Got model ID: " + model_id - ) - # get credential object from model - credential_values = _get_masked_values( - CredentialLiteLLMParams(**model.litellm_params.model_dump()).model_dump( - exclude_none=True - ), + raise HTTPException(status_code=404, detail="Model not found") + credential_values = llm_router.get_deployment_credentials(model_id) + if credential_values is None: + raise HTTPException(status_code=404, detail="Model not found") + masked_credential_values = _get_masked_values( + credential_values, unmasked_length=4, number_of_asterisks=4, ) credential = CredentialItem( credential_name="{}-credential-{}".format(model.model_name, model_id), - credential_values=credential_values, + credential_values=masked_credential_values, credential_info={}, ) # return credential object @@ -159,7 +182,9 @@ async def get_credential( masked_credential = CredentialItem( credential_name=credential.credential_name, credential_values=_get_masked_values( - credential.credential_values + credential.credential_values, + unmasked_length=4, + number_of_asterisks=4, ), credential_info=credential.credential_info, ) diff --git a/litellm/router.py b/litellm/router.py index f7f361354b2..a395c851dd8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -111,6 +111,7 @@ from litellm.types.router import ( AlertingConfig, AllowedFailsPolicy, AssistantsTypedDict, + CredentialLiteLLMParams, CustomRoutingStrategyBase, Deployment, DeploymentTypedDict, @@ -636,29 +637,6 @@ class Router: if self.cache.redis_cache is None: self.cache.redis_cache = cache - def initialize_assistants_endpoint(self): - ## INITIALIZE PASS THROUGH ASSISTANTS ENDPOINT ## - self.acreate_assistants = self.factory_function(litellm.acreate_assistants) - self.adelete_assistant = self.factory_function(litellm.adelete_assistant) - self.aget_assistants = self.factory_function(litellm.aget_assistants) - self.acreate_thread = self.factory_function(litellm.acreate_thread) - self.aget_thread = self.factory_function(litellm.aget_thread) - self.a_add_message = self.factory_function(litellm.a_add_message) - self.aget_messages = self.factory_function(litellm.aget_messages) - self.arun_thread = self.factory_function(litellm.arun_thread) - - def initialize_router_endpoints(self): - self.amoderation = self.factory_function( - litellm.amoderation, call_type="moderation" - ) - self.aanthropic_messages = self.factory_function( - litellm.anthropic_messages, call_type="anthropic_messages" - ) - self.aresponses = self.factory_function( - litellm.aresponses, call_type="aresponses" - ) - self.responses = self.factory_function(litellm.responses, call_type="responses") - def routing_strategy_init( self, routing_strategy: Union[RoutingStrategy, str], routing_strategy_args: dict ): @@ -724,6 +702,29 @@ class Router: else: pass + def initialize_assistants_endpoint(self): + ## INITIALIZE PASS THROUGH ASSISTANTS ENDPOINT ## + self.acreate_assistants = self.factory_function(litellm.acreate_assistants) + self.adelete_assistant = self.factory_function(litellm.adelete_assistant) + self.aget_assistants = self.factory_function(litellm.aget_assistants) + self.acreate_thread = self.factory_function(litellm.acreate_thread) + self.aget_thread = self.factory_function(litellm.aget_thread) + self.a_add_message = self.factory_function(litellm.a_add_message) + self.aget_messages = self.factory_function(litellm.aget_messages) + self.arun_thread = self.factory_function(litellm.arun_thread) + + def initialize_router_endpoints(self): + self.amoderation = self.factory_function( + litellm.amoderation, call_type="moderation" + ) + self.aanthropic_messages = self.factory_function( + litellm.anthropic_messages, call_type="anthropic_messages" + ) + self.aresponses = self.factory_function( + litellm.aresponses, call_type="aresponses" + ) + self.responses = self.factory_function(litellm.responses, call_type="responses") + def validate_fallbacks(self, fallback_param: Optional[List]): """ Validate the fallbacks parameter. @@ -4625,6 +4626,17 @@ class Router: raise Exception("Model invalid format - {}".format(type(model))) return None + def get_deployment_credentials(self, model_id: str) -> Optional[dict]: + """ + Returns -> dict of credentials for a given model id + """ + deployment = self.get_deployment(model_id=model_id) + if deployment is None: + return None + return CredentialLiteLLMParams( + **deployment.litellm_params.model_dump(exclude_none=True) + ).model_dump(exclude_none=True) + def get_deployment_by_model_group_name( self, model_group_name: str ) -> Optional[Deployment]: diff --git a/litellm/types/completion.py b/litellm/types/completion.py index 7b5ed4e502b..b06bb733c4f 100644 --- a/litellm/types/completion.py +++ b/litellm/types/completion.py @@ -1,6 +1,6 @@ from typing import Iterable, List, Optional, Union -from pydantic import BaseModel, ConfigDict, validator +from pydantic import BaseModel, ConfigDict from typing_extensions import Literal, Required, TypedDict diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 9608c099a33..a2d41d8fb9d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -18,7 +18,7 @@ from openai.types.moderation import ( CategoryScores, ) from openai.types.moderation_create_response import Moderation, ModerationCreateResponse -from pydantic import BaseModel, ConfigDict, Field, PrivateAttr +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator from typing_extensions import Callable, Dict, Required, TypedDict, override import litellm @@ -2053,7 +2053,22 @@ class RawRequestTypedDict(TypedDict, total=False): error: Optional[str] -class CredentialItem(BaseModel): +class CredentialBase(BaseModel): credential_name: str - credential_values: dict credential_info: dict + + +class CredentialItem(CredentialBase): + credential_values: dict + + +class CreateCredentialItem(CredentialBase): + credential_values: Optional[dict] = None + model_id: Optional[str] = None + + @model_validator(mode="before") + @classmethod + def check_credential_params(cls, values): + if not values.get("credential_values") and not values.get("model_id"): + raise ValueError("Either credential_values or model_id must be set") + return values From 9362fc175907007aad969203d419602682c4957f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 14 Mar 2025 12:52:49 -0700 Subject: [PATCH 5/6] feat(ui/): allow admin to reuse existing model credentials Prevents need to go to backend llm provider for getting credentials --- .../add_model/provider_specific_fields.tsx | 525 ++++++++++++------ .../model_add/reuse_credentials.tsx | 119 ++++ .../src/components/model_info_view.tsx | 77 ++- .../src/components/networking.tsx | 36 ++ 4 files changed, 598 insertions(+), 159 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/model_add/reuse_credentials.tsx diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx index b7565b0494f..62fd3e4eb6d 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx +++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx @@ -3,132 +3,392 @@ import { Form, Select } from "antd"; import { TextInput, Text } from "@tremor/react"; import { Row, Col, Typography, Button as Button2, Upload, UploadProps } from "antd"; import { UploadOutlined } from "@ant-design/icons"; -import { Providers } from "../provider_info_helpers"; +import { provider_map, Providers } from "../provider_info_helpers"; +import { CredentialItem } from "../networking"; const { Link } = Typography; + interface ProviderSpecificFieldsProps { selectedProvider: Providers; uploadProps?: UploadProps; } +interface ProviderCredentialField { + key: string; + label: string; + placeholder?: string; + tooltip?: string; + required?: boolean; + type?: "text" | "password" | "select" | "upload"; + options?: string[]; + defaultValue?: string; +} + +export interface CredentialValues { + key: string; + value: string; +} + + +export const createCredentialFromModel = (provider: string, modelData: any): CredentialItem => { + console.log("provider", provider); + console.log("modelData", modelData); + const enumKey = Object.keys(provider_map).find( + key => provider_map[key].toLowerCase() === provider.toLowerCase() + ); + if (!enumKey) { + throw new Error(`Provider ${provider} not found in provider_map`); + } + const providerEnum = Providers[enumKey as keyof typeof Providers]; + const providerFields = PROVIDER_CREDENTIAL_FIELDS[providerEnum] || []; + const credentialValues: object = {}; + + console.log("providerFields", providerFields); + + // Go through each field defined for this provider + providerFields.forEach(field => { + const value = modelData.litellm_params[field.key]; + console.log("field", field); + console.log("value", value); + if (value !== undefined) { + (credentialValues as Record)[field.key] = value.toString(); + } + }); + + const credential: CredentialItem = { + credential_name: `${provider}-credential-${Math.floor(Math.random() * 1000000)}`, + credential_values: credentialValues, + credential_info: { + custom_llm_provider: provider, + description: `Credential for ${provider}. Created from model ${modelData.model_name}`, + } + } + + return credential; +}; + +const PROVIDER_CREDENTIAL_FIELDS: Record = { + [Providers.OpenAI]: [ + { + key: "api_base", + label: "API Base", + type: "select", + options: [ + "https://api.openai.com/v1", + "https://eu.api.openai.com" + ], + defaultValue: "https://api.openai.com/v1" + }, + { + key: "organization", + label: "OpenAI Organization ID", + placeholder: "[OPTIONAL] my-unique-org" + }, + { + key: "api_key", + label: "OpenAI API Key", + type: "password", + required: true + } + ], + [Providers.OpenAI_Text]: [ + { + key: "api_base", + label: "API Base", + type: "select", + options: [ + "https://api.openai.com/v1", + "https://eu.api.openai.com" + ], + defaultValue: "https://api.openai.com/v1" + }, + { + key: "organization", + label: "OpenAI Organization ID", + placeholder: "[OPTIONAL] my-unique-org" + }, + { + key: "api_key", + label: "OpenAI API Key", + type: "password", + required: true + } + ], + [Providers.Vertex_AI]: [ + { + key: "vertex_project", + label: "Vertex Project", + placeholder: "adroit-cadet-1234..", + required: true + }, + { + key: "vertex_location", + label: "Vertex Location", + placeholder: "us-east-1", + required: true + }, + { + key: "vertex_credentials", + label: "Vertex Credentials", + required: true, + type: "upload" + } + ], + [Providers.AssemblyAI]: [ + { + key: "api_base", + label: "API Base", + type: "select", + required: true, + options: [ + "https://api.assemblyai.com", + "https://api.eu.assemblyai.com" + ] + }, + { + key: "api_key", + label: "AssemblyAI API Key", + type: "password", + required: true + } + ], + [Providers.Azure]: [ + { + key: "api_base", + label: "API Base", + placeholder: "https://...", + required: true + }, + { + key: "api_version", + label: "API Version", + placeholder: "2023-07-01-preview", + tooltip: "By default litellm will use the latest version. If you want to use a different version, you can specify it here" + }, + { + key: "base_model", + label: "Base Model", + placeholder: "azure/gpt-3.5-turbo" + }, + { + key: "api_key", + label: "Azure API Key", + type: "password", + required: true + } + ], + [Providers.Azure_AI_Studio]: [ + { + key: "api_base", + label: "API Base", + placeholder: "https://...", + required: true + }, + { + key: "api_key", + label: "Azure API Key", + type: "password", + required: true + } + ], + [Providers.OpenAI_Compatible]: [ + { + key: "api_base", + label: "API Base", + placeholder: "https://...", + required: true + }, + { + key: "api_key", + label: "OpenAI API Key", + type: "password", + required: true + } + ], + [Providers.OpenAI_Text_Compatible]: [ + { + key: "api_base", + label: "API Base", + placeholder: "https://...", + required: true + }, + { + key: "api_key", + label: "OpenAI API Key", + type: "password", + required: true + } + ], + [Providers.Bedrock]: [ + { + key: "aws_access_key_id", + label: "AWS Access Key ID", + required: true, + tooltip: "You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`)." + }, + { + key: "aws_secret_access_key", + label: "AWS Secret Access Key", + required: true, + tooltip: "You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`)." + }, + { + key: "aws_region_name", + label: "AWS Region Name", + placeholder: "us-east-1", + required: true, + tooltip: "You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`)." + } + ], + [Providers.Ollama]: [], // No specific fields needed + [Providers.Anthropic]: [{ + key: "api_key", + label: "API Key", + placeholder: "sk-", + type: "password", + required: true + }], + [Providers.Google_AI_Studio]: [{ + key: "api_key", + label: "API Key", + placeholder: "aig-", + type: "password", + required: true + }], + [Providers.Groq]: [{ + key: "api_key", + label: "API Key", + type: "password", + required: true + }], + [Providers.MistralAI]: [{ + key: "api_key", + label: "API Key", + type: "password", + required: true + }], + [Providers.Deepseek]: [{ + key: "api_key", + label: "API Key", + type: "password", + required: true + }], + [Providers.Cohere]: [{ + key: "api_key", + label: "API Key", + type: "password", + required: true + }], + [Providers.Databricks]: [{ + key: "api_key", + label: "API Key", + type: "password", + required: true + }], + [Providers.xAI]: [{ + key: "api_key", + label: "API Key", + type: "password", + required: true + }], + [Providers.Cerebras]: [{ + key: "api_key", + label: "API Key", + type: "password", + required: true + }], + [Providers.Sambanova]: [{ + key: "api_key", + label: "API Key", + type: "password", + required: true + }], + [Providers.Perplexity]: [{ + key: "api_key", + label: "API Key", + type: "password", + required: true + }], + [Providers.TogetherAI]: [{ + key: "api_key", + label: "API Key", + type: "password", + required: true + }], + [Providers.Openrouter]: [{ + key: "api_key", + label: "API Key", + type: "password", + required: true + }], + [Providers.FireworksAI]: [{ + key: "api_key", + label: "API Key", + type: "password", + required: true + }] +}; + const ProviderSpecificFields: React.FC = ({ selectedProvider, uploadProps }) => { - console.log(`Selected provider: ${selectedProvider}`); - console.log(`type of selectedProvider: ${typeof selectedProvider}`); - // cast selectedProvider to Providers const selectedProviderEnum = Providers[selectedProvider as keyof typeof Providers] as Providers; - console.log(`selectedProviderEnum: ${selectedProviderEnum}`); - console.log(`type of selectedProviderEnum: ${typeof selectedProviderEnum}`); + + // Simply use the fields as defined in PROVIDER_CREDENTIAL_FIELDS + const allFields = React.useMemo(() => { + return PROVIDER_CREDENTIAL_FIELDS[selectedProviderEnum] || []; + }, [selectedProviderEnum]); + return ( <> - {selectedProviderEnum === Providers.OpenAI || selectedProviderEnum === Providers.OpenAI_Text && ( - <> + {allFields.map((field) => ( + - - - - - - - - )} - - {selectedProviderEnum === Providers.Vertex_AI && ( - <> - - + {field.type === "select" ? ( + + ) : field.type === "upload" ? ( + + }>Click to Upload + + ) : ( + + )} - - - + {/* Special case for Vertex Credentials help text */} + {field.key === "vertex_credentials" && ( + + + + + Give litellm a gcp service account(.json file), so it + can make the relevant calls + + + + )} - - - }> - Click to Upload - - - - - - - - - Give litellm a gcp service account(.json file), so it - can make the relevant calls - - - - - )} - - {selectedProviderEnum === Providers.AssemblyAI && ( - - - - )} - - {(selectedProviderEnum === Providers.Azure || - selectedProviderEnum === Providers.Azure_AI_Studio || - selectedProviderEnum === Providers.OpenAI_Compatible || - selectedProviderEnum === Providers.OpenAI_Text_Compatible - ) && ( - - - - )} - - {selectedProviderEnum === Providers.Azure && ( - <> - - - - -
- - - + {/* Special case for Azure Base Model help text */} + {field.key === "base_model" && ( @@ -144,54 +404,9 @@ const ProviderSpecificFields: React.FC = ({ -
- - )} - - {selectedProviderEnum === Providers.Bedrock && ( - <> - - - - - - - - - - - - - )} - - {selectedProviderEnum != Providers.Bedrock && - selectedProviderEnum != Providers.Vertex_AI && - selectedProviderEnum != Providers.Ollama && - ( - - - - )} + )} + + ))} ); }; diff --git a/ui/litellm-dashboard/src/components/model_add/reuse_credentials.tsx b/ui/litellm-dashboard/src/components/model_add/reuse_credentials.tsx new file mode 100644 index 00000000000..50aa4e956ec --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_add/reuse_credentials.tsx @@ -0,0 +1,119 @@ +import React, { useState } from "react"; +import { + Card, + Form, + Button, + Tooltip, + Typography, + Select as AntdSelect, + Input, + Switch, + Modal +} from "antd"; +import type { UploadProps } from "antd/es/upload"; +import { Providers, providerLogoMap } from "../provider_info_helpers"; +import type { FormInstance } from "antd"; +import ProviderSpecificFields from "../add_model/provider_specific_fields"; +import { TextInput } from "@tremor/react"; +import { CredentialItem } from "../networking"; +const { Title, Link } = Typography; + +interface ReuseCredentialsModalProps { + isVisible: boolean; + onCancel: () => void; + onAddCredential: (values: any) => void; + existingCredential: CredentialItem | null; + setIsCredentialModalOpen: (isVisible: boolean) => void; +} + +const ReuseCredentialsModal: React.FC = ({ + isVisible, + onCancel, + onAddCredential, + existingCredential, + setIsCredentialModalOpen +}) => { + const [form] = Form.useForm(); + + console.log(`existingCredential in add credentials tab: ${JSON.stringify(existingCredential)}`); + + const handleSubmit = (values: any) => { + onAddCredential(values); + form.resetFields(); + setIsCredentialModalOpen(false); + }; + + return ( + { + onCancel(); + form.resetFields(); + }} + footer={null} + width={600} + > +
+ {/* Credential Name */} + + + + + {/* Display Credential Values of existingCredential, don't allow user to edit. Credential values is a dictionary */} + {Object.entries(existingCredential?.credential_values || {}).map(([key, value]) => ( + + + + ))} + + {/* Modal Footer */} +
+ + + Need Help? + + + +
+ + +
+
+
+ + ); +}; + +export default ReuseCredentialsModal; \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index bc6a831d628..7768aa71cbb 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -14,13 +14,15 @@ import { TextInput, NumberInput, } from "@tremor/react"; -import { ArrowLeftIcon, TrashIcon } from "@heroicons/react/outline"; -import { modelDeleteCall, modelUpdateCall } from "./networking"; -import { Button, Form, Input, InputNumber, message, Select } from "antd"; +import { ArrowLeftIcon, TrashIcon, KeyIcon } from "@heroicons/react/outline"; +import { modelDeleteCall, modelUpdateCall, CredentialItem, credentialGetCall, credentialCreateCall } from "./networking"; +import { Button, Form, Input, InputNumber, message, Select, Modal } from "antd"; import EditModelModal from "./edit_model/edit_model_modal"; import { handleEditModelSubmit } from "./edit_model/edit_model_modal"; import { getProviderLogoAndName } from "./provider_info_helpers"; import { getDisplayModelName } from "./view_model/model_name_display"; +import AddCredentialsModal from "./model_add/add_credentials_tab"; +import ReuseCredentialsModal from "./model_add/reuse_credentials"; interface ModelInfoViewProps { modelId: string; @@ -48,11 +50,51 @@ export default function ModelInfoView({ const [form] = Form.useForm(); const [localModelData, setLocalModelData] = useState(modelData); const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); + const [isCredentialModalOpen, setIsCredentialModalOpen] = useState(false); const [isDirty, setIsDirty] = useState(false); const [isSaving, setIsSaving] = useState(false); const [isEditing, setIsEditing] = useState(false); + const [existingCredential, setExistingCredential] = useState(null); const canEditModel = userRole === "Admin"; + const isAdmin = userRole === "Admin"; + + const usingExistingCredential = modelData.litellm_params?.litellm_credential_name != null && modelData.litellm_params?.litellm_credential_name != undefined; + console.log("usingExistingCredential, ", usingExistingCredential); + console.log("modelData.litellm_params.litellm_credential_name, ", modelData.litellm_params.litellm_credential_name); + + + useEffect(() => { + const getExistingCredential = async () => { + console.log("accessToken, ", accessToken); + if (!accessToken) return; + if (usingExistingCredential) return; + let existingCredentialResponse = await credentialGetCall(accessToken, null, modelId); + console.log("existingCredentialResponse, ", existingCredentialResponse); + setExistingCredential({ + credential_name: existingCredentialResponse["credential_name"], + credential_values: existingCredentialResponse["credential_values"], + credential_info: existingCredentialResponse["credential_info"] + }); + } + getExistingCredential(); + }, [accessToken, modelId]); + + const handleReuseCredential = async (values: any) => { + console.log("values, ", values); + if (!accessToken) return; + let credentialItem = { + credential_name: values.credential_name, + model_id: modelId, + credential_info: { + "custom_llm_provider": localModelData.litellm_params?.custom_llm_provider, + } + } + message.info("Storing credential.."); + let credentialResponse = await credentialCreateCall(accessToken, credentialItem); + console.log("credentialResponse, ", credentialResponse); + message.success("Credential stored successfully"); + } const handleModelUpdate = async (values: any) => { try { @@ -143,8 +185,16 @@ export default function ModelInfoView({ Public Model Name: {getDisplayModelName(modelData)} {modelData.model_info.id} - {canEditModel && ( + {isAdmin && (
+ setIsCredentialModalOpen(true)} + className="flex items-center" + > + Re-use Credentials +
)} + + {isCredentialModalOpen && + !usingExistingCredential ? ( + setIsCredentialModalOpen(false)} + onAddCredential={handleReuseCredential} + existingCredential={existingCredential} + setIsCredentialModalOpen={setIsCredentialModalOpen} + /> + ): ( + setIsCredentialModalOpen(false)} + title="Using Existing Credential" + > + {modelData.litellm_params.litellm_credential_name} + + )} ); } \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 2cd122aba80..0c1ba991aca 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2652,6 +2652,42 @@ export const credentialListCall = async ( } }; +export const credentialGetCall = async (accessToken: String, credentialName: String | null, modelId: String | null) => { + try { + let url = proxyBaseUrl ? `${proxyBaseUrl}/credentials` : `/credentials`; + + if (credentialName) { + url += `/by_name/${credentialName}`; + } else if (modelId) { + url += `/by_model/${modelId}`; + } + + console.log("in credentialListCall"); + + const response = await fetch(url, { + method: "GET", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + handleError(errorData); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + console.log("/credentials API Response:", data); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to create key:", error); + throw error; + } +}; + export const credentialDeleteCall = async (accessToken: String, credentialName: String) => { try { const url = proxyBaseUrl ? `${proxyBaseUrl}/credentials/${credentialName}` : `/credentials/${credentialName}`; From 08abee199014726b8c0d7778dbb00d1abd288acf Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 14 Mar 2025 14:17:28 -0700 Subject: [PATCH 6/6] fix: fix linting error --- litellm/proxy/credential_endpoints/endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 5a82f44e803..bbbbbfd6b74 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -2,7 +2,7 @@ CRUD endpoints for storing reusable credentials. """ -from typing import Optional, Union +from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Request, Response