diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.32-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.32-py3-none-any.whl new file mode 100644 index 00000000000..deb9653aa78 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.32-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.32.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.32.tar.gz new file mode 100644 index 00000000000..212194e31e2 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.32.tar.gz differ diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260207093506_add_available_on_public_internet_to_mcp_servers/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260207093506_add_available_on_public_internet_to_mcp_servers/migration.sql new file mode 100644 index 00000000000..1efde3dbe0f --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260207093506_add_available_on_public_internet_to_mcp_servers/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "available_on_public_internet" BOOLEAN NOT NULL DEFAULT false; + diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 0e72cd90813..d957da1e6df 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.4.31" +version = "0.4.32" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.4.31" +version = "0.4.32" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 7df86d44008..4ea22dbd90f 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1418,37 +1418,47 @@ def completion_cost( # noqa: PLR0915 # Apply discount from module-level config if configured original_cost = _final_cost - _final_cost, discount_percent, discount_amount = _apply_cost_discount( - base_cost=_final_cost, - custom_llm_provider=custom_llm_provider, - ) + if litellm.cost_discount_config: + _final_cost, discount_percent, discount_amount = _apply_cost_discount( + base_cost=_final_cost, + custom_llm_provider=custom_llm_provider, + ) + else: + discount_percent = 0.0 + discount_amount = 0.0 # Apply margin from module-level config if configured - ( - _final_cost, - margin_percent, - margin_fixed_amount, - margin_total_amount, - ) = _apply_cost_margin( - base_cost=_final_cost, - custom_llm_provider=custom_llm_provider, - ) + if litellm.cost_margin_config: + ( + _final_cost, + margin_percent, + margin_fixed_amount, + margin_total_amount, + ) = _apply_cost_margin( + base_cost=_final_cost, + custom_llm_provider=custom_llm_provider, + ) + else: + margin_percent = 0.0 + margin_fixed_amount = 0.0 + margin_total_amount = 0.0 # Store cost breakdown in logging object if available - _store_cost_breakdown_in_logging_obj( - litellm_logging_obj=litellm_logging_obj, - prompt_tokens_cost_usd_dollar=prompt_tokens_cost_usd_dollar, - completion_tokens_cost_usd_dollar=completion_tokens_cost_usd_dollar, - cost_for_built_in_tools_cost_usd_dollar=cost_for_built_in_tools, - total_cost_usd_dollar=_final_cost, - additional_costs=additional_costs, - original_cost=original_cost, - discount_percent=discount_percent, - discount_amount=discount_amount, - margin_percent=margin_percent, - margin_fixed_amount=margin_fixed_amount, - margin_total_amount=margin_total_amount, - ) + if litellm_logging_obj is not None: + _store_cost_breakdown_in_logging_obj( + litellm_logging_obj=litellm_logging_obj, + prompt_tokens_cost_usd_dollar=prompt_tokens_cost_usd_dollar, + completion_tokens_cost_usd_dollar=completion_tokens_cost_usd_dollar, + cost_for_built_in_tools_cost_usd_dollar=cost_for_built_in_tools, + total_cost_usd_dollar=_final_cost, + original_cost=original_cost, + additional_costs=additional_costs, + discount_percent=discount_percent, + discount_amount=discount_amount, + margin_percent=margin_percent, + margin_fixed_amount=margin_fixed_amount, + margin_total_amount=margin_total_amount, + ) return _final_cost except Exception as e: diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 07d237c4758..4dc346bbc18 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -740,7 +740,9 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac return OLD_LITELLM_METADATA_FIELD def redact_standard_logging_payload_from_model_call_details( - self, model_call_details: Dict + self, + model_call_details: Dict, + global_redaction_applied: bool = False, ) -> Dict: """ Only redacts messages and responses when self.turn_off_message_logging is True @@ -752,6 +754,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac This is useful for logging payloads that contain sensitive information. """ + # Only skip if global redaction applied AND method is not overridden + if global_redaction_applied: + # Check if method was overridden anywhere in the inheritance chain (walks full MRO) + method_name = "redact_standard_logging_payload_from_model_call_details" + is_overridden = getattr(type(self), method_name) is not getattr(CustomLogger, method_name) + + if not is_overridden: + # Safe to skip - using default implementation + return model_call_details + # Method was overridden - might do additional redaction, so proceed + from copy import copy from litellm import Choices, Message, ModelResponse diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7a82a7ff788..16c95d15512 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -68,6 +68,7 @@ from litellm.litellm_core_utils.model_param_helper import ModelParamHelper from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_custom_logger, redact_message_input_output_from_logging, + should_redact_message_logging, ) from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.llms.base_llm.search.transformation import SearchResponse @@ -203,6 +204,10 @@ except Exception as e: EnterpriseStandardLoggingPayloadSetupVAR = None _in_memory_loggers: List[Any] = [] +_STANDARD_LOGGING_METADATA_KEYS: frozenset = frozenset( + StandardLoggingMetadata.__annotations__.keys() +) + ### GLOBAL VARIABLES ### # Cache custom pricing keys as frozenset for O(1) lookups instead of looping through 49 keys @@ -2492,11 +2497,12 @@ class Logging(LiteLLMLoggingBaseClass): global_callbacks=litellm._async_success_callback, ) + _model_call_details = self.model_call_details if hasattr(self, "model_call_details") else {} + global_redaction_applied = should_redact_message_logging(_model_call_details) result = redact_message_input_output_from_logging( - model_call_details=( - self.model_call_details if hasattr(self, "model_call_details") else {} - ), + model_call_details=_model_call_details, result=result, + should_redact=global_redaction_applied, ) ## LOGGING HOOK ## @@ -2521,7 +2527,10 @@ class Logging(LiteLLMLoggingBaseClass): ) elif isinstance(callback, CustomLogger): result = redact_message_input_output_from_custom_logger( - result=result, litellm_logging_obj=self, custom_logger=callback + result=result, + litellm_logging_obj=self, + custom_logger=callback, + global_redaction_applied=global_redaction_applied, ) self.model_call_details, result = await callback.async_logging_hook( kwargs=self.model_call_details, @@ -2576,7 +2585,8 @@ class Logging(LiteLLMLoggingBaseClass): ################################## # call redaction hook for custom logger model_call_details = callback.redact_standard_logging_payload_from_model_call_details( - model_call_details=model_call_details + model_call_details=model_call_details, + global_redaction_applied=global_redaction_applied, ) ################################## if self.stream is True: @@ -4530,17 +4540,12 @@ class StandardLoggingPayloadSetup: user_api_key_auth_metadata=None, ) if isinstance(metadata, dict): - # Filter the metadata dictionary to include only the specified keys - supported_keys = StandardLoggingMetadata.__annotations__.keys() - for key in supported_keys: - if key in metadata: - clean_metadata[key] = metadata[key] # type: ignore + for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS: + clean_metadata[key] = metadata[key] # type: ignore - if metadata.get("user_api_key") is not None: - if is_valid_sha256_hash(str(metadata.get("user_api_key"))): - clean_metadata["user_api_key_hash"] = metadata.get( - "user_api_key" - ) # this is the hash + user_api_key = metadata.get("user_api_key") + if user_api_key and isinstance(user_api_key, str) and is_valid_sha256_hash(user_api_key): + clean_metadata["user_api_key_hash"] = user_api_key _potential_requester_metadata = metadata.get( "metadata", None ) # check if user passed metadata in the sdk request - e.g. metadata for langsmith logging - https://docs.litellm.ai/docs/observability/langsmith_integration#set-langsmith-fields diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index aa763dc9899..c268af20e3e 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -30,8 +30,15 @@ else: def redact_message_input_output_from_custom_logger( - litellm_logging_obj: LiteLLMLoggingObject, result, custom_logger: CustomLogger + litellm_logging_obj: LiteLLMLoggingObject, + result, + custom_logger: CustomLogger, + global_redaction_applied: bool = False, ): + # skip redundant redaction if global redaction was already applied + if global_redaction_applied: + return result + if ( hasattr(custom_logger, "message_logging") and custom_logger.message_logging is not True @@ -72,6 +79,44 @@ def _redact_responses_api_output(output_items): summary_item.text = "redacted-by-litellm" +def _redact_standard_logging_object(model_call_details: dict): + """Redact messages and response inside standard_logging_object if present.""" + standard_logging_object = model_call_details.get("standard_logging_object") + if standard_logging_object is None: + return + + redacted_str = "redacted-by-litellm" + + if standard_logging_object.get("messages") is not None: + standard_logging_object["messages"] = [ + {"role": "user", "content": redacted_str} + ] + + response = standard_logging_object.get("response") + if response is not None: + if isinstance(response, dict) and "output" in response: + # ResponsesAPIResponse format - redact content in output items + if isinstance(response.get("output"), list): + for output_item in response["output"]: + if isinstance(output_item, dict) and "content" in output_item: + if isinstance(output_item["content"], list): + for content_item in output_item["content"]: + if ( + isinstance(content_item, dict) + and "text" in content_item + ): + content_item["text"] = redacted_str + elif isinstance(response, str): + standard_logging_object["response"] = redacted_str + else: + # Standard ModelResponse dict format + standard_logging_object["response"] = { + "choices": [ + {"message": {"content": redacted_str}} + ] + } + + def perform_redaction(model_call_details: dict, result): """ Performs the actual redaction on the logging object and result. @@ -83,6 +128,9 @@ def perform_redaction(model_call_details: dict, result): model_call_details["prompt"] = "" model_call_details["input"] = "" + # Redact standard_logging_object if present + _redact_standard_logging_object(model_call_details) + # Redact streaming response if ( model_call_details.get("stream", False) is True @@ -177,13 +225,18 @@ def should_redact_message_logging(model_call_details: dict) -> bool: def redact_message_input_output_from_logging( - model_call_details: dict, result, input: Optional[Any] = None + model_call_details: dict, + result, + input: Optional[Any] = None, + should_redact: Optional[bool] = None, ) -> Any: """ Removes messages, prompts, input, response from logging. This modifies the data in-place only redacts when litellm.turn_off_message_logging == True """ - if should_redact_message_logging(model_call_details): + if should_redact is None: + should_redact = should_redact_message_logging(model_call_details) + if should_redact: return perform_redaction(model_call_details, result) return result diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 2cd56e0ff3f..2fe40bb197e 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -99,6 +99,62 @@ if MCP_AVAILABLE: ) return mcp_auth_header, mcp_server_auth_headers, raw_headers + async def _resolve_allowed_mcp_servers_with_ip_filter( + request: Request, + user_api_key_dict: UserAPIKeyAuth, + server_id: str, + ) -> List[MCPServer]: + """ + Resolve allowed MCP servers for a tool call with IP filtering. + + Args: + request: The HTTP request object + user_api_key_dict: The user's API key auth object + server_id: The server ID to validate access for + + Returns: + List of allowed MCPServer objects + + Raises: + HTTPException: If the server_id is not allowed + """ + # Get all auth contexts + auth_contexts = await build_effective_auth_contexts(user_api_key_dict) + + # Collect allowed server IDs from all contexts, then apply IP filtering + _rest_client_ip = IPAddressUtils.get_mcp_client_ip(request) + allowed_server_ids_set = set() + for auth_context in auth_contexts: + servers = await global_mcp_server_manager.get_allowed_mcp_servers( + user_api_key_auth=auth_context, + ) + allowed_server_ids_set.update(servers) + + allowed_server_ids_set = set( + global_mcp_server_manager.filter_server_ids_by_ip( + list(allowed_server_ids_set), _rest_client_ip + ) + ) + + # Check if the specified server_id is allowed + if server_id not in allowed_server_ids_set: + raise HTTPException( + status_code=403, + detail={ + "error": "access_denied", + "message": f"The key is not allowed to access server {server_id}", + }, + ) + + # Build allowed_mcp_servers list (only include allowed servers) + allowed_mcp_servers: List[MCPServer] = [] + for allowed_server_id in allowed_server_ids_set: + server = global_mcp_server_manager.get_mcp_server_by_id(allowed_server_id) + if server is not None: + allowed_mcp_servers.append(server) + + return allowed_mcp_servers + async def _get_tools_for_single_server( server, server_auth_header, @@ -381,43 +437,11 @@ if MCP_AVAILABLE: if "metadata" in data and "user_api_key_auth" in data["metadata"]: data["user_api_key_auth"] = data["metadata"]["user_api_key_auth"] - # Get all auth contexts - auth_contexts = await build_effective_auth_contexts(user_api_key_dict) - - # Collect allowed server IDs from all contexts, then apply IP filtering - _rest_client_ip = IPAddressUtils.get_mcp_client_ip(request) - allowed_server_ids_set = set() - for auth_context in auth_contexts: - servers = await global_mcp_server_manager.get_allowed_mcp_servers( - user_api_key_auth=auth_context, - ) - allowed_server_ids_set.update(servers) - - allowed_server_ids_set = set( - global_mcp_server_manager.filter_server_ids_by_ip( - list(allowed_server_ids_set), _rest_client_ip - ) + # Resolve allowed MCP servers with IP filtering + allowed_mcp_servers = await _resolve_allowed_mcp_servers_with_ip_filter( + request, user_api_key_dict, server_id ) - # Check if the specified server_id is allowed - if server_id not in allowed_server_ids_set: - raise HTTPException( - status_code=403, - detail={ - "error": "access_denied", - "message": f"The key is not allowed to access server {server_id}", - }, - ) - - # Build allowed_mcp_servers list (only include allowed servers) - allowed_mcp_servers: List[MCPServer] = [] - for allowed_server_id in allowed_server_ids_set: - server = global_mcp_server_manager.get_mcp_server_by_id( - allowed_server_id - ) - if server is not None: - allowed_mcp_servers.append(server) - # Call execute_mcp_tool directly (permission checks already done) result = await execute_mcp_tool( name=tool_name, diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html index e40a85b4ddb..63f46b9c025 100644 --- a/litellm/proxy/_experimental/out/404.html +++ b/litellm/proxy/_experimental/out/404.html @@ -1 +1 @@ -
=1e7/2&&++S;do f=0,(u=e(T,v,P,m))<0?(b=v[0],P!=m&&(b=1e7*b+(v[1]||0)),(f=b/S|0)>1?(f>=1e7&&(f=1e7-1),h=(p=t(T,f)).length,m=v.length,1==(u=e(p,v,h,m))&&(f--,r(p,P 0?i=i.charAt(0)+"."+i.slice(1)+j(n):a>1&&(i=i.charAt(0)+"."+i.slice(1)),i=i+(o<0?"e":"e+")+o):o<0?(i="0."+j(-o-1)+i,r&&(n=r-a)>0&&(i+=j(n))):o>=a?(i+=j(o+1-a),r&&(n=r-o-1)>0&&(i=i+"."+j(n))):((n=o+1)0&&(o+1===a&&(i+="."),i+=j(n))),t.s<0?"-"+i:i}function M(t,e){if(t.length>e)return t.length=e,!0}function _(t){if(!t||"object"!=typeof t)throw Error(c+"Object expected");var e,r,n,o=["precision",1,1e9,"rounding",0,8,"toExpNeg",-1/0,0,"toExpPos",0,1/0];for(e=0;e-1&&t%1==0&&t0&&360>Math.abs(g-x)?d({cx:a,cy:c,innerRadius:s,outerRadius:p,cornerRadius:Math.min(S,j/2),forceCornerRadius:m,cornerIsExternal:b,startAngle:g,endAngle:x}):h({cx:a,cy:c,innerRadius:s,outerRadius:p,startAngle:g,endAngle:x}),n.createElement("path",l({},(0,i.L6)(r,!0),{className:O,d:e,role:"img"}))}},14870:function(t,e,r){"use strict";r.d(e,{v:function(){return N}});var n=r(2265),o=r(75551),i=r.n(o);let a=Math.cos,u=Math.sin,c=Math.sqrt,l=Math.PI,s=2*l;var f={draw(t,e){let r=c(e/l);t.moveTo(r,0),t.arc(0,0,r,0,s)}};let p=c(1/3),h=2*p,d=u(l/10)/u(7*l/10),y=u(s/10)*d,v=-a(s/10)*d,m=c(3),b=c(3)/2,g=1/c(12),x=(g/2+1)*3;var w=r(76115),O=r(67790);c(3),c(3);var j=r(61994),S=r(82944);function P(t){return(P="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}var E=["type","size","sizeType"];function k(){return(k=Object.assign?Object.assign.bind():function(t){for(var e=1;e=s&&f<=l}return r?p(p({},e),{},{radius:o,angle:f+360*Math.min(Math.floor(e.startAngle/360),Math.floor(e.endAngle/360))}):null},j=function(t){return(0,i.isValidElement)(t)||u()(t)||"boolean"==typeof t?"":t.className}},82944:function(t,e,r){"use strict";r.d(e,{$R:function(){return R},Bh:function(){return B},Gf:function(){return j},L6:function(){return N},NN:function(){return k},TT:function(){return M},eu:function(){return L},jf:function(){return T},rL:function(){return D},sP:function(){return A}});var n=r(13735),o=r.n(n),i=r(77571),a=r.n(i),u=r(42715),c=r.n(u),l=r(86757),s=r.n(l),f=r(28302),p=r.n(f),h=r(2265),d=r(14326),y=r(16630),v=r(46485),m=r(41637),b=["children"],g=["children"];function x(t,e){if(null==t)return{};var r,n,o=function(t,e){if(null==t)return{};var r={};for(var n in t)if(Object.prototype.hasOwnProperty.call(t,n)){if(e.indexOf(n)>=0)continue;r[n]=t[n]}return r}(t,e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(t);for(n=0;n