From 765e020e75e66bb508df4198899a673fccff27ad Mon Sep 17 00:00:00 2001 From: Ognjen Francuski Date: Tue, 20 Aug 2024 10:16:03 +0200 Subject: [PATCH 1/4] Update handling of 'litellm.ssl_verify' in HTTP handlers to allow for custom, self-signed certificates. --- litellm/__init__.py | 2 +- litellm/llms/custom_httpx/http_handler.py | 30 +++++++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 6c7529477f5..80e76c2efa1 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -111,7 +111,7 @@ common_cloud_provider_auth_params: dict = { "providers": ["vertex_ai", "bedrock", "watsonx", "azure", "vertex_ai_beta"], } use_client: bool = False -ssl_verify: bool = True +ssl_verify: Union[str, bool] = True ssl_certificate: Optional[str] = None disable_streaming_logging: bool = False in_memory_llm_clients_cache: dict = {} diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 1828a92d2eb..1b3ad5e35e1 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -35,11 +35,18 @@ class AsyncHTTPHandler: self, timeout: Optional[Union[float, httpx.Timeout]], concurrent_limit: int ) -> httpx.AsyncClient: - # Check if the HTTP_PROXY and HTTPS_PROXY environment variables are set and use them accordingly. - ssl_verify = bool(os.getenv("SSL_VERIFY", litellm.ssl_verify)) + # SSL certificates (a.k.a CA bundle) used to verify the identity of requested hosts. + # /path/to/certificate.pem + ssl_verify = os.getenv( + "SSL_VERIFY", + litellm.ssl_verify + ) + # An SSL certificate used by the requested host to authenticate the client. + # /path/to/client.pem cert = os.getenv( - "SSL_CERTIFICATE", litellm.ssl_certificate - ) # /path/to/client.pem + "SSL_CERTIFICATE", + litellm.ssl_certificate + ) if timeout is None: timeout = _DEFAULT_TIMEOUT @@ -212,11 +219,18 @@ class HTTPHandler: if timeout is None: timeout = _DEFAULT_TIMEOUT - # Check if the HTTP_PROXY and HTTPS_PROXY environment variables are set and use them accordingly. - ssl_verify = bool(os.getenv("SSL_VERIFY", litellm.ssl_verify)) + # SSL certificates (a.k.a CA bundle) used to verify the identity of requested hosts. + # /path/to/certificate.pem + ssl_verify = os.getenv( + "SSL_VERIFY", + litellm.ssl_verify + ) + # An SSL certificate used by the requested host to authenticate the client. + # /path/to/client.pem cert = os.getenv( - "SSL_CERTIFICATE", litellm.ssl_certificate - ) # /path/to/client.pem + "SSL_CERTIFICATE", + litellm.ssl_certificate + ) if client is None: # Create a client with a connection pool From 9492277feaa21be042c9be52bd684d6b1a35c403 Mon Sep 17 00:00:00 2001 From: Ognjen Francuski Date: Tue, 20 Aug 2024 10:29:37 +0200 Subject: [PATCH 2/4] Fix using sync 'litellm.client_session' for async calls in azure.py --- litellm/llms/azure.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/litellm/llms/azure.py b/litellm/llms/azure.py index 876d3b899a7..cd99f3f75dd 100644 --- a/litellm/llms/azure.py +++ b/litellm/llms/azure.py @@ -692,7 +692,7 @@ class AzureChatCompletion(BaseLLM): "api_version": api_version, "azure_endpoint": api_base, "azure_deployment": model, - "http_client": litellm.client_session, + "http_client": litellm.aclient_session, "max_retries": max_retries, "timeout": timeout, } @@ -860,7 +860,7 @@ class AzureChatCompletion(BaseLLM): "api_version": api_version, "azure_endpoint": api_base, "azure_deployment": model, - "http_client": litellm.client_session, + "http_client": litellm.aclient_session, "max_retries": data.pop("max_retries", 2), "timeout": timeout, } @@ -989,13 +989,16 @@ class AzureChatCompletion(BaseLLM): "api_version": api_version, "azure_endpoint": api_base, "azure_deployment": model, - "http_client": litellm.client_session, "max_retries": max_retries, "timeout": timeout, } azure_client_params = select_azure_base_url_or_endpoint( azure_client_params=azure_client_params ) + if aembedding: + azure_client_params["http_client"] = litellm.aclient_session + else: + azure_client_params["http_client"] = litellm.client_session if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: From 924dfe422627a1914c73a0d90d8498310536e8ca Mon Sep 17 00:00:00 2001 From: Ognjen Francuski Date: Tue, 20 Aug 2024 14:18:30 +0200 Subject: [PATCH 3/4] Update 'init_bedrock_client' to use 'litellm.ssl_verify' or 'SSL_VERIFY' environment variable. --- litellm/llms/bedrock.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/litellm/llms/bedrock.py b/litellm/llms/bedrock.py index 2185ec459e3..916826ce313 100644 --- a/litellm/llms/bedrock.py +++ b/litellm/llms/bedrock.py @@ -605,6 +605,9 @@ def init_bedrock_client( aws_web_identity_token, ) = params_to_check + # SSL certificates (a.k.a CA bundle) used to verify the identity of requested hosts. + ssl_verify = os.getenv("SSL_VERIFY", litellm.ssl_verify) + ### SET REGION NAME if region_name: pass @@ -673,6 +676,7 @@ def init_bedrock_client( region_name=region_name, endpoint_url=endpoint_url, config=config, + verify=ssl_verify, ) elif aws_role_name is not None and aws_session_name is not None: # use sts if role name passed in @@ -694,6 +698,7 @@ def init_bedrock_client( region_name=region_name, endpoint_url=endpoint_url, config=config, + verify=ssl_verify, ) elif aws_access_key_id is not None: # uses auth params passed to completion @@ -706,6 +711,7 @@ def init_bedrock_client( region_name=region_name, endpoint_url=endpoint_url, config=config, + verify=ssl_verify, ) elif aws_profile_name is not None: # uses auth values from AWS profile usually stored in ~/.aws/credentials @@ -715,6 +721,7 @@ def init_bedrock_client( region_name=region_name, endpoint_url=endpoint_url, config=config, + verify=ssl_verify, ) else: # aws_access_key_id is None, assume user is trying to auth using env variables @@ -725,6 +732,7 @@ def init_bedrock_client( region_name=region_name, endpoint_url=endpoint_url, config=config, + verify=ssl_verify, ) if extra_headers: client.meta.events.register( From 31aac9a1e47ad692ab4183168a1f44a025fdaf48 Mon Sep 17 00:00:00 2001 From: Ognjen Francuski Date: Tue, 20 Aug 2024 14:55:12 +0200 Subject: [PATCH 4/4] Update Huggingface provider to utilize the SSL verification through 'SSL_VERIFY' env var or 'litellm.ssl_verify'. --- litellm/llms/huggingface_restapi.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/litellm/llms/huggingface_restapi.py b/litellm/llms/huggingface_restapi.py index 06ef0e6fcd3..426afd22713 100644 --- a/litellm/llms/huggingface_restapi.py +++ b/litellm/llms/huggingface_restapi.py @@ -613,6 +613,10 @@ class Huggingface(BaseLLM): }, ) ## COMPLETION CALL + + # SSL certificates (a.k.a CA bundle) used to verify the identity of requested hosts. + ssl_verify = os.getenv("SSL_VERIFY", litellm.ssl_verify) + if acompletion is True: ### ASYNC STREAMING if optional_params.get("stream", False): @@ -627,12 +631,16 @@ class Huggingface(BaseLLM): headers=headers, data=json.dumps(data), stream=optional_params["stream"], + verify=ssl_verify ) return response.iter_lines() ### SYNC COMPLETION else: response = requests.post( - completion_url, headers=headers, data=json.dumps(data) + completion_url, + headers=headers, + data=json.dumps(data), + verify=ssl_verify ) ## Some servers might return streaming responses even though stream was not set to true. (e.g. Baseten) @@ -728,9 +736,12 @@ class Huggingface(BaseLLM): optional_params: dict, timeout: float, ): + # SSL certificates (a.k.a CA bundle) used to verify the identity of requested hosts. + ssl_verify = os.getenv("SSL_VERIFY", litellm.ssl_verify) + response = None try: - async with httpx.AsyncClient(timeout=timeout) as client: + async with httpx.AsyncClient(timeout=timeout, verify=ssl_verify) as client: response = await client.post(url=api_base, json=data, headers=headers) response_json = response.json() if response.status_code != 200: @@ -782,7 +793,10 @@ class Huggingface(BaseLLM): model: str, timeout: float, ): - async with httpx.AsyncClient(timeout=timeout) as client: + # SSL certificates (a.k.a CA bundle) used to verify the identity of requested hosts. + ssl_verify = os.getenv("SSL_VERIFY", litellm.ssl_verify) + + async with httpx.AsyncClient(timeout=timeout, verify=ssl_verify) as client: response = client.stream( "POST", url=f"{api_base}", json=data, headers=headers )