From a41f124bdc93c3f67e4fb64f7876c8a5708a32db Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 16 Nov 2026 18:32:54 -0800 Subject: [PATCH] refactor: share Replicate HTTP client factories and declare PagerDuty type alias (#51594) * refactor: deduplicate Replicate clients and declare PagerDuty type alias Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: cover shared Replicate HTTP client factories Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: exercise real Replicate HTTP client factories Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../pagerduty/pagerduty.py | 4 +- litellm/llms/replicate/common_utils.py | 17 ++++++++- .../image_generation/transformation.py | 18 +++------ .../text_to_speech/transformation.py | 18 +++------ ...plicate_image_generation_transformation.py | 37 ++++++++++++++++++- 5 files changed, 64 insertions(+), 30 deletions(-) diff --git a/litellm/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py b/litellm/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py index 589e3e27c47..9920180b763 100644 --- a/litellm/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py +++ b/litellm/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py @@ -13,7 +13,7 @@ However, this is under the enterprise license import asyncio import os from datetime import datetime, timedelta, timezone -from typing import Final, Literal +from typing import Final, Literal, TypeAlias from litellm._logging import verbose_logger from litellm.caching import DualCache @@ -48,7 +48,7 @@ def _metadata_of(payload: StandardLoggingPayload) -> StandardLoggingMetadata: return payload["metadata"] -_CustomerIdKey = Literal[ +_CustomerIdKey: TypeAlias = Literal[ "user_api_key_customer_member_id", "user_api_key_customer_team_id", "user_api_key_customer_org_id" ] diff --git a/litellm/llms/replicate/common_utils.py b/litellm/llms/replicate/common_utils.py index 4626babe14c..29f74f96758 100644 --- a/litellm/llms/replicate/common_utils.py +++ b/litellm/llms/replicate/common_utils.py @@ -24,8 +24,15 @@ from litellm.litellm_core_utils.url_utils import ( safe_get, ) from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, HTTPResponseLimitError +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + HTTPResponseLimitError, + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory lacks typed params + get_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory lacks typed params +) from litellm.secret_managers.main import get_secret_str +from litellm.types.utils import LlmProviders REPLICATE_DEFAULT_API_BASE: Final[str] = "https://api.replicate.com/v1" REPLICATE_TERMINAL_STATUSES: Final[frozenset[str]] = frozenset({"succeeded", "failed", "canceled"}) @@ -51,6 +58,14 @@ class ReplicatePredictionRequest: body: Mapping[str, object] +def replicate_httpx_client(ssl_verify: object) -> HTTPHandler: + return get_httpx_client(params={"ssl_verify": ssl_verify}) + + +def replicate_async_httpx_client(ssl_verify: object) -> AsyncHTTPHandler: + return get_async_httpx_client(llm_provider=LlmProviders.REPLICATE, params={"ssl_verify": ssl_verify}) + + def replicate_api_base(api_base: str | None) -> str: return ensure_api_route(api_base or get_secret_str("REPLICATE_API_BASE") or REPLICATE_DEFAULT_API_BASE, "v1") diff --git a/litellm/llms/replicate/image_generation/transformation.py b/litellm/llms/replicate/image_generation/transformation.py index b80a5f6811d..afa433dca61 100644 --- a/litellm/llms/replicate/image_generation/transformation.py +++ b/litellm/llms/replicate/image_generation/transformation.py @@ -12,16 +12,16 @@ from litellm.llms.base_llm.image_generation.transformation import BaseImageGener from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, - get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory lacks typed params - get_httpx_client, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # shared HTTP factory is private ) from litellm.llms.replicate.common_utils import ( ReplicateError, async_replicate_download, async_wait_for_replicate_prediction, replicate_api_base, + replicate_async_httpx_client, replicate_auth_headers, replicate_download, + replicate_httpx_client, replicate_output_urls, replicate_positive_int, replicate_prediction_failure, @@ -30,7 +30,7 @@ from litellm.llms.replicate.common_utils import ( wait_for_replicate_prediction, ) from litellm.types.llms.openai import AllMessageValues, OpenAIImageGenerationOptionalParams -from litellm.types.utils import ImageObject, ImageResponse, LlmProviders +from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -53,19 +53,11 @@ def _aspect_ratio(size: object) -> Mapping[str, str]: _NATIVE_INPUT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) -def _get_replicate_httpx_client(ssl_verify: object) -> HTTPHandler: - return get_httpx_client(params={"ssl_verify": ssl_verify}) - - -def _get_replicate_async_httpx_client(ssl_verify: object) -> AsyncHTTPHandler: - return get_async_httpx_client(llm_provider=LlmProviders.REPLICATE, params={"ssl_verify": ssl_verify}) - - class ReplicateImageGenerationConfig(BaseImageGenerationConfig): def __init__( self, - sync_client_factory: Callable[[object], HTTPHandler] = _get_replicate_httpx_client, - async_client_factory: Callable[[object], AsyncHTTPHandler] = _get_replicate_async_httpx_client, + sync_client_factory: Callable[[object], HTTPHandler] = replicate_httpx_client, + async_client_factory: Callable[[object], AsyncHTTPHandler] = replicate_async_httpx_client, ) -> None: self._sync_client_factory: Final = sync_client_factory self._async_client_factory: Final = async_client_factory diff --git a/litellm/llms/replicate/text_to_speech/transformation.py b/litellm/llms/replicate/text_to_speech/transformation.py index d75fc29ba54..c2a1ee41f8c 100644 --- a/litellm/llms/replicate/text_to_speech/transformation.py +++ b/litellm/llms/replicate/text_to_speech/transformation.py @@ -10,16 +10,16 @@ from litellm.llms.base_llm.text_to_speech.transformation import BaseTextToSpeech from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, - get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory lacks typed params - get_httpx_client, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # shared HTTP factory is private ) from litellm.llms.replicate.common_utils import ( ReplicateError, async_replicate_download, async_wait_for_replicate_prediction, replicate_api_base, + replicate_async_httpx_client, replicate_auth_headers, replicate_download, + replicate_httpx_client, replicate_model_owner, replicate_output_urls, replicate_positive_int, @@ -28,7 +28,7 @@ from litellm.llms.replicate.common_utils import ( replicate_prediction_request, wait_for_replicate_prediction, ) -from litellm.types.utils import LlmProviders, Usage +from litellm.types.utils import Usage if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -46,14 +46,6 @@ def _voice_key(model: str) -> str: return _VOICE_KEY_BY_OWNER.get(replicate_model_owner(model), "voice") -def _get_replicate_httpx_client(ssl_verify: object) -> HTTPHandler: - return get_httpx_client(params={"ssl_verify": ssl_verify}) - - -def _get_replicate_async_httpx_client(ssl_verify: object) -> AsyncHTTPHandler: - return get_async_httpx_client(llm_provider=LlmProviders.REPLICATE, params={"ssl_verify": ssl_verify}) - - def _ssl_verify(logging_obj: "LiteLLMLoggingObj") -> object: litellm_params: Final = getattr(logging_obj, "litellm_params", None) return litellm_params.get("ssl_verify") if isinstance(litellm_params, Mapping) else None @@ -62,8 +54,8 @@ def _ssl_verify(logging_obj: "LiteLLMLoggingObj") -> object: class ReplicateTextToSpeechConfig(BaseTextToSpeechConfig): def __init__( self, - sync_client_factory: Callable[[object], HTTPHandler] = _get_replicate_httpx_client, - async_client_factory: Callable[[object], AsyncHTTPHandler] = _get_replicate_async_httpx_client, + sync_client_factory: Callable[[object], HTTPHandler] = replicate_httpx_client, + async_client_factory: Callable[[object], AsyncHTTPHandler] = replicate_async_httpx_client, ) -> None: super().__init__() self._sync_client_factory: Final = sync_client_factory diff --git a/tests/unit/llms/replicate/image_generation/test_replicate_image_generation_transformation.py b/tests/unit/llms/replicate/image_generation/test_replicate_image_generation_transformation.py index 6ae0d364b6a..d513a3ecd0e 100644 --- a/tests/unit/llms/replicate/image_generation/test_replicate_image_generation_transformation.py +++ b/tests/unit/llms/replicate/image_generation/test_replicate_image_generation_transformation.py @@ -1,4 +1,5 @@ import base64 +import ssl import time from typing import Final from unittest.mock import Mock @@ -7,7 +8,8 @@ import httpx import pytest import litellm -from litellm.llms.custom_httpx.http_handler import HTTPResponseLimitError +from litellm.caching.llm_caching_handler import LLMClientCache +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, HTTPResponseLimitError from litellm.constants import REPLICATE_MAX_OUTPUT_DOWNLOAD_BYTES from litellm.litellm_core_utils.url_utils import SSRFError from litellm.llms.replicate import common_utils @@ -22,6 +24,39 @@ PREDICTIONS_URL: Final = f"{API_BASE}/models/{MODEL}/predictions" IMAGE_URLS: Final = ("https://replicate.delivery/out/0.webp", "https://replicate.delivery/out/1.webp") +@pytest.mark.parametrize("ssl_verify", [None, False, True]) +def test_replicate_httpx_client_configures_certificate_verification( + ssl_verify: bool | None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + monkeypatch.setenv("SSL_VERIFY", "True") + result: Final = common_utils.replicate_httpx_client(ssl_verify) + try: + assert isinstance(result, HTTPHandler) + context: Final = result.client._transport._pool._ssl_context + assert context.verify_mode == (ssl.CERT_NONE if ssl_verify is False else ssl.CERT_REQUIRED) + assert context.check_hostname is (ssl_verify is not False) + finally: + result.close() + + +@pytest.mark.parametrize("ssl_verify", [None, False, True]) +async def test_replicate_async_httpx_client_configures_certificate_verification( + ssl_verify: bool | None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("SSL_VERIFY", "True") + result: Final = common_utils.replicate_async_httpx_client(ssl_verify) + try: + assert isinstance(result, AsyncHTTPHandler) + context: Final = result.client._transport._pool._ssl_context + assert context.verify_mode == (ssl.CERT_NONE if ssl_verify is False else ssl.CERT_REQUIRED) + assert context.check_hostname is (ssl_verify is not False) + finally: + await result.close() + + @pytest.fixture(autouse=True) def _skip_url_dns_validation(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "user_url_validation", False)