refactor: share Replicate HTTP client factories and declare PagerDuty type alias (#51594)

* refactor: deduplicate Replicate clients and declare PagerDuty type alias

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test: cover shared Replicate HTTP client factories

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test: exercise real Replicate HTTP client factories

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Mateo Wang 2026-11-16 18:32:54 -08:00 • committed by GitHub
parent 7f7bb13707
commit a41f124bdc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 64 additions and 30 deletions

View file

@ -13,7 +13,7 @@ However, this is under the enterprise license
import asyncio
import os
from datetime import datetime, timedelta, timezone
from typing import Final, Literal
from typing import Final, Literal, TypeAlias
from litellm._logging import verbose_logger
from litellm.caching import DualCache
@ -48,7 +48,7 @@ def _metadata_of(payload: StandardLoggingPayload) -> StandardLoggingMetadata:
return payload["metadata"]
_CustomerIdKey = Literal[
_CustomerIdKey: TypeAlias = Literal[
"user_api_key_customer_member_id", "user_api_key_customer_team_id", "user_api_key_customer_org_id"
]

View file

@ -24,8 +24,15 @@ from litellm.litellm_core_utils.url_utils import (
safe_get,
)
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, HTTPResponseLimitError
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
HTTPResponseLimitError,
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory lacks typed params
get_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory lacks typed params
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import LlmProviders
REPLICATE_DEFAULT_API_BASE: Final[str] = "https://api.replicate.com/v1"
REPLICATE_TERMINAL_STATUSES: Final[frozenset[str]] = frozenset({"succeeded", "failed", "canceled"})
@ -51,6 +58,14 @@ class ReplicatePredictionRequest:
body: Mapping[str, object]
def replicate_httpx_client(ssl_verify: object) -> HTTPHandler:
return get_httpx_client(params={"ssl_verify": ssl_verify})
def replicate_async_httpx_client(ssl_verify: object) -> AsyncHTTPHandler:
return get_async_httpx_client(llm_provider=LlmProviders.REPLICATE, params={"ssl_verify": ssl_verify})
def replicate_api_base(api_base: str | None) -> str:
return ensure_api_route(api_base or get_secret_str("REPLICATE_API_BASE") or REPLICATE_DEFAULT_API_BASE, "v1")

View file

@ -12,16 +12,16 @@ from litellm.llms.base_llm.image_generation.transformation import BaseImageGener
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory lacks typed params
get_httpx_client, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # shared HTTP factory is private
)
from litellm.llms.replicate.common_utils import (
ReplicateError,
async_replicate_download,
async_wait_for_replicate_prediction,
replicate_api_base,
replicate_async_httpx_client,
replicate_auth_headers,
replicate_download,
replicate_httpx_client,
replicate_output_urls,
replicate_positive_int,
replicate_prediction_failure,
@ -30,7 +30,7 @@ from litellm.llms.replicate.common_utils import (
wait_for_replicate_prediction,
)
from litellm.types.llms.openai import AllMessageValues, OpenAIImageGenerationOptionalParams
from litellm.types.utils import ImageObject, ImageResponse, LlmProviders
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -53,19 +53,11 @@ def _aspect_ratio(size: object) -> Mapping[str, str]:
_NATIVE_INPUT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
def _get_replicate_httpx_client(ssl_verify: object) -> HTTPHandler:
return get_httpx_client(params={"ssl_verify": ssl_verify})
def _get_replicate_async_httpx_client(ssl_verify: object) -> AsyncHTTPHandler:
return get_async_httpx_client(llm_provider=LlmProviders.REPLICATE, params={"ssl_verify": ssl_verify})
class ReplicateImageGenerationConfig(BaseImageGenerationConfig):
def __init__(
self,
sync_client_factory: Callable[[object], HTTPHandler] = _get_replicate_httpx_client,
async_client_factory: Callable[[object], AsyncHTTPHandler] = _get_replicate_async_httpx_client,
sync_client_factory: Callable[[object], HTTPHandler] = replicate_httpx_client,
async_client_factory: Callable[[object], AsyncHTTPHandler] = replicate_async_httpx_client,
) -> None:
self._sync_client_factory: Final = sync_client_factory
self._async_client_factory: Final = async_client_factory

View file

@ -10,16 +10,16 @@ from litellm.llms.base_llm.text_to_speech.transformation import BaseTextToSpeech
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory lacks typed params
get_httpx_client, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # shared HTTP factory is private
)
from litellm.llms.replicate.common_utils import (
ReplicateError,
async_replicate_download,
async_wait_for_replicate_prediction,
replicate_api_base,
replicate_async_httpx_client,
replicate_auth_headers,
replicate_download,
replicate_httpx_client,
replicate_model_owner,
replicate_output_urls,
replicate_positive_int,
@ -28,7 +28,7 @@ from litellm.llms.replicate.common_utils import (
replicate_prediction_request,
wait_for_replicate_prediction,
)
from litellm.types.utils import LlmProviders, Usage
from litellm.types.utils import Usage
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -46,14 +46,6 @@ def _voice_key(model: str) -> str:
return _VOICE_KEY_BY_OWNER.get(replicate_model_owner(model), "voice")
def _get_replicate_httpx_client(ssl_verify: object) -> HTTPHandler:
return get_httpx_client(params={"ssl_verify": ssl_verify})
def _get_replicate_async_httpx_client(ssl_verify: object) -> AsyncHTTPHandler:
return get_async_httpx_client(llm_provider=LlmProviders.REPLICATE, params={"ssl_verify": ssl_verify})
def _ssl_verify(logging_obj: "LiteLLMLoggingObj") -> object:
litellm_params: Final = getattr(logging_obj, "litellm_params", None)
return litellm_params.get("ssl_verify") if isinstance(litellm_params, Mapping) else None
@ -62,8 +54,8 @@ def _ssl_verify(logging_obj: "LiteLLMLoggingObj") -> object:
class ReplicateTextToSpeechConfig(BaseTextToSpeechConfig):
def __init__(
self,
sync_client_factory: Callable[[object], HTTPHandler] = _get_replicate_httpx_client,
async_client_factory: Callable[[object], AsyncHTTPHandler] = _get_replicate_async_httpx_client,
sync_client_factory: Callable[[object], HTTPHandler] = replicate_httpx_client,
async_client_factory: Callable[[object], AsyncHTTPHandler] = replicate_async_httpx_client,
) -> None:
super().__init__()
self._sync_client_factory: Final = sync_client_factory

View file

@ -1,4 +1,5 @@
import base64
import ssl
import time
from typing import Final
from unittest.mock import Mock
@ -7,7 +8,8 @@ import httpx
import pytest
import litellm
from litellm.llms.custom_httpx.http_handler import HTTPResponseLimitError
from litellm.caching.llm_caching_handler import LLMClientCache
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, HTTPResponseLimitError
from litellm.constants import REPLICATE_MAX_OUTPUT_DOWNLOAD_BYTES
from litellm.litellm_core_utils.url_utils import SSRFError
from litellm.llms.replicate import common_utils
@ -22,6 +24,39 @@ PREDICTIONS_URL: Final = f"{API_BASE}/models/{MODEL}/predictions"
IMAGE_URLS: Final = ("https://replicate.delivery/out/0.webp", "https://replicate.delivery/out/1.webp")
@pytest.mark.parametrize("ssl_verify", [None, False, True])
def test_replicate_httpx_client_configures_certificate_verification(
ssl_verify: bool | None, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
monkeypatch.setenv("SSL_VERIFY", "True")
result: Final = common_utils.replicate_httpx_client(ssl_verify)
try:
assert isinstance(result, HTTPHandler)
context: Final = result.client._transport._pool._ssl_context
assert context.verify_mode == (ssl.CERT_NONE if ssl_verify is False else ssl.CERT_REQUIRED)
assert context.check_hostname is (ssl_verify is not False)
finally:
result.close()
@pytest.mark.parametrize("ssl_verify", [None, False, True])
async def test_replicate_async_httpx_client_configures_certificate_verification(
ssl_verify: bool | None, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setenv("SSL_VERIFY", "True")
result: Final = common_utils.replicate_async_httpx_client(ssl_verify)
try:
assert isinstance(result, AsyncHTTPHandler)
context: Final = result.client._transport._pool._ssl_context
assert context.verify_mode == (ssl.CERT_NONE if ssl_verify is False else ssl.CERT_REQUIRED)
assert context.check_hostname is (ssl_verify is not False)
finally:
await result.close()
@pytest.fixture(autouse=True)
def _skip_url_dns_validation(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "user_url_validation", False)