mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
refactor: share Replicate HTTP client factories and declare PagerDuty type alias (#51594)
* refactor: deduplicate Replicate clients and declare PagerDuty type alias Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: cover shared Replicate HTTP client factories Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: exercise real Replicate HTTP client factories Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
7f7bb13707
commit
a41f124bdc
5 changed files with 64 additions and 30 deletions
|
|
@ -13,7 +13,7 @@ However, this is under the enterprise license
|
|||
import asyncio
|
||||
import os
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Final, Literal
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.caching import DualCache
|
||||
|
|
@ -48,7 +48,7 @@ def _metadata_of(payload: StandardLoggingPayload) -> StandardLoggingMetadata:
|
|||
return payload["metadata"]
|
||||
|
||||
|
||||
_CustomerIdKey = Literal[
|
||||
_CustomerIdKey: TypeAlias = Literal[
|
||||
"user_api_key_customer_member_id", "user_api_key_customer_team_id", "user_api_key_customer_org_id"
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -24,8 +24,15 @@ from litellm.litellm_core_utils.url_utils import (
|
|||
safe_get,
|
||||
)
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, HTTPResponseLimitError
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
HTTPResponseLimitError,
|
||||
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory lacks typed params
|
||||
get_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory lacks typed params
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
REPLICATE_DEFAULT_API_BASE: Final[str] = "https://api.replicate.com/v1"
|
||||
REPLICATE_TERMINAL_STATUSES: Final[frozenset[str]] = frozenset({"succeeded", "failed", "canceled"})
|
||||
|
|
@ -51,6 +58,14 @@ class ReplicatePredictionRequest:
|
|||
body: Mapping[str, object]
|
||||
|
||||
|
||||
def replicate_httpx_client(ssl_verify: object) -> HTTPHandler:
|
||||
return get_httpx_client(params={"ssl_verify": ssl_verify})
|
||||
|
||||
|
||||
def replicate_async_httpx_client(ssl_verify: object) -> AsyncHTTPHandler:
|
||||
return get_async_httpx_client(llm_provider=LlmProviders.REPLICATE, params={"ssl_verify": ssl_verify})
|
||||
|
||||
|
||||
def replicate_api_base(api_base: str | None) -> str:
|
||||
return ensure_api_route(api_base or get_secret_str("REPLICATE_API_BASE") or REPLICATE_DEFAULT_API_BASE, "v1")
|
||||
|
||||
|
|
|
|||
|
|
@ -12,16 +12,16 @@ from litellm.llms.base_llm.image_generation.transformation import BaseImageGener
|
|||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory lacks typed params
|
||||
get_httpx_client, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # shared HTTP factory is private
|
||||
)
|
||||
from litellm.llms.replicate.common_utils import (
|
||||
ReplicateError,
|
||||
async_replicate_download,
|
||||
async_wait_for_replicate_prediction,
|
||||
replicate_api_base,
|
||||
replicate_async_httpx_client,
|
||||
replicate_auth_headers,
|
||||
replicate_download,
|
||||
replicate_httpx_client,
|
||||
replicate_output_urls,
|
||||
replicate_positive_int,
|
||||
replicate_prediction_failure,
|
||||
|
|
@ -30,7 +30,7 @@ from litellm.llms.replicate.common_utils import (
|
|||
wait_for_replicate_prediction,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues, OpenAIImageGenerationOptionalParams
|
||||
from litellm.types.utils import ImageObject, ImageResponse, LlmProviders
|
||||
from litellm.types.utils import ImageObject, ImageResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -53,19 +53,11 @@ def _aspect_ratio(size: object) -> Mapping[str, str]:
|
|||
_NATIVE_INPUT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
|
||||
|
||||
def _get_replicate_httpx_client(ssl_verify: object) -> HTTPHandler:
|
||||
return get_httpx_client(params={"ssl_verify": ssl_verify})
|
||||
|
||||
|
||||
def _get_replicate_async_httpx_client(ssl_verify: object) -> AsyncHTTPHandler:
|
||||
return get_async_httpx_client(llm_provider=LlmProviders.REPLICATE, params={"ssl_verify": ssl_verify})
|
||||
|
||||
|
||||
class ReplicateImageGenerationConfig(BaseImageGenerationConfig):
|
||||
def __init__(
|
||||
self,
|
||||
sync_client_factory: Callable[[object], HTTPHandler] = _get_replicate_httpx_client,
|
||||
async_client_factory: Callable[[object], AsyncHTTPHandler] = _get_replicate_async_httpx_client,
|
||||
sync_client_factory: Callable[[object], HTTPHandler] = replicate_httpx_client,
|
||||
async_client_factory: Callable[[object], AsyncHTTPHandler] = replicate_async_httpx_client,
|
||||
) -> None:
|
||||
self._sync_client_factory: Final = sync_client_factory
|
||||
self._async_client_factory: Final = async_client_factory
|
||||
|
|
|
|||
|
|
@ -10,16 +10,16 @@ from litellm.llms.base_llm.text_to_speech.transformation import BaseTextToSpeech
|
|||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory lacks typed params
|
||||
get_httpx_client, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # shared HTTP factory is private
|
||||
)
|
||||
from litellm.llms.replicate.common_utils import (
|
||||
ReplicateError,
|
||||
async_replicate_download,
|
||||
async_wait_for_replicate_prediction,
|
||||
replicate_api_base,
|
||||
replicate_async_httpx_client,
|
||||
replicate_auth_headers,
|
||||
replicate_download,
|
||||
replicate_httpx_client,
|
||||
replicate_model_owner,
|
||||
replicate_output_urls,
|
||||
replicate_positive_int,
|
||||
|
|
@ -28,7 +28,7 @@ from litellm.llms.replicate.common_utils import (
|
|||
replicate_prediction_request,
|
||||
wait_for_replicate_prediction,
|
||||
)
|
||||
from litellm.types.utils import LlmProviders, Usage
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -46,14 +46,6 @@ def _voice_key(model: str) -> str:
|
|||
return _VOICE_KEY_BY_OWNER.get(replicate_model_owner(model), "voice")
|
||||
|
||||
|
||||
def _get_replicate_httpx_client(ssl_verify: object) -> HTTPHandler:
|
||||
return get_httpx_client(params={"ssl_verify": ssl_verify})
|
||||
|
||||
|
||||
def _get_replicate_async_httpx_client(ssl_verify: object) -> AsyncHTTPHandler:
|
||||
return get_async_httpx_client(llm_provider=LlmProviders.REPLICATE, params={"ssl_verify": ssl_verify})
|
||||
|
||||
|
||||
def _ssl_verify(logging_obj: "LiteLLMLoggingObj") -> object:
|
||||
litellm_params: Final = getattr(logging_obj, "litellm_params", None)
|
||||
return litellm_params.get("ssl_verify") if isinstance(litellm_params, Mapping) else None
|
||||
|
|
@ -62,8 +54,8 @@ def _ssl_verify(logging_obj: "LiteLLMLoggingObj") -> object:
|
|||
class ReplicateTextToSpeechConfig(BaseTextToSpeechConfig):
|
||||
def __init__(
|
||||
self,
|
||||
sync_client_factory: Callable[[object], HTTPHandler] = _get_replicate_httpx_client,
|
||||
async_client_factory: Callable[[object], AsyncHTTPHandler] = _get_replicate_async_httpx_client,
|
||||
sync_client_factory: Callable[[object], HTTPHandler] = replicate_httpx_client,
|
||||
async_client_factory: Callable[[object], AsyncHTTPHandler] = replicate_async_httpx_client,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._sync_client_factory: Final = sync_client_factory
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import base64
|
||||
import ssl
|
||||
import time
|
||||
from typing import Final
|
||||
from unittest.mock import Mock
|
||||
|
|
@ -7,7 +8,8 @@ import httpx
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPResponseLimitError
|
||||
from litellm.caching.llm_caching_handler import LLMClientCache
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, HTTPResponseLimitError
|
||||
from litellm.constants import REPLICATE_MAX_OUTPUT_DOWNLOAD_BYTES
|
||||
from litellm.litellm_core_utils.url_utils import SSRFError
|
||||
from litellm.llms.replicate import common_utils
|
||||
|
|
@ -22,6 +24,39 @@ PREDICTIONS_URL: Final = f"{API_BASE}/models/{MODEL}/predictions"
|
|||
IMAGE_URLS: Final = ("https://replicate.delivery/out/0.webp", "https://replicate.delivery/out/1.webp")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ssl_verify", [None, False, True])
|
||||
def test_replicate_httpx_client_configures_certificate_verification(
|
||||
ssl_verify: bool | None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
monkeypatch.setenv("SSL_VERIFY", "True")
|
||||
result: Final = common_utils.replicate_httpx_client(ssl_verify)
|
||||
try:
|
||||
assert isinstance(result, HTTPHandler)
|
||||
context: Final = result.client._transport._pool._ssl_context
|
||||
assert context.verify_mode == (ssl.CERT_NONE if ssl_verify is False else ssl.CERT_REQUIRED)
|
||||
assert context.check_hostname is (ssl_verify is not False)
|
||||
finally:
|
||||
result.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ssl_verify", [None, False, True])
|
||||
async def test_replicate_async_httpx_client_configures_certificate_verification(
|
||||
ssl_verify: bool | None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setenv("SSL_VERIFY", "True")
|
||||
result: Final = common_utils.replicate_async_httpx_client(ssl_verify)
|
||||
try:
|
||||
assert isinstance(result, AsyncHTTPHandler)
|
||||
context: Final = result.client._transport._pool._ssl_context
|
||||
assert context.verify_mode == (ssl.CERT_NONE if ssl_verify is False else ssl.CERT_REQUIRED)
|
||||
assert context.check_hostname is (ssl_verify is not False)
|
||||
finally:
|
||||
await result.close()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _skip_url_dns_validation(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "user_url_validation", False)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue