diff --git a/litellm/llms/openai_like/chat/handler.py b/litellm/llms/openai_like/chat/handler.py index bd9635b0861..c9b0fe3a46c 100644 --- a/litellm/llms/openai_like/chat/handler.py +++ b/litellm/llms/openai_like/chat/handler.py @@ -7,6 +7,7 @@ For handling OpenAI-like chat completions, like IBM WatsonX, etc. import json from typing import Any, Callable, Optional, Union +import aiohttp import httpx import litellm @@ -160,7 +161,7 @@ class OpenAILikeChatHandler(OpenAILikeBase): model_response: ModelResponse, custom_llm_provider: str, print_verbose: Callable, - client: Optional[AsyncHTTPHandler], + client: Optional[aiohttp.ClientSession], encoding, api_key, logging_obj, @@ -171,33 +172,32 @@ class OpenAILikeChatHandler(OpenAILikeBase): litellm_params=None, logger_fn=None, headers={}, - timeout: Optional[Union[float, httpx.Timeout]] = None, + timeout: Optional[Union[float, aiohttp.ClientTimeout]] = None, json_mode: bool = False, ) -> ModelResponse: if timeout is None: - timeout = httpx.Timeout(timeout=600.0, connect=5.0) + timeout = aiohttp.ClientTimeout(total=600.0, connect=5.0) if client is None: - client = litellm.module_level_aclient + client = aiohttp.ClientSession(timeout=timeout) try: - response = await client.post( - api_base, headers=headers, data=json.dumps(data), timeout=timeout - ) - response.raise_for_status() - except httpx.HTTPStatusError as e: + async with client.post( + api_base, headers=headers, data=json.dumps(data) + ) as response: + response.raise_for_status() + response_data = await response.json() + except aiohttp.ClientResponseError as e: raise OpenAILikeError( - status_code=e.response.status_code, - message=e.response.text, + status_code=e.status, + message=e.message, ) - except httpx.TimeoutException: - raise OpenAILikeError(status_code=408, message="Timeout error occurred.") except Exception as e: raise OpenAILikeError(status_code=500, message=str(e)) return OpenAILikeChatConfig._transform_response( model=model, - response=response, + response=response_data, model_response=model_response, stream=stream, logging_obj=logging_obj, @@ -331,9 +331,9 @@ class OpenAILikeChatHandler(OpenAILikeBase): litellm_params=litellm_params, logger_fn=logger_fn, headers=headers, - timeout=timeout, + timeout=timeout, # type: ignore base_model=base_model, - client=client, + client=client, # type: ignore ) else: ## COMPLETION CALL diff --git a/litellm/main.py b/litellm/main.py index 846eb9f2dde..83a0c83d35d 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -474,6 +474,7 @@ async def acompletion( or custom_llm_provider == "clarifai" or custom_llm_provider == "watsonx" or custom_llm_provider == "cloudflare" + or custom_llm_provider == "openai_like" or custom_llm_provider in litellm.openai_compatible_providers or custom_llm_provider in litellm._custom_providers ): # currently implemented aiohttp calls for just azure, openai, hf, ollama, vertex ai soon all. @@ -2822,6 +2823,18 @@ def completion( # type: ignore # noqa: PLR0915 ) return response response = model_response + elif custom_llm_provider == "openai_like": + response = openai_like_chat.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + ) elif custom_llm_provider == "petals" or model in litellm.petals_models: api_base = api_base or litellm.api_base diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 6c375752dd7..978c02dcba0 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,33 +1,6 @@ model_list: - - model_name: "openai/*" + - model_name: "openai_like/*" litellm_params: - model: "openai/*" - api_key: os.environ/OPENAI_API_KEY - - model_name: "azure/*" - litellm_params: - model: azure/chatgpt-v-2 - api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ - client_id: os.environ/AZURE_CLIENT_ID - azure_username: os.environ/AZURE_USERNAME - azure_password: os.environ/AZURE_PASSWORD -litellm_settings: - callbacks: ["datadog"] - - -general_settings: - alerting: ["pagerduty"] - alerting_args: - failure_threshold: 4 # Number of requests failing in a window - failure_threshold_window_seconds: 10 # Window in seconds - - # Requests hanging threshold - hanging_threshold_seconds: 0.0000001 # Number of seconds of waiting for a response before a request is considered hanging - hanging_threshold_window_seconds: 10 # Window in seconds - key_management_system: "hashicorp_vault" - -# For /fine_tuning/jobs endpoints -finetune_settings: - - custom_llm_provider: "vertex_ai" - vertex_project: "adroit-crow-413218" - vertex_location: "us-central1" - vertex_credentials: "/Users/ishaanjaffer/Downloads/adroit-crow-413218-a956eef1a2a8.json" + model: openai_like/* + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_key: "ishaan"