diff --git a/litellm/main.py b/litellm/main.py index a96633f72e1..48cdcea629c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -87,6 +87,7 @@ from litellm.llms.base_llm.base_model_iterator import ( convert_model_response_to_streaming, ) from litellm.llms.bedrock.common_utils import BedrockModelInfo +from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.vertex_ai.common_utils import ( VertexAIModelRoute, @@ -157,6 +158,7 @@ from .llms.bedrock.chat import BedrockConverseLLM, BedrockLLM from .llms.bedrock.embed.embedding import BedrockEmbedding from .llms.bedrock.image.image_handler import BedrockImageGeneration from .llms.bytez.chat.transformation import BytezChatConfig +from .llms.clarifai.chat.transformation import ClarifaiConfig from .llms.codestral.completion.handler import CodestralTextCompletion from .llms.cohere.embed import handler as cohere_embed from .llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler @@ -386,6 +388,7 @@ async def acompletion( Literal["none", "minimal", "low", "medium", "high", "default"] ] = None, safety_identifier: Optional[str] = None, + service_tier: Optional[str] = None, # set api_base, api_version, api_key base_url: Optional[str] = None, api_version: Optional[str] = None, @@ -535,6 +538,7 @@ async def acompletion( "model_list": model_list, "reasoning_effort": reasoning_effort, "safety_identifier": safety_identifier, + "service_tier": service_tier, "extra_headers": extra_headers, "acompletion": True, # assuming this is a required parameter "thinking": thinking, @@ -820,10 +824,9 @@ def mock_completion( # convert to ModelResponseStream mock_response = convert_model_response_to_streaming(mock_response) # type: ignore - model_response: Union[ModelResponse, ModelResponseStream] = ModelResponse() + model_response = ModelResponseStream() if stream is True: - model_response = ModelResponseStream() # don't try to access stream object, if kwargs.get("acompletion", False) is True: return CustomStreamWrapper( @@ -965,6 +968,7 @@ def completion( # type: ignore # noqa: PLR0915 deployment_id=None, extra_headers: Optional[dict] = None, safety_identifier: Optional[str] = None, + service_tier: Optional[str] = None, # soon to be deprecated params by OpenAI functions: Optional[List] = None, function_call: Optional[str] = None, @@ -1307,6 +1311,7 @@ def completion( # type: ignore # noqa: PLR0915 "thinking": thinking, "web_search_options": web_search_options, "safety_identifier": safety_identifier, + "service_tier": service_tier, "allowed_openai_params": kwargs.get("allowed_openai_params"), } optional_params = get_optional_params( @@ -2042,6 +2047,7 @@ def completion( # type: ignore # noqa: PLR0915 or custom_llm_provider == "together_ai" or custom_llm_provider == "nebius" or custom_llm_provider == "wandb" + or custom_llm_provider == "clarifai" or custom_llm_provider in litellm.openai_compatible_providers or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo ): # allow user to make an openai call with a custom base @@ -2236,40 +2242,7 @@ def completion( # type: ignore # noqa: PLR0915 or custom_llm_provider == "clarifai" or model in litellm.clarifai_models ): - clarifai_key = None - clarifai_key = ( - api_key - or litellm.clarifai_key - or litellm.api_key - or get_secret("CLARIFAI_API_KEY") - or get_secret("CLARIFAI_API_TOKEN") - ) - - api_base = ( - api_base - or litellm.api_base - or get_secret("CLARIFAI_API_BASE") - or "https://api.clarifai.com/v2" - ) - api_base = litellm.ClarifaiConfig()._convert_model_to_url(model, api_base) - response = base_llm_http_handler.completion( - model=model, - stream=stream, - fake_stream=True, # clarifai does not support streaming, we fake it - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="clarifai", - timeout=timeout, - headers=headers, - encoding=encoding, - api_key=clarifai_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) + pass # Deprecated - handled in the openai compatible provider section above elif custom_llm_provider == "anthropic_text": api_key = ( api_key @@ -2462,7 +2435,7 @@ def completion( # type: ignore # noqa: PLR0915 ) return response response = model_response - elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere": + elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere": cohere_key = ( api_key or litellm.cohere_key @@ -2471,12 +2444,26 @@ def completion( # type: ignore # noqa: PLR0915 or litellm.api_key ) - api_base = ( - api_base - or litellm.api_base - or get_secret_str("COHERE_API_BASE") - or "https://api.cohere.ai/v1/chat" - ) + cohere_route = CohereModelInfo.get_cohere_route(model) + verbose_logger.debug(f"Cohere route: {cohere_route}") + # Set API base based on route + if cohere_route == "v2": + api_base = ( + api_base + or litellm.api_base + or get_secret_str("COHERE_API_BASE") + or "https://api.cohere.com/v2/chat" + ) + # Remove v2/ prefix from model name for the actual API call + if "v2/" in model: + model = model.replace("v2/", "") + else: + api_base = ( + api_base + or litellm.api_base + or get_secret_str("COHERE_API_BASE") + or "https://api.cohere.ai/v1/chat" + ) headers = headers or litellm.headers or {} if headers is None: @@ -2485,6 +2472,8 @@ def completion( # type: ignore # noqa: PLR0915 if extra_headers is not None: headers.update(extra_headers) + verbose_logger.debug(f"Model: {model}, API Base: {api_base}") + verbose_logger.debug(f"Provider Config: {provider_config}") response = base_llm_http_handler.completion( model=model, stream=stream, @@ -2500,6 +2489,7 @@ def completion( # type: ignore # noqa: PLR0915 headers=headers, encoding=encoding, api_key=cohere_key, + provider_config=provider_config, logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements ) elif custom_llm_provider == "maritalk": @@ -4012,6 +4002,7 @@ def embedding( # noqa: PLR0915 """ azure = kwargs.get("azure", None) client = kwargs.pop("client", None) + shared_session = kwargs.get("shared_session", None) max_retries = kwargs.get("max_retries", None) litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore mock_response: Optional[List[float]] = kwargs.get("mock_response", None) # type: ignore @@ -4201,6 +4192,7 @@ def embedding( # noqa: PLR0915 client=client, aembedding=aembedding, max_retries=max_retries, + shared_session=shared_session, ) elif custom_llm_provider == "databricks": api_base = api_base or litellm.api_base or get_secret("DATABRICKS_API_BASE") # type: ignore @@ -4768,6 +4760,33 @@ def embedding( # noqa: PLR0915 aembedding=aembedding, litellm_params={}, ) + elif custom_llm_provider == "cometapi": + api_key = ( + api_key + or litellm.cometapi_key + or get_secret_str("COMETAPI_KEY") + or litellm.api_key + ) + api_base = ( + api_base + or litellm.api_base + or get_secret_str("COMETAPI_API_BASE") + or "https://api.cometapi.com/v1" + ) + response = base_llm_http_handler.embedding( + model=model, + input=input, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + logging_obj=logging, + timeout=timeout, + model_response=EmbeddingResponse(), + optional_params=optional_params, + client=client, + aembedding=aembedding, + litellm_params={}, + ) elif custom_llm_provider in litellm._custom_providers: custom_handler: Optional[CustomLLM] = None for item in litellm.custom_provider_map: @@ -5703,17 +5722,35 @@ def speech( # noqa: PLR0915 optional_params["speed"] = speed # type: ignore if instructions is not None: optional_params["instructions"] = instructions + if timeout is None: timeout = litellm.request_timeout if max_retries is None: max_retries = litellm.num_retries or openai.DEFAULT_MAX_RETRIES litellm_params_dict = get_litellm_params(**kwargs) + + # Get provider-specific text-to-speech config and map parameters + text_to_speech_provider_config = ProviderConfigManager.get_provider_text_to_speech_config( + model=model, + provider=litellm.LlmProviders(custom_llm_provider), + ) + + # Map OpenAI params to provider-specific params if config exists + if text_to_speech_provider_config is not None: + voice, optional_params = text_to_speech_provider_config.map_openai_params( + model=model, + optional_params=optional_params, + voice=voice, + drop_params=False, + kwargs=kwargs, + ) + logging_obj: Logging = cast(Logging, kwargs.get("litellm_logging_obj")) logging_obj.update_environment_variables( model=model, user=user, - optional_params={}, + optional_params=optional_params, litellm_params={ "litellm_call_id": litellm_call_id, "proxy_server_request": proxy_server_request, @@ -5782,52 +5819,85 @@ def speech( # noqa: PLR0915 aspeech=aspeech, ) elif custom_llm_provider == "azure": - # azure configs - if voice is None or not (isinstance(voice, str)): - raise litellm.BadRequestError( - message="'voice' is required to be passed as a string for Azure TTS", - model=model, - llm_provider=custom_llm_provider, + # Check if this is Azure Speech Service (Cognitive Services TTS) + if model.startswith("speech/"): + from litellm.llms.azure.text_to_speech.transformation import ( + AzureAVATextToSpeechConfig, ) - api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore - api_version = api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore + # Azure AVA (Cognitive Services) Text-to-Speech + if text_to_speech_provider_config is None: + raise litellm.BadRequestError( + message="Azure Speech Service configuration not found", + model=model, + llm_provider=custom_llm_provider, + ) - api_key = ( - api_key - or litellm.api_key - or litellm.azure_key - or get_secret("AZURE_OPENAI_API_KEY") - or get_secret("AZURE_API_KEY") - ) # type: ignore + # Cast to specific Azure config type to access dispatch method + azure_config = cast(AzureAVATextToSpeechConfig, text_to_speech_provider_config) + + response = azure_config.dispatch_text_to_speech( # type: ignore + model=model, + input=input, + voice=voice, + optional_params=optional_params, + litellm_params_dict=litellm_params_dict, + logging_obj=logging_obj, + timeout=timeout, + extra_headers=extra_headers, + base_llm_http_handler=base_llm_http_handler, + aspeech=aspeech or False, + api_base=api_base, + api_key=api_key, + **kwargs, + ) + else: + # Azure OpenAI TTS + if voice is None or not (isinstance(voice, str)): + raise litellm.BadRequestError( + message="'voice' is required to be passed as a string for Azure TTS", + model=model, + llm_provider=custom_llm_provider, + ) + api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore - azure_ad_token: Optional[str] = optional_params.get("extra_body", {}).pop( # type: ignore - "azure_ad_token", None - ) or get_secret( - "AZURE_AD_TOKEN" - ) - azure_ad_token_provider = kwargs.get("azure_ad_token_provider", None) + api_version = api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore - if extra_headers: - optional_params["extra_headers"] = extra_headers + api_key = ( + api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) # type: ignore - response = azure_chat_completions.audio_speech( - model=model, - input=input, - voice=voice, - optional_params=optional_params, - api_key=api_key, - api_base=api_base, - api_version=api_version, - azure_ad_token=azure_ad_token, - azure_ad_token_provider=azure_ad_token_provider, - organization=organization, - max_retries=max_retries, - timeout=timeout, - client=client, # pass AsyncOpenAI, OpenAI client - aspeech=aspeech, - litellm_params=litellm_params_dict, - ) + azure_ad_token: Optional[str] = optional_params.get("extra_body", {}).pop( # type: ignore + "azure_ad_token", None + ) or get_secret( + "AZURE_AD_TOKEN" + ) + azure_ad_token_provider = kwargs.get("azure_ad_token_provider", None) + + if extra_headers: + optional_params["extra_headers"] = extra_headers + + response = azure_chat_completions.audio_speech( + model=model, + input=input, + voice=voice, + optional_params=optional_params, + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + azure_ad_token_provider=azure_ad_token_provider, + organization=organization, + max_retries=max_retries, + timeout=timeout, + client=client, # pass AsyncOpenAI, OpenAI client + aspeech=aspeech, + litellm_params=litellm_params_dict, + ) elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta": generic_optional_params = GenericLiteLLMParams(**kwargs) @@ -5923,6 +5993,8 @@ async def ahealth_check( "batch", "rerank", "realtime", + "responses", + "ocr", ] ] = "chat", prompt: Optional[str] = None, @@ -5985,53 +6057,13 @@ async def ahealth_check( litellm_logging_obj=litellm_logging_obj, ) - mode_handlers = { - "chat": lambda: litellm.acompletion( - **model_params, - ), - "completion": lambda: litellm.atext_completion( - **_filter_model_params(model_params), - prompt=prompt or "test", - ), - "embedding": lambda: litellm.aembedding( - **_filter_model_params(model_params), - input=input or ["test"], - ), - "audio_speech": lambda: litellm.aspeech( - **{ - **_filter_model_params(model_params), - **( - {"voice": "alloy"} - if "voice" not in _filter_model_params(model_params) - else {} - ), - }, - input=prompt or "test", - ), - "audio_transcription": lambda: litellm.atranscription( - **_filter_model_params(model_params), - file=get_audio_file_for_health_check(), - ), - "image_generation": lambda: litellm.aimage_generation( - **_filter_model_params(model_params), - prompt=prompt, - ), - "rerank": lambda: litellm.arerank( - **_filter_model_params(model_params), - query=prompt or "", - documents=["my sample text"], - ), - "realtime": lambda: _realtime_health_check( - model=model, - custom_llm_provider=custom_llm_provider, - api_base=model_params.get("api_base", None), - api_key=model_params.get("api_key", None), - api_version=model_params.get("api_version", None), - ), - "batch": lambda: litellm.alist_batches( - **_filter_model_params(model_params), - ), - } + mode_handlers = HealthCheckHelpers.get_mode_handlers( + model=model, + custom_llm_provider=custom_llm_provider, + model_params=model_params, + prompt=prompt, + input=input, + ) if mode in mode_handlers: _response = await mode_handlers[mode]() @@ -6266,6 +6298,18 @@ def stream_chunk_builder( # noqa: PLR0915 processor.get_combined_reasoning_content(reasoning_chunks) ) + annotation_chunks = [ + chunk + for chunk in chunks + if len(chunk["choices"]) > 0 + and "annotations" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["annotations"] is not None + ] + + if len(annotation_chunks) > 0: + annotations = annotation_chunks[0]["choices"][0]["delta"]["annotations"] + response["choices"][0]["message"]["annotations"] = annotations + audio_chunks = [ chunk for chunk in chunks