diff --git a/docs/my-website/docs/providers/azure.md b/docs/my-website/docs/providers/azure.md index dda7384c099..a3385b5ade1 100644 --- a/docs/my-website/docs/providers/azure.md +++ b/docs/my-website/docs/providers/azure.md @@ -118,7 +118,7 @@ response = completion( ``` -### Usage - with Azure Vision enhancements +#### Usage - with Azure Vision enhancements Note: **Azure requires the `base_url` to be set with `/extensions`** @@ -170,12 +170,30 @@ response = completion( ## Azure Instruct Models +Use `model="azure_text/"` + | Model Name | Function Call | |---------------------|----------------------------------------------------| -| gpt-3.5-turbo-instruct | `response = completion(model="azure/", messages=messages)` | -| gpt-3.5-turbo-instruct-0914 | `response = completion(model="azure/", messages=messages)` | +| gpt-3.5-turbo-instruct | `response = completion(model="azure_text/", messages=messages)` | +| gpt-3.5-turbo-instruct-0914 | `response = completion(model="azure_text/", messages=messages)` | +```python +import litellm + +## set ENV variables +os.environ["AZURE_API_KEY"] = "" +os.environ["AZURE_API_BASE"] = "" +os.environ["AZURE_API_VERSION"] = "" + +response = litellm.completion( + model="azure_text/ None: + super().__init__( + frequency_penalty, + function_call, + functions, + logit_bias, + max_tokens, + n, + presence_penalty, + stop, + temperature, + top_p, + ) + + +def select_azure_base_url_or_endpoint(azure_client_params: dict): + # azure_client_params = { + # "api_version": api_version, + # "azure_endpoint": api_base, + # "azure_deployment": model, + # "http_client": litellm.client_session, + # "max_retries": max_retries, + # "timeout": timeout, + # } + azure_endpoint = azure_client_params.get("azure_endpoint", None) + if azure_endpoint is not None: + # see : https://github.com/openai/openai-python/blob/3d61ed42aba652b547029095a7eb269ad4e1e957/src/openai/lib/azure.py#L192 + if "/openai/deployments" in azure_endpoint: + # this is base_url, not an azure_endpoint + azure_client_params["base_url"] = azure_endpoint + azure_client_params.pop("azure_endpoint") + + return azure_client_params + + +class AzureTextCompletion(BaseLLM): + def __init__(self) -> None: + super().__init__() + + def validate_environment(self, api_key, azure_ad_token): + headers = { + "content-type": "application/json", + } + if api_key is not None: + headers["api-key"] = api_key + elif azure_ad_token is not None: + headers["Authorization"] = f"Bearer {azure_ad_token}" + return headers + + def completion( + self, + model: str, + messages: list, + model_response: ModelResponse, + api_key: str, + api_base: str, + api_version: str, + api_type: str, + azure_ad_token: str, + print_verbose: Callable, + timeout, + logging_obj, + optional_params, + litellm_params, + logger_fn, + acompletion: bool = False, + headers: Optional[dict] = None, + client=None, + ): + super().completion() + exception_mapping_worked = False + try: + if model is None or messages is None: + raise AzureOpenAIError( + status_code=422, message=f"Missing model or messages" + ) + + max_retries = optional_params.pop("max_retries", 2) + prompt = prompt_factory( + messages=messages, model=model, custom_llm_provider="azure_text" + ) + + ### CHECK IF CLOUDFLARE AI GATEWAY ### + ### if so - set the model as part of the base url + if "gateway.ai.cloudflare.com" in api_base: + ## build base url - assume api base includes resource name + if client is None: + if not api_base.endswith("/"): + api_base += "/" + api_base += f"{model}" + + azure_client_params = { + "api_version": api_version, + "base_url": f"{api_base}", + "http_client": litellm.client_session, + "max_retries": max_retries, + "timeout": timeout, + } + if api_key is not None: + azure_client_params["api_key"] = api_key + elif azure_ad_token is not None: + azure_client_params["azure_ad_token"] = azure_ad_token + + if acompletion is True: + client = AsyncAzureOpenAI(**azure_client_params) + else: + client = AzureOpenAI(**azure_client_params) + + data = {"model": None, "prompt": prompt, **optional_params} + else: + data = { + "model": model, # type: ignore + "prompt": prompt, + **optional_params, + } + + if acompletion is True: + if optional_params.get("stream", False): + return self.async_streaming( + logging_obj=logging_obj, + api_base=api_base, + data=data, + model=model, + api_key=api_key, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + client=client, + ) + else: + return self.acompletion( + api_base=api_base, + data=data, + model_response=model_response, + api_key=api_key, + api_version=api_version, + model=model, + azure_ad_token=azure_ad_token, + timeout=timeout, + client=client, + logging_obj=logging_obj, + ) + elif "stream" in optional_params and optional_params["stream"] == True: + return self.streaming( + logging_obj=logging_obj, + api_base=api_base, + data=data, + model=model, + api_key=api_key, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + client=client, + ) + else: + ## LOGGING + logging_obj.pre_call( + input=prompt, + api_key=api_key, + additional_args={ + "headers": { + "api_key": api_key, + "azure_ad_token": azure_ad_token, + }, + "api_version": api_version, + "api_base": api_base, + "complete_input_dict": data, + }, + ) + if not isinstance(max_retries, int): + raise AzureOpenAIError( + status_code=422, message="max retries must be an int" + ) + # init AzureOpenAI Client + azure_client_params = { + "api_version": api_version, + "azure_endpoint": api_base, + "azure_deployment": model, + "http_client": litellm.client_session, + "max_retries": max_retries, + "timeout": timeout, + } + azure_client_params = select_azure_base_url_or_endpoint( + azure_client_params=azure_client_params + ) + if api_key is not None: + azure_client_params["api_key"] = api_key + elif azure_ad_token is not None: + azure_client_params["azure_ad_token"] = azure_ad_token + if client is None: + azure_client = AzureOpenAI(**azure_client_params) + else: + azure_client = client + if api_version is not None and isinstance( + azure_client._custom_query, dict + ): + # set api_version to version passed by user + azure_client._custom_query.setdefault( + "api-version", api_version + ) + + response = azure_client.completions.create(**data, timeout=timeout) # type: ignore + stringified_response = response.model_dump() + ## LOGGING + logging_obj.post_call( + input=prompt, + api_key=api_key, + original_response=stringified_response, + additional_args={ + "headers": headers, + "api_version": api_version, + "api_base": api_base, + }, + ) + return openai_text_completion.convert_to_model_response_object( + response_object=stringified_response, + model_response_object=model_response, + ) + except AzureOpenAIError as e: + exception_mapping_worked = True + raise e + except Exception as e: + if hasattr(e, "status_code"): + raise AzureOpenAIError(status_code=e.status_code, message=str(e)) + else: + raise AzureOpenAIError(status_code=500, message=str(e)) + + async def acompletion( + self, + api_key: str, + api_version: str, + model: str, + api_base: str, + data: dict, + timeout: Any, + model_response: ModelResponse, + azure_ad_token: Optional[str] = None, + client=None, # this is the AsyncAzureOpenAI + logging_obj=None, + ): + response = None + try: + max_retries = data.pop("max_retries", 2) + if not isinstance(max_retries, int): + raise AzureOpenAIError( + status_code=422, message="max retries must be an int" + ) + + # init AzureOpenAI Client + azure_client_params = { + "api_version": api_version, + "azure_endpoint": api_base, + "azure_deployment": model, + "http_client": litellm.client_session, + "max_retries": max_retries, + "timeout": timeout, + } + azure_client_params = select_azure_base_url_or_endpoint( + azure_client_params=azure_client_params + ) + if api_key is not None: + azure_client_params["api_key"] = api_key + elif azure_ad_token is not None: + azure_client_params["azure_ad_token"] = azure_ad_token + + # setting Azure client + if client is None: + azure_client = AsyncAzureOpenAI(**azure_client_params) + else: + azure_client = client + if api_version is not None and isinstance( + azure_client._custom_query, dict + ): + # set api_version to version passed by user + azure_client._custom_query.setdefault("api-version", api_version) + ## LOGGING + logging_obj.pre_call( + input=data["prompt"], + api_key=azure_client.api_key, + additional_args={ + "headers": {"Authorization": f"Bearer {azure_client.api_key}"}, + "api_base": azure_client._base_url._uri_reference, + "acompletion": True, + "complete_input_dict": data, + }, + ) + response = await azure_client.completions.create(**data, timeout=timeout) + return openai_text_completion.convert_to_model_response_object( + response_object=response.model_dump(), + model_response_object=model_response, + ) + except AzureOpenAIError as e: + exception_mapping_worked = True + raise e + except Exception as e: + if hasattr(e, "status_code"): + raise e + else: + raise AzureOpenAIError(status_code=500, message=str(e)) + + def streaming( + self, + logging_obj, + api_base: str, + api_key: str, + api_version: str, + data: dict, + model: str, + timeout: Any, + azure_ad_token: Optional[str] = None, + client=None, + ): + max_retries = data.pop("max_retries", 2) + if not isinstance(max_retries, int): + raise AzureOpenAIError( + status_code=422, message="max retries must be an int" + ) + # init AzureOpenAI Client + azure_client_params = { + "api_version": api_version, + "azure_endpoint": api_base, + "azure_deployment": model, + "http_client": litellm.client_session, + "max_retries": max_retries, + "timeout": timeout, + } + azure_client_params = select_azure_base_url_or_endpoint( + azure_client_params=azure_client_params + ) + if api_key is not None: + azure_client_params["api_key"] = api_key + elif azure_ad_token is not None: + azure_client_params["azure_ad_token"] = azure_ad_token + if client is None: + azure_client = AzureOpenAI(**azure_client_params) + else: + azure_client = client + if api_version is not None and isinstance(azure_client._custom_query, dict): + # set api_version to version passed by user + azure_client._custom_query.setdefault("api-version", api_version) + ## LOGGING + logging_obj.pre_call( + input=data["prompt"], + api_key=azure_client.api_key, + additional_args={ + "headers": {"Authorization": f"Bearer {azure_client.api_key}"}, + "api_base": azure_client._base_url._uri_reference, + "acompletion": True, + "complete_input_dict": data, + }, + ) + response = azure_client.completions.create(**data, timeout=timeout) + streamwrapper = CustomStreamWrapper( + completion_stream=response, + model=model, + custom_llm_provider="azure_text", + logging_obj=logging_obj, + ) + return streamwrapper + + async def async_streaming( + self, + logging_obj, + api_base: str, + api_key: str, + api_version: str, + data: dict, + model: str, + timeout: Any, + azure_ad_token: Optional[str] = None, + client=None, + ): + try: + # init AzureOpenAI Client + azure_client_params = { + "api_version": api_version, + "azure_endpoint": api_base, + "azure_deployment": model, + "http_client": litellm.client_session, + "max_retries": data.pop("max_retries", 2), + "timeout": timeout, + } + azure_client_params = select_azure_base_url_or_endpoint( + azure_client_params=azure_client_params + ) + if api_key is not None: + azure_client_params["api_key"] = api_key + elif azure_ad_token is not None: + azure_client_params["azure_ad_token"] = azure_ad_token + if client is None: + azure_client = AsyncAzureOpenAI(**azure_client_params) + else: + azure_client = client + if api_version is not None and isinstance( + azure_client._custom_query, dict + ): + # set api_version to version passed by user + azure_client._custom_query.setdefault("api-version", api_version) + ## LOGGING + logging_obj.pre_call( + input=data["prompt"], + api_key=azure_client.api_key, + additional_args={ + "headers": {"Authorization": f"Bearer {azure_client.api_key}"}, + "api_base": azure_client._base_url._uri_reference, + "acompletion": True, + "complete_input_dict": data, + }, + ) + response = await azure_client.completions.create(**data, timeout=timeout) + # return response + streamwrapper = CustomStreamWrapper( + completion_stream=response, + model=model, + custom_llm_provider="azure_text", + logging_obj=logging_obj, + ) + return streamwrapper ## DO NOT make this into an async for ... loop, it will yield an async generator, which won't raise errors if the response fails + except Exception as e: + if hasattr(e, "status_code"): + raise AzureOpenAIError(status_code=e.status_code, message=str(e)) + else: + raise AzureOpenAIError(status_code=500, message=str(e)) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index dc4207a0510..ae12d954a89 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -807,6 +807,20 @@ def gemini_text_image_pt(messages: list): return content +def azure_text_pt(messages: list): + prompt = "" + for message in messages: + if isinstance(message["content"], str): + prompt += message["content"] + elif isinstance(message["content"], list): + # see https://docs.litellm.ai/docs/providers/openai#openai-vision-models + for element in message["content"]: + if isinstance(element, dict): + if element["type"] == "text": + prompt += element["text"] + return prompt + + # Function call template def function_call_prompt(messages: list, functions: list): function_prompt = ( @@ -907,6 +921,8 @@ def prompt_factory( for message in messages: message.pop("name", None) return messages + elif custom_llm_provider == "azure_text": + return azure_text_pt(messages=messages) try: if "meta-llama/llama-2" in model and "chat" in model: return llama_2_chat_pt(messages=messages) diff --git a/litellm/main.py b/litellm/main.py index 114b469488e..a73cf6b8811 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -64,6 +64,7 @@ from .llms import ( ) from .llms.openai import OpenAIChatCompletion, OpenAITextCompletion from .llms.azure import AzureChatCompletion +from .llms.azure_text import AzureTextCompletion from .llms.huggingface_restapi import Huggingface from .llms.prompt_templates.factory import ( prompt_factory, @@ -96,6 +97,7 @@ dotenv.load_dotenv() # Loading env variables using dotenv openai_chat_completions = OpenAIChatCompletion() openai_text_completions = OpenAITextCompletion() azure_chat_completions = AzureChatCompletion() +azure_text_completions = AzureTextCompletion() huggingface = Huggingface() ####### COMPLETION ENDPOINTS ################ @@ -254,6 +256,7 @@ async def acompletion( if ( custom_llm_provider == "openai" or custom_llm_provider == "azure" + or custom_llm_provider == "azure_text" or custom_llm_provider == "custom_openai" or custom_llm_provider == "anyscale" or custom_llm_provider == "mistral" @@ -800,6 +803,71 @@ def completion( client=client, # pass AsyncAzureOpenAI, AzureOpenAI client ) + if optional_params.get("stream", False) or acompletion == True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={ + "headers": headers, + "api_version": api_version, + "api_base": api_base, + }, + ) + elif custom_llm_provider == "azure_text": + # azure configs + api_type = get_secret("AZURE_API_TYPE") or "azure" + + api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") + + api_version = ( + api_version or litellm.api_version or get_secret("AZURE_API_VERSION") + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) + + azure_ad_token = optional_params.get("extra_body", {}).pop( + "azure_ad_token", None + ) or get_secret("AZURE_AD_TOKEN") + + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config = litellm.AzureOpenAIConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + response = azure_text_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=api_version, + api_type=api_type, + azure_ad_token=azure_ad_token, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + ) + if optional_params.get("stream", False) or acompletion == True: ## LOGGING logging.post_call( diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 729bf7bd998..2b372f57a6f 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -1152,6 +1152,30 @@ def test_completion_azure_key_completion_arg(): # test_completion_azure_key_completion_arg() +def test_azure_instruct(): + litellm.set_verbose = True + response = completion( + model="azure_text/instruct-model", + messages=[{"role": "user", "content": "What is the weather like in Boston?"}], + max_tokens=10, + ) + print("response", response) + + +@pytest.mark.asyncio +async def test_azure_instruct_stream(): + litellm.set_verbose = False + response = await litellm.acompletion( + model="azure_text/instruct-model", + messages=[{"role": "user", "content": "What is the weather like in Boston?"}], + max_tokens=10, + stream=True, + ) + print("response", response) + async for chunk in response: + print(chunk) + + async def test_re_use_azure_async_client(): try: print("azure gpt-3.5 ASYNC with clie nttest\n\n") diff --git a/litellm/utils.py b/litellm/utils.py index 3b6169770df..262935faa58 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8656,6 +8656,27 @@ class CustomStreamWrapper: traceback.print_exc() raise e + def handle_azure_text_completion_chunk(self, chunk): + try: + print_verbose(f"\nRaw OpenAI Chunk\n{chunk}\n") + text = "" + is_finished = False + finish_reason = None + choices = getattr(chunk, "choices", []) + if len(choices) > 0: + text = choices[0].text + if choices[0].finish_reason is not None: + is_finished = True + finish_reason = choices[0].finish_reason + return { + "text": text, + "is_finished": is_finished, + "finish_reason": finish_reason, + } + + except Exception as e: + raise e + def handle_openai_text_completion_chunk(self, chunk): try: print_verbose(f"\nRaw OpenAI Chunk\n{chunk}\n") @@ -9129,6 +9150,14 @@ class CustomStreamWrapper: model_response.choices[0].finish_reason = response_obj[ "finish_reason" ] + elif self.custom_llm_provider == "azure_text": + response_obj = self.handle_azure_text_completion_chunk(chunk) + completion_obj["content"] = response_obj["text"] + print_verbose(f"completion obj content: {completion_obj['content']}") + if response_obj["is_finished"]: + model_response.choices[0].finish_reason = response_obj[ + "finish_reason" + ] elif self.custom_llm_provider == "cached_response": response_obj = { "text": chunk.choices[0].delta.content, @@ -9406,6 +9435,7 @@ class CustomStreamWrapper: or self.custom_llm_provider == "azure" or self.custom_llm_provider == "custom_openai" or self.custom_llm_provider == "text-completion-openai" + or self.custom_llm_provider == "azure_text" or self.custom_llm_provider == "huggingface" or self.custom_llm_provider == "ollama" or self.custom_llm_provider == "ollama_chat"