scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge pull request #1475 from BerriAI/litellm_azure_vision_enhancements

Browse source 
[Feat] Support Azure GPT-4 Vision Enhancements
This commit is contained in:
Ishaan Jaff 2024-01-17 11:18:56 -08:00 • committed by GitHub
commit a8ee3511ef
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 164 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

65
docs/my-website/docs/providers/azure.md
View file

@ -116,8 +116,17 @@ response = completion(
```
#### Usage - with Azure Vision enhancements
### Usage - with Azure Vision enhancements
Note: **Azure requires the `base_url` to be set with `/extensions`**
Example
```python
base_url=https://gpt-4-vision-resource.openai.azure.com/openai/deployments/gpt-4-vision/extensions
# base_url="{azure_endpoint}/openai/deployments/{azure_deployment}/extensions"
```
**Usage**
```python
import os
from litellm import completion
@ -126,34 +135,34 @@ os.environ["AZURE_API_KEY"] = "your-api-key"
# azure call
response = completion(
model = "azure/<your deployment name>",
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "What’s in this image?"
model="azure/gpt-4-vision",
timeout=5,
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "Whats in this image?"},
{
"type": "image_url",
"image_url": {
"url": "https://avatars.githubusercontent.com/u/29436595?v=4"
},
{
"type": "image_url",
"image_url": {
"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
}
}
]
}
],
enhancements = {"ocr": {"enabled": True}, "grounding": {"enabled": True}},
dataSources = [
{
"type": "AzureComputerVision",
"parameters": {
"endpoint": "<your_computer_vision_endpoint>",
"key": "<your_computer_vision_key>",
},
}
],
},
],
}
],
base_url="https://gpt-4-vision-resource.openai.azure.com/openai/deployments/gpt-4-vision/extensions",
api_key=os.getenv("AZURE_VISION_API_KEY"),
enhancements={"ocr": {"enabled": True}, "grounding": {"enabled": True}},
dataSources=[
{
"type": "AzureComputerVision",
"parameters": {
"endpoint": "https://gpt-4-vision-enhancement.cognitiveservices.azure.com/",
"key": os.environ["AZURE_VISION_ENHANCE_KEY"],
},
}
],
)
```

38
litellm/llms/azure.py
View file

@ -95,6 +95,26 @@ class AzureOpenAIConfig(OpenAIConfig):
)
def select_azure_base_url_or_endpoint(azure_client_params: dict):
# azure_client_params = {
# "api_version": api_version,
# "azure_endpoint": api_base,
# "azure_deployment": model,
# "http_client": litellm.client_session,
# "max_retries": max_retries,
# "timeout": timeout,
# }
azure_endpoint = azure_client_params.get("azure_endpoint", None)
if azure_endpoint is not None:
# see : https://github.com/openai/openai-python/blob/3d61ed42aba652b547029095a7eb269ad4e1e957/src/openai/lib/azure.py#L192
if "/openai/deployments" in azure_endpoint:
# this is base_url, not an azure_endpoint
azure_client_params["base_url"] = azure_endpoint
azure_client_params.pop("azure_endpoint")
return azure_client_params
class AzureChatCompletion(BaseLLM):
def __init__(self) -> None:
super().__init__()
@ -239,6 +259,9 @@ class AzureChatCompletion(BaseLLM):
"max_retries": max_retries,
"timeout": timeout,
}
azure_client_params = select_azure_base_url_or_endpoint(
azure_client_params=azure_client_params
)
if api_key is not None:
azure_client_params["api_key"] = api_key
elif azure_ad_token is not None:
@ -303,6 +326,9 @@ class AzureChatCompletion(BaseLLM):
"max_retries": max_retries,
"timeout": timeout,
}
azure_client_params = select_azure_base_url_or_endpoint(
azure_client_params=azure_client_params
)
if api_key is not None:
azure_client_params["api_key"] = api_key
elif azure_ad_token is not None:
@ -364,6 +390,9 @@ class AzureChatCompletion(BaseLLM):
"max_retries": max_retries,
"timeout": timeout,
}
azure_client_params = select_azure_base_url_or_endpoint(
azure_client_params=azure_client_params
)
if api_key is not None:
azure_client_params["api_key"] = api_key
elif azure_ad_token is not None:
@ -414,6 +443,9 @@ class AzureChatCompletion(BaseLLM):
"max_retries": data.pop("max_retries", 2),
"timeout": timeout,
}
azure_client_params = select_azure_base_url_or_endpoint(
azure_client_params=azure_client_params
)
if api_key is not None:
azure_client_params["api_key"] = api_key
elif azure_ad_token is not None:
@ -527,6 +559,9 @@ class AzureChatCompletion(BaseLLM):
"max_retries": max_retries,
"timeout": timeout,
}
azure_client_params = select_azure_base_url_or_endpoint(
azure_client_params=azure_client_params
)
if api_key is not None:
azure_client_params["api_key"] = api_key
elif azure_ad_token is not None:
@ -659,6 +694,9 @@ class AzureChatCompletion(BaseLLM):
"max_retries": max_retries,
"timeout": timeout,
}
azure_client_params = select_azure_base_url_or_endpoint(
azure_client_params=azure_client_params
)
if api_key is not None:
azure_client_params["api_key"] = api_key
elif azure_ad_token is not None:

28
litellm/router.py
View file

@ -1443,12 +1443,22 @@ class Router:
verbose_router_logger.debug(
f"Initializing Azure OpenAI Client for {model_name}, Api Base: {str(api_base)}, Api Key:{api_key}"
)
azure_client_params = {
"api_key": api_key,
"azure_endpoint": api_base,
"api_version": api_version,
}
from litellm.llms.azure import select_azure_base_url_or_endpoint
# this decides if we should set azure_endpoint or base_url on Azure OpenAI Client
# required to support GPT-4 vision enhancements, since base_url needs to be set on Azure OpenAI Client
azure_client_params = select_azure_base_url_or_endpoint(
azure_client_params
)
cache_key = f"{model_id}_async_client"
_client = openai.AsyncAzureOpenAI( # type: ignore
api_key=api_key,
azure_endpoint=api_base,
api_version=api_version,
**azure_client_params,
timeout=timeout,
max_retries=max_retries,
http_client=httpx.AsyncClient(
@ -1467,9 +1477,7 @@ class Router:
cache_key = f"{model_id}_client"
_client = openai.AzureOpenAI( # type: ignore
api_key=api_key,
azure_endpoint=api_base,
api_version=api_version,
**azure_client_params,
timeout=timeout,
max_retries=max_retries,
http_client=httpx.Client(
@ -1489,9 +1497,7 @@ class Router:
# streaming clients should have diff timeouts
cache_key = f"{model_id}_stream_async_client"
_client = openai.AsyncAzureOpenAI( # type: ignore
api_key=api_key,
azure_endpoint=api_base,
api_version=api_version,
**azure_client_params,
timeout=stream_timeout,
max_retries=max_retries,
http_client=httpx.AsyncClient(
@ -1510,9 +1516,7 @@ class Router:
cache_key = f"{model_id}_stream_client"
_client = openai.AzureOpenAI( # type: ignore
api_key=api_key,
azure_endpoint=api_base,
api_version=api_version,
**azure_client_params,
timeout=stream_timeout,
max_retries=max_retries,
http_client=httpx.Client(

20
litellm/tests/test_completion.py
View file

@ -229,7 +229,7 @@ def test_completion_azure_gpt4_vision():
litellm.set_verbose = True
response = completion(
model="azure/gpt-4-vision",
timeout=1,
timeout=5,
messages=[
{
"role": "user",
@ -244,21 +244,31 @@ def test_completion_azure_gpt4_vision():
],
}
],
base_url="https://gpt-4-vision-resource.openai.azure.com/",
base_url="https://gpt-4-vision-resource.openai.azure.com/openai/deployments/gpt-4-vision/extensions",
api_key=os.getenv("AZURE_VISION_API_KEY"),
enhancements={"ocr": {"enabled": True}, "grounding": {"enabled": True}},
dataSources=[
{
"type": "AzureComputerVision",
"parameters": {
"endpoint": "https://gpt-4-vision-enhancement.cognitiveservices.azure.com/",
"key": os.environ["AZURE_VISION_ENHANCE_KEY"],
},
}
],
)
print(response)
except openai.APITimeoutError:
print("got a timeout error")
pass
except openai.RateLimitError:
print("got a rate liimt error")
except openai.RateLimitError as e:
print("got a rate liimt error", e)
pass
except Exception as e:
pytest.fail(f"Error occurred: {e}")
# test_completion_azure_gpt4_vision()
test_completion_azure_gpt4_vision()
@pytest.mark.skip(reason="this test is flaky")

59
litellm/tests/test_router_init.py
View file

@ -23,6 +23,10 @@ load_dotenv()
def test_init_clients():
litellm.set_verbose = True
import logging
from litellm._logging import verbose_router_logger
verbose_router_logger.setLevel(logging.DEBUG)
try:
print("testing init 4 clients with diff timeouts")
model_list = [
@ -39,7 +43,7 @@ def test_init_clients():
},
},
]
router = Router(model_list=model_list)
router = Router(model_list=model_list, set_verbose=True)
for elem in router.model_list:
model_id = elem["model_info"]["id"]
assert router.cache.get_cache(f"{model_id}_client") is not None
@ -55,6 +59,18 @@ def test_init_clients():
assert async_client.timeout == 0.01
assert stream_async_client.timeout == 0.000_001
print(vars(async_client))
print()
print(async_client._base_url)
assert (
async_client._base_url
== "https://openai-gpt-4-test-v-1.openai.azure.com//openai/"
) # openai python adds the extra /
assert (
stream_async_client._base_url
== "https://openai-gpt-4-test-v-1.openai.azure.com//openai/"
)
print("PASSED !")
except Exception as e:
@ -307,3 +323,44 @@ def test_xinference_embedding():
# test_xinference_embedding()
def test_router_init_gpt_4_vision_enhancements():
try:
# tests base_url set when any base_url with /openai/deployments passed to router
print("Testing Azure GPT_Vision enhancements")
model_list = [
{
"model_name": "gpt-4-vision-enhancements",
"litellm_params": {
"model": "azure/gpt-4-vision",
"api_key": os.getenv("AZURE_API_KEY"),
"base_url": "https://gpt-4-vision-resource.openai.azure.com/openai/deployments/gpt-4-vision/extensions/",
},
}
]
router = Router(model_list=model_list)
print(router.model_list)
print(router.model_list[0])
assert (
router.model_list[0]["litellm_params"]["base_url"]
== "https://gpt-4-vision-resource.openai.azure.com/openai/deployments/gpt-4-vision/extensions/"
) # set in env
azure_client = router._get_client(
deployment=router.model_list[0],
kwargs={"stream": True, "model": "gpt-4-vision-enhancements"},
client_type="async",
)
assert (
azure_client._base_url
== "https://gpt-4-vision-resource.openai.azure.com/openai/deployments/gpt-4-vision/extensions/"
)
print("passed")
except Exception as e:
pytest.fail(f"Error occurred: {e}")