From 995019c08a77cc36dc009047763bddd7e4015364 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Wed, 4 Sep 2024 16:36:27 -0700 Subject: [PATCH] =?UTF-8?q?Revert=20"(bedrock):=20Fix=20usage=20with=20Clo?= =?UTF-8?q?udflare=20AI=20Gateway,=20and=20proxies=20in=20gener=E2=80=A6"?= =?UTF-8?q?=20(#5519)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 3fac0349c257d88d497fe95e29016cb75f4d6506. --- litellm/llms/bedrock/chat.py | 28 +++++++++++-------------- litellm/llms/bedrock/common_utils.py | 18 +++------------- litellm/llms/bedrock/embed/embedding.py | 2 +- 3 files changed, 16 insertions(+), 32 deletions(-) diff --git a/litellm/llms/bedrock/chat.py b/litellm/llms/bedrock/chat.py index 972a3abd39a..73e649c5b6b 100644 --- a/litellm/llms/bedrock/chat.py +++ b/litellm/llms/bedrock/chat.py @@ -728,7 +728,7 @@ class BedrockLLM(BaseAWSLLM): ) ### SET RUNTIME ENDPOINT ### - endpoint_url, proxy_endpoint_url = get_runtime_endpoint( + endpoint_url = get_runtime_endpoint( api_base=api_base, aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, aws_region_name=aws_region_name, @@ -736,10 +736,8 @@ class BedrockLLM(BaseAWSLLM): if (stream is not None and stream is True) and provider != "ai21": endpoint_url = f"{endpoint_url}/model/{modelId}/invoke-with-response-stream" - proxy_endpoint_url = f"{proxy_endpoint_url}/model/{proxy_endpoint_url}/invoke-with-response-stream" else: endpoint_url = f"{endpoint_url}/model/{modelId}/invoke" - proxy_endpoint_url = f"{proxy_endpoint_url}/model/{proxy_endpoint_url}/invoke" sigv4 = SigV4Auth(credentials, "bedrock", aws_region_name) @@ -905,7 +903,7 @@ class BedrockLLM(BaseAWSLLM): api_key="", additional_args={ "complete_input_dict": data, - "api_base": proxy_endpoint_url, + "api_base": prepped.url, "headers": prepped.headers, }, ) @@ -919,7 +917,7 @@ class BedrockLLM(BaseAWSLLM): model=model, messages=messages, data=data, - api_base=proxy_endpoint_url, + api_base=prepped.url, model_response=model_response, print_verbose=print_verbose, encoding=encoding, @@ -937,7 +935,7 @@ class BedrockLLM(BaseAWSLLM): model=model, messages=messages, data=data, - api_base=proxy_endpoint_url, + api_base=prepped.url, model_response=model_response, print_verbose=print_verbose, encoding=encoding, @@ -962,7 +960,7 @@ class BedrockLLM(BaseAWSLLM): self.client = client if (stream is not None and stream == True) and provider != "ai21": response = self.client.post( - url=proxy_endpoint_url, + url=prepped.url, headers=prepped.headers, # type: ignore data=data, stream=stream, @@ -993,7 +991,7 @@ class BedrockLLM(BaseAWSLLM): return streaming_response try: - response = self.client.post(url=proxy_endpoint_url, headers=prepped.headers, data=data) # type: ignore + response = self.client.post(url=prepped.url, headers=prepped.headers, data=data) # type: ignore response.raise_for_status() except httpx.HTTPStatusError as err: error_code = err.response.status_code @@ -1606,17 +1604,15 @@ class BedrockConverseLLM(BaseAWSLLM): ) ### SET RUNTIME ENDPOINT ### - endpoint_url, proxy_endpoint_url = get_runtime_endpoint( + endpoint_url = get_runtime_endpoint( api_base=api_base, aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, aws_region_name=aws_region_name, ) if (stream is not None and stream is True) and provider != "ai21": endpoint_url = f"{endpoint_url}/model/{modelId}/converse-stream" - proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/converse-stream" else: endpoint_url = f"{endpoint_url}/model/{modelId}/converse" - proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/converse" sigv4 = SigV4Auth(credentials, "bedrock", aws_region_name) @@ -1723,7 +1719,7 @@ class BedrockConverseLLM(BaseAWSLLM): api_key="", additional_args={ "complete_input_dict": data, - "api_base": proxy_endpoint_url, + "api_base": prepped.url, "headers": prepped.headers, }, ) @@ -1737,7 +1733,7 @@ class BedrockConverseLLM(BaseAWSLLM): model=model, messages=messages, data=data, - api_base=proxy_endpoint_url, + api_base=prepped.url, model_response=model_response, print_verbose=print_verbose, encoding=encoding, @@ -1755,7 +1751,7 @@ class BedrockConverseLLM(BaseAWSLLM): model=model, messages=messages, data=data, - api_base=proxy_endpoint_url, + api_base=prepped.url, model_response=model_response, print_verbose=print_verbose, encoding=encoding, @@ -1776,7 +1772,7 @@ class BedrockConverseLLM(BaseAWSLLM): make_call=partial( make_sync_call, client=None, - api_base=proxy_endpoint_url, + api_base=prepped.url, headers=prepped.headers, # type: ignore data=data, model=model, @@ -1801,7 +1797,7 @@ class BedrockConverseLLM(BaseAWSLLM): else: client = client try: - response = client.post(url=proxy_endpoint_url, headers=prepped.headers, data=data) # type: ignore + response = client.post(url=prepped.url, headers=prepped.headers, data=data) # type: ignore response.raise_for_status() except httpx.HTTPStatusError as err: error_code = err.response.status_code diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 25379474ef5..f2032d110b3 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -5,7 +5,7 @@ Common utilities used across bedrock chat/embedding/image generation import os import types from enum import Enum -from typing import List, Optional, Union, Tuple +from typing import List, Optional, Union import httpx @@ -729,7 +729,7 @@ def get_runtime_endpoint( api_base: Optional[str], aws_bedrock_runtime_endpoint: Optional[str], aws_region_name: str, -) -> Tuple[str, str]: +) -> str: env_aws_bedrock_runtime_endpoint = get_secret("AWS_BEDROCK_RUNTIME_ENDPOINT") if api_base is not None: endpoint_url = api_base @@ -744,19 +744,7 @@ def get_runtime_endpoint( else: endpoint_url = f"https://bedrock-runtime.{aws_region_name}.amazonaws.com" - # Determine proxy_endpoint_url - if env_aws_bedrock_runtime_endpoint and isinstance( - env_aws_bedrock_runtime_endpoint, str - ): - proxy_endpoint_url = env_aws_bedrock_runtime_endpoint - elif aws_bedrock_runtime_endpoint is not None and isinstance( - aws_bedrock_runtime_endpoint, str - ): - proxy_endpoint_url = aws_bedrock_runtime_endpoint - else: - proxy_endpoint_url = endpoint_url - - return endpoint_url, proxy_endpoint_url + return endpoint_url class ModelResponseIterator: diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index e6a1319b012..6398c2c3412 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -393,7 +393,7 @@ class BedrockEmbedding(BaseAWSLLM): batch_data.append(transformed_request) ### SET RUNTIME ENDPOINT ### - endpoint_url, proxy_endpoint_url = get_runtime_endpoint( + endpoint_url = get_runtime_endpoint( api_base=api_base, aws_bedrock_runtime_endpoint=optional_params.pop( "aws_bedrock_runtime_endpoint", None