From 4b7feb3261e947ebd7a0ef572b5538253f3b401d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 1 Jul 2024 17:01:42 -0700 Subject: [PATCH 1/6] feat - return response headers for async openai requests --- litellm/__init__.py | 3 +++ litellm/llms/openai.py | 33 +++++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 0fa822a98e2..a9e6b69ae60 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -125,6 +125,9 @@ llm_guard_mode: Literal["all", "key-specific", "request-specific"] = "all" ################## ### PREVIEW FEATURES ### enable_preview_features: bool = False +return_response_headers: bool = ( + False # get response headers from LLM Api providers - example x-remaining-requests, +) ################## logging: bool = True caching: bool = ( diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py index 32e63b95760..d98dd9d8c28 100644 --- a/litellm/llms/openai.py +++ b/litellm/llms/openai.py @@ -652,6 +652,31 @@ class OpenAIChatCompletion(BaseLLM): else: return client + async def make_openai_chat_completion_request( + self, + openai_aclient: AsyncOpenAI, + data: dict, + timeout: Union[float, httpx.Timeout], + ): + try: + if litellm.return_response_headers is True: + raw_response = ( + await openai_aclient.chat.completions.with_raw_response.create( + **data, timeout=timeout + ) + ) + + headers = dict(raw_response.headers) + response = raw_response.parse() + return headers, response + else: + response = await openai_aclient.chat.completions.create( + **data, timeout=timeout + ) + return None, response + except Exception as e: + raise e + def completion( self, model_response: ModelResponse, @@ -869,8 +894,8 @@ class OpenAIChatCompletion(BaseLLM): }, ) - response = await openai_aclient.chat.completions.create( - **data, timeout=timeout + headers, response = await self.make_openai_chat_completion_request( + openai_aclient=openai_aclient, data=data, timeout=timeout ) stringified_response = response.model_dump() logging_obj.post_call( @@ -965,8 +990,8 @@ class OpenAIChatCompletion(BaseLLM): }, ) - response = await openai_aclient.chat.completions.create( - **data, timeout=timeout + headers, response = await self.make_openai_chat_completion_request( + openai_aclient=openai_aclient, data=data, timeout=timeout ) streamwrapper = CustomStreamWrapper( completion_stream=response, From 48946f7528983fbce3f4297fa403ad78b7ba1ce0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 1 Jul 2024 17:02:15 -0700 Subject: [PATCH 2/6] fix config --- litellm/proxy/proxy_config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 88b778a6d47..9f2324e51c3 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -36,6 +36,7 @@ general_settings: LANGFUSE_SECRET_KEY: "os.environ/LANGFUSE_DEV_SK_KEY" litellm_settings: + return_response_headers: true success_callback: ["prometheus"] callbacks: ["otel", "hide_secrets"] failure_callback: ["prometheus"] From 140f7fe2544536e4d4b8606e1d78fd6fdcf8f8e4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 1 Jul 2024 17:09:06 -0700 Subject: [PATCH 3/6] return azure response headers --- litellm/llms/azure.py | 45 ++++++++++++++++++++++++++++++++++++++---- litellm/llms/openai.py | 5 +++++ 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/litellm/llms/azure.py b/litellm/llms/azure.py index e127ecea6a4..4b36edff28b 100644 --- a/litellm/llms/azure.py +++ b/litellm/llms/azure.py @@ -458,6 +458,36 @@ class AzureChatCompletion(BaseLLM): return azure_client + async def make_azure_openai_chat_completion_request( + self, + azure_client: AsyncAzureOpenAI, + data: dict, + timeout: Union[float, httpx.Timeout], + ): + """ + Helper to: + - call chat.completions.create.with_raw_response when litellm.return_response_headers is True + - call chat.completions.create by default + """ + try: + if litellm.return_response_headers is True: + raw_response = ( + await azure_client.chat.completions.with_raw_response.create( + **data, timeout=timeout + ) + ) + + headers = dict(raw_response.headers) + response = raw_response.parse() + return headers, response + else: + response = await azure_client.chat.completions.create( + **data, timeout=timeout + ) + return None, response + except Exception as e: + raise e + def completion( self, model: str, @@ -701,8 +731,11 @@ class AzureChatCompletion(BaseLLM): "complete_input_dict": data, }, ) - response = await azure_client.chat.completions.create( - **data, timeout=timeout + + headers, response = await self.make_azure_openai_chat_completion_request( + azure_client=azure_client, + data=data, + timeout=timeout, ) stringified_response = response.model_dump() @@ -861,9 +894,13 @@ class AzureChatCompletion(BaseLLM): "complete_input_dict": data, }, ) - response = await azure_client.chat.completions.create( - **data, timeout=timeout + + headers, response = await self.make_azure_openai_chat_completion_request( + azure_client=azure_client, + data=data, + timeout=timeout, ) + # return response streamwrapper = CustomStreamWrapper( completion_stream=response, diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py index d98dd9d8c28..c7bb0e35324 100644 --- a/litellm/llms/openai.py +++ b/litellm/llms/openai.py @@ -658,6 +658,11 @@ class OpenAIChatCompletion(BaseLLM): data: dict, timeout: Union[float, httpx.Timeout], ): + """ + Helper to: + - call chat.completions.create.with_raw_response when litellm.return_response_headers is True + - call chat.completions.create by default + """ try: if litellm.return_response_headers is True: raw_response = ( From 04a975d486bed4474fc4d5e2abc78cb412a2a492 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 1 Jul 2024 17:25:15 -0700 Subject: [PATCH 4/6] feat - add response_headers in litellm_logging_obj --- litellm/llms/openai.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py index c7bb0e35324..357bda79972 100644 --- a/litellm/llms/openai.py +++ b/litellm/llms/openai.py @@ -21,6 +21,7 @@ from pydantic import BaseModel from typing_extensions import overload, override import litellm +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.utils import ProviderField from litellm.utils import ( Choices, @@ -866,13 +867,13 @@ class OpenAIChatCompletion(BaseLLM): self, data: dict, model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, timeout: Union[float, httpx.Timeout], api_key: Optional[str] = None, api_base: Optional[str] = None, organization: Optional[str] = None, client=None, max_retries=None, - logging_obj=None, headers=None, ): response = None @@ -909,9 +910,11 @@ class OpenAIChatCompletion(BaseLLM): original_response=stringified_response, additional_args={"complete_input_dict": data}, ) + logging_obj.model_call_details["response_headers"] = headers return convert_to_model_response_object( response_object=stringified_response, model_response_object=model_response, + hidden_params={"headers": headers}, ) except Exception as e: raise e @@ -961,10 +964,10 @@ class OpenAIChatCompletion(BaseLLM): async def async_streaming( self, - logging_obj, timeout: Union[float, httpx.Timeout], data: dict, model: str, + logging_obj: LiteLLMLoggingObj, api_key: Optional[str] = None, api_base: Optional[str] = None, organization: Optional[str] = None, @@ -998,6 +1001,7 @@ class OpenAIChatCompletion(BaseLLM): headers, response = await self.make_openai_chat_completion_request( openai_aclient=openai_aclient, data=data, timeout=timeout ) + logging_obj.model_call_details["response_headers"] = headers streamwrapper = CustomStreamWrapper( completion_stream=response, model=model, @@ -1527,9 +1531,9 @@ class OpenAITextCompletion(BaseLLM): model: str, messages: list, timeout: float, + logging_obj: LiteLLMLoggingObj, print_verbose: Optional[Callable] = None, api_base: Optional[str] = None, - logging_obj=None, acompletion: bool = False, optional_params=None, litellm_params=None, From 568245b5c00209876ff17ce29539a1cea21e8041 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 1 Jul 2024 20:12:39 -0700 Subject: [PATCH 5/6] feat - set response headers in azure requests --- litellm/llms/azure.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/litellm/llms/azure.py b/litellm/llms/azure.py index 4b36edff28b..000feed4451 100644 --- a/litellm/llms/azure.py +++ b/litellm/llms/azure.py @@ -23,6 +23,7 @@ from typing_extensions import overload import litellm from litellm import OpenAIConfig from litellm.caching import DualCache +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.utils import ( Choices, CustomStreamWrapper, @@ -500,7 +501,7 @@ class AzureChatCompletion(BaseLLM): azure_ad_token: str, print_verbose: Callable, timeout: Union[float, httpx.Timeout], - logging_obj, + logging_obj: LiteLLMLoggingObj, optional_params, litellm_params, logger_fn, @@ -679,9 +680,9 @@ class AzureChatCompletion(BaseLLM): data: dict, timeout: Any, model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, azure_ad_token: Optional[str] = None, client=None, # this is the AsyncAzureOpenAI - logging_obj=None, ): response = None try: @@ -737,6 +738,7 @@ class AzureChatCompletion(BaseLLM): data=data, timeout=timeout, ) + logging_obj.model_call_details["response_headers"] = headers stringified_response = response.model_dump() logging_obj.post_call( @@ -845,7 +847,7 @@ class AzureChatCompletion(BaseLLM): async def async_streaming( self, - logging_obj, + logging_obj: LiteLLMLoggingObj, api_base: str, api_key: str, api_version: str, @@ -900,6 +902,7 @@ class AzureChatCompletion(BaseLLM): data=data, timeout=timeout, ) + logging_obj.model_call_details["response_headers"] = headers # return response streamwrapper = CustomStreamWrapper( From 403330265602ce5eb19eaf0dd699eece3af1be8f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 1 Jul 2024 20:27:27 -0700 Subject: [PATCH 6/6] feat - return headers for openai audio transcriptions --- litellm/llms/openai.py | 72 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 6 deletions(-) diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py index 357bda79972..990ef2faeb2 100644 --- a/litellm/llms/openai.py +++ b/litellm/llms/openai.py @@ -1026,17 +1026,43 @@ class OpenAIChatCompletion(BaseLLM): else: raise OpenAIError(status_code=500, message=f"{str(e)}") + # Embedding + async def make_openai_embedding_request( + self, + openai_aclient: AsyncOpenAI, + data: dict, + timeout: Union[float, httpx.Timeout], + ): + """ + Helper to: + - call embeddings.create.with_raw_response when litellm.return_response_headers is True + - call embeddings.create by default + """ + try: + if litellm.return_response_headers is True: + raw_response = await openai_aclient.embeddings.with_raw_response.create( + **data, timeout=timeout + ) # type: ignore + headers = dict(raw_response.headers) + response = raw_response.parse() + return headers, response + else: + response = await openai_aclient.embeddings.create(**data, timeout=timeout) # type: ignore + return None, response + except Exception as e: + raise e + async def aembedding( self, input: list, data: dict, model_response: litellm.utils.EmbeddingResponse, timeout: float, + logging_obj: LiteLLMLoggingObj, api_key: Optional[str] = None, api_base: Optional[str] = None, client: Optional[AsyncOpenAI] = None, max_retries=None, - logging_obj=None, ): response = None try: @@ -1048,7 +1074,10 @@ class OpenAIChatCompletion(BaseLLM): max_retries=max_retries, client=client, ) - response = await openai_aclient.embeddings.create(**data, timeout=timeout) # type: ignore + headers, response = await self.make_openai_embedding_request( + openai_aclient=openai_aclient, data=data, timeout=timeout + ) + logging_obj.model_call_details["response_headers"] = headers stringified_response = response.model_dump() ## LOGGING logging_obj.post_call( @@ -1263,6 +1292,34 @@ class OpenAIChatCompletion(BaseLLM): else: raise OpenAIError(status_code=500, message=str(e)) + # Audio Transcriptions + async def make_openai_audio_transcriptions_request( + self, + openai_aclient: AsyncOpenAI, + data: dict, + timeout: Union[float, httpx.Timeout], + ): + """ + Helper to: + - call openai_aclient.audio.transcriptions.with_raw_response when litellm.return_response_headers is True + - call openai_aclient.audio.transcriptions.create by default + """ + try: + if litellm.return_response_headers is True: + raw_response = ( + await openai_aclient.audio.transcriptions.with_raw_response.create( + **data, timeout=timeout + ) + ) # type: ignore + headers = dict(raw_response.headers) + response = raw_response.parse() + return headers, response + else: + response = await openai_aclient.audio.transcriptions.create(**data, timeout=timeout) # type: ignore + return None, response + except Exception as e: + raise e + def audio_transcriptions( self, model: str, @@ -1320,11 +1377,11 @@ class OpenAIChatCompletion(BaseLLM): data: dict, model_response: TranscriptionResponse, timeout: float, + logging_obj: LiteLLMLoggingObj, api_key: Optional[str] = None, api_base: Optional[str] = None, client=None, max_retries=None, - logging_obj=None, ): try: openai_aclient = self._get_openai_client( @@ -1336,9 +1393,12 @@ class OpenAIChatCompletion(BaseLLM): client=client, ) - response = await openai_aclient.audio.transcriptions.create( - **data, timeout=timeout - ) # type: ignore + headers, response = await self.make_openai_audio_transcriptions_request( + openai_aclient=openai_aclient, + data=data, + timeout=timeout, + ) + logging_obj.model_call_details["response_headers"] = headers stringified_response = response.model_dump() ## LOGGING logging_obj.post_call(