From 3c8b58bd80573ae514594ef40e2c676f1d827997 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 23 Feb 2024 08:48:21 -0800 Subject: [PATCH 1/3] (feat) support extra_headers --- litellm/main.py | 3 +++ litellm/tests/test_optional_params.py | 15 +++++++++++++++ litellm/utils.py | 5 +++++ 3 files changed, 23 insertions(+) diff --git a/litellm/main.py b/litellm/main.py index 1ee36504f13..120f85cb1e8 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -398,6 +398,7 @@ def completion( logprobs: Optional[bool] = None, top_logprobs: Optional[int] = None, deployment_id=None, + extra_headers: Optional[dict] = None, # soon to be deprecated params by OpenAI functions: Optional[List] = None, function_call: Optional[str] = None, @@ -514,6 +515,7 @@ def completion( "max_retries", "logprobs", "top_logprobs", + "extra_headers", ] litellm_params = [ "metadata", @@ -691,6 +693,7 @@ def completion( max_retries=max_retries, logprobs=logprobs, top_logprobs=top_logprobs, + extra_headers=extra_headers, **non_default_params, ) diff --git a/litellm/tests/test_optional_params.py b/litellm/tests/test_optional_params.py index 45c3b8a38e5..4fa8df3b6db 100644 --- a/litellm/tests/test_optional_params.py +++ b/litellm/tests/test_optional_params.py @@ -118,3 +118,18 @@ def test_azure_gpt_optional_params_gpt_vision_with_extra_body(): # test_azure_gpt_optional_params_gpt_vision_with_extra_body() + + +def test_openai_extra_headers(): + optional_params = litellm.utils.get_optional_params( + user="John", + custom_llm_provider="openai", + max_tokens=10, + temperature=0.2, + extra_headers={"AI-Resource Group": "ishaan-resource"}, + ) + + print(optional_params) + assert optional_params["max_tokens"] == 10 + assert optional_params["temperature"] == 0.2 + assert optional_params["extra_headers"] == {"AI-Resource Group": "ishaan-resource"} diff --git a/litellm/utils.py b/litellm/utils.py index e3516f7fdcb..ed130c247a4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3907,6 +3907,7 @@ def get_optional_params( max_retries=None, logprobs=None, top_logprobs=None, + extra_headers=None, **kwargs, ): # retrieve all parameters passed to the function @@ -3946,6 +3947,7 @@ def get_optional_params( "max_retries": None, "logprobs": None, "top_logprobs": None, + "extra_headers": None, } # filter out those parameters that were passed with non-default values non_default_params = { @@ -4753,6 +4755,7 @@ def get_optional_params( "max_retries", "logprobs", "top_logprobs", + "extra_headers", ] _check_valid_arg(supported_params=supported_params) if functions is not None: @@ -4793,6 +4796,8 @@ def get_optional_params( optional_params["logprobs"] = logprobs if top_logprobs is not None: optional_params["top_logprobs"] = top_logprobs + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers if custom_llm_provider in ["openai", "azure"] + litellm.openai_compatible_providers: # for openai, azure we should pass the extra/passed params within `extra_body` https://github.com/openai/openai-python/blob/ac33853ba10d13ac149b1fa3ca6dba7d613065c9/src/openai/resources/models.py#L46 extra_body = passed_params.pop("extra_body", {}) From aa9164e2ce9730fbf5468504178fc9b088e003f4 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 23 Feb 2024 08:56:09 -0800 Subject: [PATCH 2/3] (docs) setting extra_headers --- docs/my-website/docs/proxy/configs.md | 5 +---- docs/my-website/docs/simple_proxy_old_doc.md | 4 ---- litellm/main.py | 1 + 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index e9fd4cda4be..0a155828b0d 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -227,6 +227,7 @@ model_list: - model_name: openai-gpt-3.5 litellm_params: model: openai/gpt-3.5-turbo + extra_headers: {"AI-Resource Group": "ishaan-resource"} api_key: sk-123 organization: org-ikDc4ex8NB temperature: 0.2 @@ -234,10 +235,6 @@ model_list: litellm_params: model: ollama/mistral api_base: your_ollama_api_base - headers: { - "HTTP-Referer": "litellm.ai", - "X-Title": "LiteLLM Server" - } ``` **Step 2**: Start server with config diff --git a/docs/my-website/docs/simple_proxy_old_doc.md b/docs/my-website/docs/simple_proxy_old_doc.md index 01c8a5754b2..b48e345e1de 100644 --- a/docs/my-website/docs/simple_proxy_old_doc.md +++ b/docs/my-website/docs/simple_proxy_old_doc.md @@ -787,10 +787,6 @@ model_list: litellm_params: model: ollama/mistral api_base: your_ollama_api_base - headers: { - "HTTP-Referer": "litellm.ai", - "X-Title": "LiteLLM Server" - } ``` **Step 2**: Start server with config diff --git a/litellm/main.py b/litellm/main.py index 120f85cb1e8..acd2ba51358 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -436,6 +436,7 @@ def completion( api_version (str, optional): API version (default is None). api_key (str, optional): API key (default is None). model_list (list, optional): List of api base, version, keys + extra_headers (dict, optional): Additional headers to include in the request. LITELLM Specific Params mock_response (str, optional): If provided, return a mock completion response for testing or debugging purposes (default is None). From c9567c8652adff301280acaafbe0c643d9b7b49f Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 23 Feb 2024 09:27:48 -0800 Subject: [PATCH 3/3] (docs) passing extra_headers to OpenAI --- docs/my-website/docs/completion/input.md | 10 +++++----- docs/my-website/docs/providers/openai.md | 13 +++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md index 676e4d23265..e3ad9245d93 100644 --- a/docs/my-website/docs/completion/input.md +++ b/docs/my-website/docs/completion/input.md @@ -28,11 +28,11 @@ This is a list of openai params we translate across providers. This list is constantly being updated. -| Provider | temperature | max_tokens | top_p | stream | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed | tools | tool_choice | logprobs | top_logprobs | -|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| Provider | temperature | max_tokens | top_p | stream | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed | tools | tool_choice | logprobs | top_logprobs | extra_headers | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|--| |Anthropic| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | -|OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | -|Azure OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | | | +|OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ | +|Azure OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | | | ✅ | |Replicate | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | |Anyscale | ✅ | ✅ | ✅ | ✅ | |Cohere| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | @@ -42,7 +42,7 @@ This list is constantly being updated. |VertexAI| ✅ | ✅ | | ✅ | | | | | | | |Bedrock| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | |Sagemaker| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | -|TogetherAI| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | +|TogetherAI| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ | |AlephAlpha| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |Palm| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |NLP Cloud| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index dd026661d88..1ca13b6007d 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -156,6 +156,19 @@ response_message = response.choices[0].message tool_calls = response.choices[0].message.tool_calls ``` +### Setting `extra_headers` for completion calls +```python +import os +from litellm import completion + +os.environ["OPENAI_API_KEY"] = "your-api-key" + +response = completion( + model = "gpt-3.5-turbo", + messages=[{ "content": "Hello, how are you?","role": "user"}], + extra_headers={"AI-Resource Group": "ishaan-resource"} +) +``` ### Setting Organization-ID for completion calls This can be set in one of the following ways: