From b15682bc1fe8c1043e3c1f520c36d6451eb7c1c3 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 18 Dec 2023 08:55:51 +0530 Subject: [PATCH 01/11] (feat) set default openrouter configs --- litellm/llms/openrouter.py | 11 +++++++++++ litellm/utils.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 litellm/llms/openrouter.py diff --git a/litellm/llms/openrouter.py b/litellm/llms/openrouter.py new file mode 100644 index 00000000000..c1ae146c982 --- /dev/null +++ b/litellm/llms/openrouter.py @@ -0,0 +1,11 @@ +from typing import List + +class OpenrouterConfig(): + """ + Reference: https://openrouter.ai/docs#format + + """ + # OpenRouter-only parameters + transforms: List[str] = [] + models: List[str] = [] + route: str = '' diff --git a/litellm/utils.py b/litellm/utils.py index ab22c200dc2..90e5d4f09cd 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3962,7 +3962,7 @@ def convert_to_model_response_object(response_object: Optional[dict]=None, model raise Exception("Error in response object format") if model_response_object is None: - model_response_object = EmbeddingResponse() + model_response_object = ImageResponse() if "created" in response_object: model_response_object.created = response_object["created"] From 6b272076d7e411220ddc5f989fa7a5015b19b687 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 18 Dec 2023 09:16:33 +0530 Subject: [PATCH 02/11] (feat) openrouter set `transforms=[]` default --- litellm/llms/openrouter.py | 27 +++++++++++++++++++++++---- litellm/main.py | 10 +++++++--- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/litellm/llms/openrouter.py b/litellm/llms/openrouter.py index c1ae146c982..fa21ab61eec 100644 --- a/litellm/llms/openrouter.py +++ b/litellm/llms/openrouter.py @@ -1,4 +1,5 @@ -from typing import List +from typing import List, Dict +import types class OpenrouterConfig(): """ @@ -6,6 +7,24 @@ class OpenrouterConfig(): """ # OpenRouter-only parameters - transforms: List[str] = [] - models: List[str] = [] - route: str = '' + extra_body: Dict[str, List[str]] = { + 'transforms': [] # default transforms to [] + } + + + def __init__(self, + transforms: List[str] = [], + models: List[str] = [], + route: str = '', + ) -> None: + locals_ = locals() + for key, value in locals_.items(): + if key != 'self' and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return {k: v for k, v in cls.__dict__.items() + if not k.startswith('__') + and not isinstance(v, (types.FunctionType, types.BuiltinFunctionType, classmethod, staticmethod)) + and v is not None} \ No newline at end of file diff --git a/litellm/main.py b/litellm/main.py index b7e9ccce222..bb8d8ba8b17 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -52,6 +52,7 @@ from .llms import ( cohere, petals, oobabooga, + openrouter, palm, vertex_ai, maritalk) @@ -1026,14 +1027,17 @@ def completion( } ) + ## Load Config + config = openrouter.OpenrouterConfig.get_config() + for k, v in config.items(): + if k not in optional_params: # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + data = { "model": model, "messages": messages, **optional_params } - ## LOGGING - logging.pre_call(input=messages, api_key=openai.api_key, additional_args={"complete_input_dict": data, "headers": headers}) - ## COMPLETION CALL ## COMPLETION CALL response = openai_chat_completions.completion( From 1e57c0c152bfcafcb36bd404eac4e16e50cf1a3d Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 18 Dec 2023 09:54:43 +0530 Subject: [PATCH 03/11] (feat) completion set function, function_call default None --- litellm/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index bb8d8ba8b17..0ca4add03e5 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -261,8 +261,8 @@ def completion( model: str, # Optional OpenAI params: see https://platform.openai.com/docs/api-reference/chat/create messages: List = [], - functions: List = [], - function_call: str = "", # optional params + functions: Optional[List] = None, + function_call: Optional[str] = None, timeout: Optional[Union[float, int]] = None, temperature: Optional[float] = None, top_p: Optional[float] = None, @@ -1030,7 +1030,7 @@ def completion( ## Load Config config = openrouter.OpenrouterConfig.get_config() for k, v in config.items(): - if k not in optional_params: # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + if k not in optional_params: optional_params[k] = v data = { From 6a0c853ae4e81c3565a408064de48de7b6e50fb4 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 18 Dec 2023 09:55:35 +0530 Subject: [PATCH 04/11] (feat) add open router transforms, models, route --- litellm/utils.py | 59 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 90e5d4f09cd..45d5d02f0dc 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2336,8 +2336,8 @@ def get_optional_params_embeddings( def get_optional_params( # use the openai defaults # 12 optional params - functions=[], - function_call="", + functions=None, + function_call=None, temperature=None, top_p=None, n=None, @@ -2363,8 +2363,8 @@ def get_optional_params( # use the openai defaults for k, v in special_params.items(): passed_params[k] = v default_params = { - "functions":[], - "function_call":"", + "functions": None, + "function_call": None, "temperature":None, "top_p":None, "n":None, @@ -2851,6 +2851,57 @@ def get_optional_params( # use the openai defaults if random_seed is not None: extra_body["random_seed"] = random_seed optional_params["extra_body"] = extra_body # openai client supports `extra_body` param + elif custom_llm_provider == "openrouter": + supported_params = ["functions", "function_call", "temperature", "top_p", "n", "stream", "stop", "max_tokens", "presence_penalty", "frequency_penalty", "logit_bias", "user", "response_format", "seed", "tools", "tool_choice", "max_retries"] + _check_valid_arg(supported_params=supported_params) + + if functions is not None: + optional_params["functions"] = functions + if function_call is not None: + optional_params["function_call"] = function_call + if temperature is not None: + optional_params["temperature"] = temperature + if top_p is not None: + optional_params["top_p"] = top_p + if n is not None: + optional_params["n"] = n + if stream is not None: + optional_params["stream"] = stream + if stop is not None: + optional_params["stop"] = stop + if max_tokens is not None: + optional_params["max_tokens"] = max_tokens + if presence_penalty is not None: + optional_params["presence_penalty"] = presence_penalty + if frequency_penalty is not None: + optional_params["frequency_penalty"] = frequency_penalty + if logit_bias is not None: + optional_params["logit_bias"] = logit_bias + if user is not None: + optional_params["user"] = user + if response_format is not None: + optional_params["response_format"] = response_format + if seed is not None: + optional_params["seed"] = seed + if tools is not None: + optional_params["tools"] = tools + if tool_choice is not None: + optional_params["tool_choice"] = tool_choice + if max_retries is not None: + optional_params["max_retries"] = max_retries + + # OpenRouter-only parameters + extra_body = {} + transforms = passed_params.pop("transforms", None) + models = passed_params.pop("models", None) + route = passed_params.pop("route", None) + if transforms is not None: + extra_body["transforms"] = transforms + if models is not None: + extra_body["models"] = models + if route is not None: + extra_body["route"] = route + optional_params["extra_body"] = extra_body # openai client supports `extra_body` param else: # assume passing in params for openai/azure openai supported_params = ["functions", "function_call", "temperature", "top_p", "n", "stream", "stop", "max_tokens", "presence_penalty", "frequency_penalty", "logit_bias", "user", "response_format", "seed", "tools", "tool_choice", "max_retries"] _check_valid_arg(supported_params=supported_params) From 3a97a2817fbd522d0e093724a4318ea42d9c692c Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 18 Dec 2023 10:05:44 +0530 Subject: [PATCH 05/11] (fix) default args batch completion --- litellm/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 0ca4add03e5..9efd8a411c5 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1514,8 +1514,8 @@ def batch_completion( model: str, # Optional OpenAI params: see https://platform.openai.com/docs/api-reference/chat/create messages: List = [], - functions: List = [], - function_call: str = "", # optional params + functions: Optional[List] = None, + function_call: Optional[str] = None, temperature: Optional[float] = None, top_p: Optional[float] = None, n: Optional[int] = None, From 8ec7e25c06e2b90a60493274dca7bf12856f1c93 Mon Sep 17 00:00:00 2001 From: Eric Ma Date: Sun, 17 Dec 2023 23:51:02 -0500 Subject: [PATCH 06/11] Update mistral.md Eliminated typo with extra quotation mark. --- docs/my-website/docs/providers/mistral.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/mistral.md b/docs/my-website/docs/providers/mistral.md index a7869415b6a..29ca9169d9e 100644 --- a/docs/my-website/docs/providers/mistral.md +++ b/docs/my-website/docs/providers/mistral.md @@ -14,7 +14,7 @@ import os os.environ['MISTRAL_API_KEY'] = "" response = completion( - model="mistral/mistral-tiny"", + model="mistral/mistral-tiny", messages=[ {"role": "user", "content": "hello from litellm"} ], From bd15c61a65632d32c6e1891f49d7192a370fc531 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 18 Dec 2023 10:56:19 +0530 Subject: [PATCH 07/11] (feat) OR default transforms=[] --- litellm/main.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 9efd8a411c5..6e89311912a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1030,7 +1030,13 @@ def completion( ## Load Config config = openrouter.OpenrouterConfig.get_config() for k, v in config.items(): - if k not in optional_params: + if k == "extra_body": + # we use openai 'extra_body' to pass openrouter specific params - transforms, route, models + if "extra_body" in optional_params: + optional_params[k].update(v) + else: + optional_params[k] = v + elif k not in optional_params: optional_params[k] = v data = { From d6ed13fa4f41a29a4e5cbb0ef0a87844230f2a3c Mon Sep 17 00:00:00 2001 From: Ankur Garha Date: Mon, 18 Dec 2023 22:56:08 +0100 Subject: [PATCH 08/11] doc: updated langfuse ver 1.14 in pip install cmd --- docs/my-website/docs/observability/langfuse_integration.md | 2 +- docs/my-website/docs/proxy/logging.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md index 3e5f5603d0b..d5ccbd085ab 100644 --- a/docs/my-website/docs/observability/langfuse_integration.md +++ b/docs/my-website/docs/observability/langfuse_integration.md @@ -15,7 +15,7 @@ join our [discord](https://discord.gg/wuPM9dRgDw) ## Pre-Requisites Ensure you have run `pip install langfuse` for this integration ```shell -pip install langfuse litellm +pip install langfuse==1.14.0 litellm ``` ## Quick Start diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 253c299f643..69544ff3792 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -459,7 +459,7 @@ We will use the `--config` to set `litellm.success_callback = ["langfuse"]` this **Step 1** Install langfuse ```shell -pip install langfuse +pip install langfuse==1.14.0 ``` **Step 2**: Create a `config.yaml` file and set `litellm_settings`: `success_callback` From 014abddff660f84bfa371cb4a2b4fafafa869bc0 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 18 Dec 2023 15:14:57 -0800 Subject: [PATCH 09/11] fix(requirements.txt): pin all dependencies --- requirements.txt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3cf31593579..cc5f8c4926c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,24 +3,24 @@ anyio==4.2.0 # openai + http req. openai>=1.0.0 # openai req. fastapi # server dep pydantic>=2.5 # openai req. -appdirs # server dep -backoff # server dep -pyyaml # server dep -uvicorn # server dep -boto3 # aws bedrock/sagemaker calls -redis # caching -prisma # for db -mangum # for aws lambda functions -google-generativeai # for vertex ai calls +appdirs==1.4.4 # server dep +backoff==2.2.1 # server dep +pyyaml==6.0 # server dep +uvicorn==0.22.0 # server dep +boto3==1.28.58 # aws bedrock/sagemaker calls +redis==4.6.0 # caching +prisma==0.11.0 # for db +mangum==0.17.0 # for aws lambda functions +google-generativeai==0.1.0 # for vertex ai calls traceloop-sdk==0.5.3 # for open telemetry logging langfuse==1.14.0 # for langfuse self-hosted logging ### LITELLM PACKAGE DEPENDENCIES python-dotenv>=0.2.0 # for env tiktoken>=0.4.0 # for calculating usage importlib-metadata>=6.8.0 # for random utils -tokenizers # for calculating usage -click # for proxy cli +tokenizers==0.14.0 # for calculating usage +click==8.1.7 # for proxy cli jinja2==3.1.2 # for prompt templates certifi>=2023.7.22 # [TODO] clean up -aiohttp # for network calls +aiohttp==3.8.4 # for network calls #### \ No newline at end of file From 34509d8dda9db4bd033585a47edb84bd120856bb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 18 Dec 2023 17:41:41 -0800 Subject: [PATCH 10/11] fix(main.py): return async completion calls --- litellm/llms/openai.py | 54 +++++++++++++++------------ litellm/main.py | 3 +- litellm/tests/test_text_completion.py | 46 ++++++++++++++++------- 3 files changed, 65 insertions(+), 38 deletions(-) diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py index c923cbf2ddb..0731bd5096d 100644 --- a/litellm/llms/openai.py +++ b/litellm/llms/openai.py @@ -284,7 +284,7 @@ class OpenAIChatCompletion(BaseLLM): additional_args={"complete_input_dict": data}, ) return convert_to_model_response_object(response_object=json.loads(stringified_response), model_response_object=model_response) - except Exception as e: + except Exception as e: raise e def streaming(self, @@ -631,24 +631,27 @@ class OpenAITextCompletion(BaseLLM): api_key: str, model: str): async with httpx.AsyncClient() as client: - response = await client.post(api_base, json=data, headers=headers, timeout=litellm.request_timeout) - response_json = response.json() - if response.status_code != 200: - raise OpenAIError(status_code=response.status_code, message=response.text) - - ## LOGGING - logging_obj.post_call( - input=prompt, - api_key=api_key, - original_response=response, - additional_args={ - "headers": headers, - "api_base": api_base, - }, - ) + try: + response = await client.post(api_base, json=data, headers=headers, timeout=litellm.request_timeout) + response_json = response.json() + if response.status_code != 200: + raise OpenAIError(status_code=response.status_code, message=response.text) + + ## LOGGING + logging_obj.post_call( + input=prompt, + api_key=api_key, + original_response=response, + additional_args={ + "headers": headers, + "api_base": api_base, + }, + ) - ## RESPONSE OBJECT - return self.convert_to_model_response_object(response_object=response_json, model_response_object=model_response) + ## RESPONSE OBJECT + return self.convert_to_model_response_object(response_object=response_json, model_response_object=model_response) + except Exception as e: + raise e def streaming(self, logging_obj, @@ -687,9 +690,12 @@ class OpenAITextCompletion(BaseLLM): method="POST", timeout=litellm.request_timeout ) as response: - if response.status_code != 200: - raise OpenAIError(status_code=response.status_code, message=response.text) - - streamwrapper = CustomStreamWrapper(completion_stream=response.aiter_lines(), model=model, custom_llm_provider="text-completion-openai",logging_obj=logging_obj) - async for transformed_chunk in streamwrapper: - yield transformed_chunk \ No newline at end of file + try: + if response.status_code != 200: + raise OpenAIError(status_code=response.status_code, message=response.text) + + streamwrapper = CustomStreamWrapper(completion_stream=response.aiter_lines(), model=model, custom_llm_provider="text-completion-openai",logging_obj=logging_obj) + async for transformed_chunk in streamwrapper: + yield transformed_chunk + except Exception as e: + raise e \ No newline at end of file diff --git a/litellm/main.py b/litellm/main.py index 6e89311912a..52d2ae5b663 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2205,7 +2205,8 @@ def text_completion( if stream == True or kwargs.get("stream", False) == True: response = TextCompletionStreamWrapper(completion_stream=response, model=model) return response - + if kwargs.get("acompletion", False) == True: + return response transformed_logprobs = None # only supported for TGI models try: diff --git a/litellm/tests/test_text_completion.py b/litellm/tests/test_text_completion.py index 9257a07f3c1..f75bd2f7f8c 100644 --- a/litellm/tests/test_text_completion.py +++ b/litellm/tests/test_text_completion.py @@ -169,17 +169,37 @@ def test_text_completion_stream(): # test_text_completion_stream() -async def test_text_completion_async_stream(): - try: - response = await atext_completion( - model="text-completion-openai/text-davinci-003", - prompt="good morning", - stream=True, - max_tokens=10, - ) - async for chunk in response: - print(f"chunk: {chunk}") - except Exception as e: - pytest.fail(f"GOT exception for HF In streaming{e}") +# async def test_text_completion_async_stream(): +# try: +# response = await atext_completion( +# model="text-completion-openai/text-davinci-003", +# prompt="good morning", +# stream=True, +# max_tokens=10, +# ) +# async for chunk in response: +# print(f"chunk: {chunk}") +# except Exception as e: +# pytest.fail(f"GOT exception for HF In streaming{e}") -asyncio.run(test_text_completion_async_stream()) \ No newline at end of file +# asyncio.run(test_text_completion_async_stream()) + +def test_async_text_completion(): + litellm.set_verbose = True + print('test_async_text_completion') + async def test_get_response(): + try: + response = await litellm.atext_completion( + model="gpt-3.5-turbo-instruct", + prompt="good morning", + stream=False, + max_tokens=10 + ) + print(f"response: {response}") + except litellm.Timeout as e: + print(e) + except Exception as e: + print(e) + + asyncio.run(test_get_response()) +test_async_text_completion() \ No newline at end of file From 071283c102e973c4063415f3db0ded466f32664d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 18 Dec 2023 17:50:26 -0800 Subject: [PATCH 11/11] fix(router.py): init deployment_latency_map even if model_list is empty --- litellm/proxy/proxy_config.yaml | 2 +- litellm/router.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 0180d232ec2..b9f29a584ad 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -47,7 +47,7 @@ litellm_settings: # setting callback class # callbacks: custom_callbacks.proxy_handler_instance # sets litellm.callbacks = [proxy_handler_instance] -general_settings: +# general_settings: environment_variables: # otel: True # OpenTelemetry Logger diff --git a/litellm/router.py b/litellm/router.py index 410d4964ea3..0276f5a444f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -84,11 +84,11 @@ class Router: self.set_verbose = set_verbose self.deployment_names: List = [] # names of models under litellm_params. ex. azure/chatgpt-v-2 + self.deployment_latency_map = {} if model_list: model_list = copy.deepcopy(model_list) self.set_model_list(model_list) self.healthy_deployments: List = self.model_list - self.deployment_latency_map = {} for m in model_list: self.deployment_latency_map[m["litellm_params"]["model"]] = 0