diff --git a/litellm_server/.env.template b/litellm/litellm_server/.env.template similarity index 100% rename from litellm_server/.env.template rename to litellm/litellm_server/.env.template diff --git a/litellm_server/Dockerfile b/litellm/litellm_server/Dockerfile similarity index 100% rename from litellm_server/Dockerfile rename to litellm/litellm_server/Dockerfile diff --git a/litellm_server/README.md b/litellm/litellm_server/README.md similarity index 100% rename from litellm_server/README.md rename to litellm/litellm_server/README.md diff --git a/litellm_server/__init__.py b/litellm/litellm_server/__init__.py similarity index 100% rename from litellm_server/__init__.py rename to litellm/litellm_server/__init__.py diff --git a/litellm_server/main.py b/litellm/litellm_server/main.py similarity index 97% rename from litellm_server/main.py rename to litellm/litellm_server/main.py index 1b77c097c86..d8d7579201a 100644 --- a/litellm_server/main.py +++ b/litellm/litellm_server/main.py @@ -9,7 +9,7 @@ sys.path.insert( 0, os.path.abspath("../") ) # Adds the parent directory to the system path - for litellm local dev import litellm -print(f"litellm: {litellm}") + try: from utils import set_callbacks, load_router_config, print_verbose except ImportError: @@ -36,7 +36,6 @@ server_settings: Optional[dict] = None set_callbacks() # sets litellm callbacks for logging if they exist in the environment if "CONFIG_FILE_PATH" in os.environ: - print(f"CONFIG FILE DETECTED") llm_router, llm_model_list, server_settings = load_router_config(router=llm_router, config_file_path=os.getenv("CONFIG_FILE_PATH")) else: llm_router, llm_model_list, server_settings = load_router_config(router=llm_router) @@ -61,9 +60,9 @@ def model_list(): ) # for streaming def data_generator(response): - print("inside generator") + for chunk in response: - print(f"returned chunk: {chunk}") + yield f"data: {json.dumps(chunk)}\n\n" @router.post("/v1/completions") @@ -105,7 +104,6 @@ async def chat_completion(request: Request, model: Optional[str] = None): global llm_model_list, server_settings try: data = await request.json() - print(f"data: {data}") server_model = server_settings.get("completion_model", None) if server_settings else None data["model"] = server_model or model or data["model"] ## CHECK KEYS ## @@ -136,11 +134,10 @@ async def chat_completion(request: Request, model: Optional[str] = None): ) if 'stream' in data and data['stream'] == True: # use generate_responses to stream responses return StreamingResponse(data_generator(response), media_type='text/event-stream') - print(f"response: {response}") return response except Exception as e: error_traceback = traceback.format_exc() - print(f"{error_traceback}") + error_msg = f"{str(e)}\n\n{error_traceback}" # return {"error": error_msg} raise HTTPException(status_code=500, detail=error_msg) diff --git a/litellm_server/openapi.json b/litellm/litellm_server/openapi.json similarity index 100% rename from litellm_server/openapi.json rename to litellm/litellm_server/openapi.json diff --git a/litellm_server/requirements.txt b/litellm/litellm_server/requirements.txt similarity index 100% rename from litellm_server/requirements.txt rename to litellm/litellm_server/requirements.txt diff --git a/litellm_server/utils.py b/litellm/litellm_server/utils.py similarity index 87% rename from litellm_server/utils.py rename to litellm/litellm_server/utils.py index ffaa64c9151..bcc17237cee 100644 --- a/litellm_server/utils.py +++ b/litellm/litellm_server/utils.py @@ -4,9 +4,7 @@ import dotenv dotenv.load_dotenv() # load env variables def print_verbose(print_statement): - print(f"SET_VERBOSE value: {os.environ['SET_VERBOSE']}") - if os.environ["SET_VERBOSE"] == "True": - print(print_statement) + pass def get_package_version(package_name): try: @@ -19,9 +17,9 @@ def get_package_version(package_name): package_name = "litellm" version = get_package_version(package_name) if version: - print(f"The version of {package_name} is {version}") + print_verbose(f"The version of {package_name} is {version}") else: - print(f"{package_name} is not installed") + print_verbose(f"{package_name} is not installed") import yaml import dotenv from typing import Optional @@ -32,14 +30,14 @@ def set_callbacks(): if len(os.getenv("SET_VERBOSE", "")) > 0: if os.getenv("SET_VERBOSE") == "True": litellm.set_verbose = True - print("\033[92mLiteLLM: Switched on verbose logging\033[0m") + print_verbose("\033[92mLiteLLM: Switched on verbose logging\033[0m") else: litellm.set_verbose = False ### LANGFUSE if (len(os.getenv("LANGFUSE_PUBLIC_KEY", "")) > 0 and len(os.getenv("LANGFUSE_SECRET_KEY", ""))) > 0 or len(os.getenv("LANGFUSE_HOST", "")) > 0: litellm.success_callback = ["langfuse"] - print("\033[92mLiteLLM: Switched on Langfuse feature\033[0m") + print_verbose("\033[92mLiteLLM: Switched on Langfuse feature\033[0m") ## CACHING ### REDIS diff --git a/litellm_server/tests/test_bedrock.py b/litellm_server/tests/test_bedrock.py deleted file mode 100644 index 7825de891ce..00000000000 --- a/litellm_server/tests/test_bedrock.py +++ /dev/null @@ -1,39 +0,0 @@ -import openai -openai.api_base = "http://0.0.0.0:8000" -print("making request") -openai.api_key = "anything" # this gets passed as a header - - -response = openai.chat.completions.create( - model = "bedrock/anthropic.claude-instant-v1", - messages = [ - { - "role": "user", - "content": "this is a test message, what model / llm are you" - } - ], - aws_access_key_id="", - aws_secret_access_key="", - aws_region_name="us-west-2", - max_tokens = 10, -) - - -print(response) - - -# response = openai.chat.completions.create( -# model = "gpt-3.5-turbo", -# messages = [ -# { -# "role": "user", -# "content": "this is a test message, what model / llm are you" -# } -# ], -# max_tokens = 10, -# stream=True -# ) - - -# for chunk in response: -# print(chunk) \ No newline at end of file diff --git a/litellm_server/tests/test_caching.py b/litellm_server/tests/test_caching.py deleted file mode 100644 index c95d31924c3..00000000000 --- a/litellm_server/tests/test_caching.py +++ /dev/null @@ -1,80 +0,0 @@ -import openai, os, dotenv, traceback, time -openai.api_base = "http://0.0.0.0:8000" -dotenv.load_dotenv() -openai.api_key = os.getenv("ANTHROPIC_API_KEY") # this gets passed as a header - - -response1 = openai.chat.completions.create( - model = "claude-instant-1", - messages = [ - { - "role": "user", - "content": "write a short poem about litellm" - } - ], -) - -try: - print(f"response: {response1['choices'][0]['message']['content']}") -except: - print(f"response: {response1}") - -time.sleep(1) # allow time for request to be stored - -response2 = openai.chat.completions.create( - model = "claude-instant-1", - messages = [ - { - "role": "user", - "content": "write a short poem about litellm" - } - ], -) - -try: - print(f"response: {response2['choices'][0]['message']['content']}") -except: - print(f"response: {response2}") - -openai.api_key = os.getenv("OPENAI_API_KEY") - -try: - response3 = openai.chat.completions.create( - model = "gpt-3.5-turbo", - messages = [ - { - "role": "user", - "content": "write a short poem about litellm" - } - ], - ) -except Exception as e: - traceback.print_exc() - -try: - print(f"response: {response3['choices'][0]['message']['content']}") -except: - print(f"response: {response3}") - -openai.api_key = os.getenv("ANTHROPIC_API_KEY") # this gets passed as a header -# switch caching off using cache flag -response4 = openai.chat.completions.create( - model = "claude-instant-1", - messages = [ - { - "role": "user", - "content": "write a short poem about litellm" - } - ], - caching = False, -) - -try: - print(f"response: {response4['choices'][0]['message']['content']}") -except: - print(f"response: {response4}") - -assert response1["choices"][0]["message"]["content"] == response2["choices"][0]["message"]["content"] -assert response1["choices"][0]["message"]["content"] != response4["choices"][0]["message"]["content"] - -assert response1["choices"][0]["message"]["content"] != response3["choices"][0]["message"]["content"] \ No newline at end of file diff --git a/litellm_server/tests/test_openai.py b/litellm_server/tests/test_openai.py deleted file mode 100644 index 1a0556dbb20..00000000000 --- a/litellm_server/tests/test_openai.py +++ /dev/null @@ -1,39 +0,0 @@ -import openai -openai.api_base = "http://127.0.0.1:8000" -openai.api_key = "this can be anything" - -print("making request") - -api_key = "" -response = openai.chat.completions.create( - model = "gpt-3.5-turbo", - messages = [ - { - "role": "user", - "content": "this is a test message, what model / llm are you" - } - ], - api_key=api_key, - max_tokens = 10, -) - - -print(response) - - -response = openai.chat.completions.create( - model = "gpt-3.5-turbo", - messages = [ - { - "role": "user", - "content": "this is a test message, what model / llm are you" - } - ], - api_key=api_key, - max_tokens = 10, - stream=True -) - - -for chunk in response: - print(chunk) \ No newline at end of file diff --git a/litellm_server/tests/test_openrouter.py b/litellm_server/tests/test_openrouter.py deleted file mode 100644 index 861fb771e28..00000000000 --- a/litellm_server/tests/test_openrouter.py +++ /dev/null @@ -1,38 +0,0 @@ -import openai -openai.api_base = "http://0.0.0.0:8000" -openai.api_key = "this can be anything" -print("making request") - -api_key = "" -response = openai.chat.completions.create( - model = "openrouter/google/palm-2-chat-bison", - messages = [ - { - "role": "user", - "content": "this is a test message, what model / llm are you" - } - ], - api_key=api_key, - max_tokens = 10, -) - - -print(response) - - -response = openai.chat.completions.create( - model = "openrouter/google/palm-2-chat-bison", - messages = [ - { - "role": "user", - "content": "this is a test message, what model / llm are you" - } - ], - api_key=api_key, - max_tokens = 10, - stream=True -) - - -for chunk in response: - print(chunk) \ No newline at end of file diff --git a/litellm_server/tests/test_router.py b/litellm_server/tests/test_router.py deleted file mode 100644 index 13977145a53..00000000000 --- a/litellm_server/tests/test_router.py +++ /dev/null @@ -1,59 +0,0 @@ -#### What this tests #### -# This tests calling batch_completions by running 100 messages together - -import sys, os -import traceback, asyncio -import pytest -from fastapi.testclient import TestClient -from fastapi import Request -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -from openai_proxy import app - - -def test_router_completion(): - client = TestClient(app) - data = { - "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": "Hey, how's it going?"}], - "model_list": [{ # list of model deployments - "model_name": "gpt-3.5-turbo", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-v-2", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE") - }, - "tpm": 240000, - "rpm": 1800 - }, { - "model_name": "gpt-3.5-turbo", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-functioncalling", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE") - }, - "tpm": 240000, - "rpm": 1800 - }, { - "model_name": "gpt-3.5-turbo", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - "tpm": 1000000, - "rpm": 9000 - }] - } - - response = client.post("/router/completions", json=data) - print(f"response: {response.text}") - assert response.status_code == 200 - - response_data = response.json() - # Perform assertions on the response data - assert isinstance(response_data['choices'][0]['message']['content'], str) - -test_router_completion()