scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

(chore) move litellm_server inside litellm

Browse source
This commit is contained in:
ishaan-jaff 2023-11-15 16:56:32 -08:00
parent 03efc9185e
commit 08c182c57c
13 changed files with 9 additions and 269 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
litellm_server/.env.template → litellm/litellm_server/.env.template
View file

0
litellm_server/Dockerfile → litellm/litellm_server/Dockerfile
View file

0
litellm_server/README.md → litellm/litellm_server/README.md
View file

0
litellm_server/__init__.py → litellm/litellm_server/__init__.py
View file

11
litellm_server/main.py → litellm/litellm_server/main.py
View file

@ -9,7 +9,7 @@ sys.path.insert(
0, os.path.abspath("../")
) # Adds the parent directory to the system path - for litellm local dev
import litellm
print(f"litellm: {litellm}")
try:
from utils import set_callbacks, load_router_config, print_verbose
except ImportError:
@ -36,7 +36,6 @@ server_settings: Optional[dict] = None
set_callbacks() # sets litellm callbacks for logging if they exist in the environment
if "CONFIG_FILE_PATH" in os.environ:
print(f"CONFIG FILE DETECTED")
llm_router, llm_model_list, server_settings = load_router_config(router=llm_router, config_file_path=os.getenv("CONFIG_FILE_PATH"))
else:
llm_router, llm_model_list, server_settings = load_router_config(router=llm_router)
@ -61,9 +60,9 @@ def model_list():
)
# for streaming
def data_generator(response):
print("inside generator")
for chunk in response:
print(f"returned chunk: {chunk}")
yield f"data: {json.dumps(chunk)}\n\n"
@router.post("/v1/completions")
@ -105,7 +104,6 @@ async def chat_completion(request: Request, model: Optional[str] = None):
global llm_model_list, server_settings
try:
data = await request.json()
print(f"data: {data}")
server_model = server_settings.get("completion_model", None) if server_settings else None
data["model"] = server_model or model or data["model"]
## CHECK KEYS ##
@ -136,11 +134,10 @@ async def chat_completion(request: Request, model: Optional[str] = None):
)
if 'stream' in data and data['stream'] == True: # use generate_responses to stream responses
return StreamingResponse(data_generator(response), media_type='text/event-stream')
print(f"response: {response}")
return response
except Exception as e:
error_traceback = traceback.format_exc()
print(f"{error_traceback}")
error_msg = f"{str(e)}\n\n{error_traceback}"
# return {"error": error_msg}
raise HTTPException(status_code=500, detail=error_msg)

0
litellm_server/openapi.json → litellm/litellm_server/openapi.json
View file

0
litellm_server/requirements.txt → litellm/litellm_server/requirements.txt
View file

12
litellm_server/utils.py → litellm/litellm_server/utils.py
View file

@ -4,9 +4,7 @@ import dotenv
dotenv.load_dotenv() # load env variables
def print_verbose(print_statement):
print(f"SET_VERBOSE value: {os.environ['SET_VERBOSE']}")
if os.environ["SET_VERBOSE"] == "True":
print(print_statement)
pass
def get_package_version(package_name):
try:
@ -19,9 +17,9 @@ def get_package_version(package_name):
package_name = "litellm"
version = get_package_version(package_name)
if version:
print(f"The version of {package_name} is {version}")
print_verbose(f"The version of {package_name} is {version}")
else:
print(f"{package_name} is not installed")
print_verbose(f"{package_name} is not installed")
import yaml
import dotenv
from typing import Optional
@ -32,14 +30,14 @@ def set_callbacks():
if len(os.getenv("SET_VERBOSE", "")) > 0:
if os.getenv("SET_VERBOSE") == "True":
litellm.set_verbose = True
print("\033[92mLiteLLM: Switched on verbose logging\033[0m")
print_verbose("\033[92mLiteLLM: Switched on verbose logging\033[0m")
else:
litellm.set_verbose = False
### LANGFUSE
if (len(os.getenv("LANGFUSE_PUBLIC_KEY", "")) > 0 and len(os.getenv("LANGFUSE_SECRET_KEY", ""))) > 0 or len(os.getenv("LANGFUSE_HOST", "")) > 0:
litellm.success_callback = ["langfuse"]
print("\033[92mLiteLLM: Switched on Langfuse feature\033[0m")
print_verbose("\033[92mLiteLLM: Switched on Langfuse feature\033[0m")
## CACHING
### REDIS

39
litellm_server/tests/test_bedrock.py
View file

@ -1,39 +0,0 @@
import openai
openai.api_base = "http://0.0.0.0:8000"
print("making request")
openai.api_key = "anything" # this gets passed as a header
response = openai.chat.completions.create(
model = "bedrock/anthropic.claude-instant-v1",
messages = [
{
"role": "user",
"content": "this is a test message, what model / llm are you"
}
],
aws_access_key_id="",
aws_secret_access_key="",
aws_region_name="us-west-2",
max_tokens = 10,
)
print(response)
# response = openai.chat.completions.create(
# model = "gpt-3.5-turbo",
# messages = [
# {
# "role": "user",
# "content": "this is a test message, what model / llm are you"
# }
# ],
# max_tokens = 10,
# stream=True
# )
# for chunk in response:
# print(chunk)

80
litellm_server/tests/test_caching.py
View file

@ -1,80 +0,0 @@
import openai, os, dotenv, traceback, time
openai.api_base = "http://0.0.0.0:8000"
dotenv.load_dotenv()
openai.api_key = os.getenv("ANTHROPIC_API_KEY") # this gets passed as a header
response1 = openai.chat.completions.create(
model = "claude-instant-1",
messages = [
{
"role": "user",
"content": "write a short poem about litellm"
}
],
)
try:
print(f"response: {response1['choices'][0]['message']['content']}")
except:
print(f"response: {response1}")
time.sleep(1) # allow time for request to be stored
response2 = openai.chat.completions.create(
model = "claude-instant-1",
messages = [
{
"role": "user",
"content": "write a short poem about litellm"
}
],
)
try:
print(f"response: {response2['choices'][0]['message']['content']}")
except:
print(f"response: {response2}")
openai.api_key = os.getenv("OPENAI_API_KEY")
try:
response3 = openai.chat.completions.create(
model = "gpt-3.5-turbo",
messages = [
{
"role": "user",
"content": "write a short poem about litellm"
}
],
)
except Exception as e:
traceback.print_exc()
try:
print(f"response: {response3['choices'][0]['message']['content']}")
except:
print(f"response: {response3}")
openai.api_key = os.getenv("ANTHROPIC_API_KEY") # this gets passed as a header
# switch caching off using cache flag
response4 = openai.chat.completions.create(
model = "claude-instant-1",
messages = [
{
"role": "user",
"content": "write a short poem about litellm"
}
],
caching = False,
)
try:
print(f"response: {response4['choices'][0]['message']['content']}")
except:
print(f"response: {response4}")
assert response1["choices"][0]["message"]["content"] == response2["choices"][0]["message"]["content"]
assert response1["choices"][0]["message"]["content"] != response4["choices"][0]["message"]["content"]
assert response1["choices"][0]["message"]["content"] != response3["choices"][0]["message"]["content"]

39
litellm_server/tests/test_openai.py
View file

@ -1,39 +0,0 @@
import openai
openai.api_base = "http://127.0.0.1:8000"
openai.api_key = "this can be anything"
print("making request")
api_key = ""
response = openai.chat.completions.create(
model = "gpt-3.5-turbo",
messages = [
{
"role": "user",
"content": "this is a test message, what model / llm are you"
}
],
api_key=api_key,
max_tokens = 10,
)
print(response)
response = openai.chat.completions.create(
model = "gpt-3.5-turbo",
messages = [
{
"role": "user",
"content": "this is a test message, what model / llm are you"
}
],
api_key=api_key,
max_tokens = 10,
stream=True
)
for chunk in response:
print(chunk)

38
litellm_server/tests/test_openrouter.py
View file

@ -1,38 +0,0 @@
import openai
openai.api_base = "http://0.0.0.0:8000"
openai.api_key = "this can be anything"
print("making request")
api_key = ""
response = openai.chat.completions.create(
model = "openrouter/google/palm-2-chat-bison",
messages = [
{
"role": "user",
"content": "this is a test message, what model / llm are you"
}
],
api_key=api_key,
max_tokens = 10,
)
print(response)
response = openai.chat.completions.create(
model = "openrouter/google/palm-2-chat-bison",
messages = [
{
"role": "user",
"content": "this is a test message, what model / llm are you"
}
],
api_key=api_key,
max_tokens = 10,
stream=True
)
for chunk in response:
print(chunk)

59
litellm_server/tests/test_router.py
View file

@ -1,59 +0,0 @@
#### What this tests ####
# This tests calling batch_completions by running 100 messages together
import sys, os
import traceback, asyncio
import pytest
from fastapi.testclient import TestClient
from fastapi import Request
sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
from openai_proxy import app
def test_router_completion():
client = TestClient(app)
data = {
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hey, how's it going?"}],
"model_list": [{ # list of model deployments
"model_name": "gpt-3.5-turbo", # openai model name
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-v-2",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE")
},
"tpm": 240000,
"rpm": 1800
}, {
"model_name": "gpt-3.5-turbo", # openai model name
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-functioncalling",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE")
},
"tpm": 240000,
"rpm": 1800
}, {
"model_name": "gpt-3.5-turbo", # openai model name
"litellm_params": { # params for litellm completion/embedding call
"model": "gpt-3.5-turbo",
"api_key": os.getenv("OPENAI_API_KEY"),
},
"tpm": 1000000,
"rpm": 9000
}]
}
response = client.post("/router/completions", json=data)
print(f"response: {response.text}")
assert response.status_code == 200
response_data = response.json()
# Perform assertions on the response data
assert isinstance(response_data['choices'][0]['message']['content'], str)
test_router_completion()