scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 86

Merge pull request #628 from coconut49/main

Browse source 
Update the Dockerfile of the LiteLLM Proxy server and some refactorings
This commit is contained in:
Krish Dholakia 2023-10-17 11:28:00 -07:00 • committed by GitHub
commit dbd1d702e4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 208 additions and 200 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -2,6 +2,7 @@
.env
litellm_uuid.txt
__pycache__/
*.pyc
bun.lockb
**/.DS_Store
.aider*
@ -10,3 +11,4 @@ secrets.toml
.gitignore
litellm/proxy/litellm_secrets.toml
litellm/proxy/api_log.json
.idea/

18
Dockerfile
View file

@ -1,15 +1,13 @@
FROM python:3.10
RUN pip install poetry
WORKDIR /app
COPY . .
COPY . /app
WORKDIR /app
RUN mkdir -p /root/.config/litellm/ && cp /app/secrets_template.toml /root/.config/litellm/litellm.secrets.toml
RUN pip install -r requirements.txt
WORKDIR /app/litellm/proxy
RUN python proxy_cli.py --config -f /app/secrets_template.toml
RUN python proxy_cli.py
EXPOSE 8000
ENTRYPOINT [ "python3", "proxy_cli.py" ]
# TODO - Set up a GitHub Action to automatically create the Docker image,
# and then we can quickly deploy the litellm proxy in the following way
# `docker run -p 8000:8000 -v ./secrets_template.toml:/root/.config/litellm/litellm.secrets.toml ghcr.io/BerriAI/litellm:v0.8.4`

BIN
litellm/__pycache__/__init__.cpython-311.pyc
View file

Binary file not shown.

BIN
litellm/__pycache__/main.cpython-311.pyc
View file

Binary file not shown.

BIN
litellm/__pycache__/timeout.cpython-311.pyc
View file

Binary file not shown.

BIN
litellm/__pycache__/utils.cpython-311.pyc
View file

Binary file not shown.

BIN
litellm/integrations/__pycache__/__init__.cpython-311.pyc
View file

Binary file not shown.

BIN
litellm/integrations/__pycache__/aispend.cpython-311.pyc
View file

Binary file not shown.

BIN
litellm/integrations/__pycache__/berrispend.cpython-311.pyc
View file

Binary file not shown.

BIN
litellm/integrations/__pycache__/helicone.cpython-311.pyc
View file

Binary file not shown.

BIN
litellm/integrations/__pycache__/supabase.cpython-311.pyc
View file

Binary file not shown.

1
litellm/proxy/proxy_cli.py
View file

@ -7,7 +7,6 @@ from dotenv import load_dotenv
import operator
config_filename = "litellm.secrets.toml"
pkg_config_filename = "template.secrets.toml"
# Using appdirs to determine user-specific config path
config_dir = appdirs.user_config_dir("litellm")
user_config_path = os.path.join(config_dir, config_filename)

371
litellm/proxy/proxy_server.py
View file

@ -1,12 +1,12 @@
import sys, os, platform, time, copy
import threading
import shutil, random, traceback
messages = []
sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path - for litellm local dev
try:
import uvicorn
import fastapi
@ -23,13 +23,14 @@ except ImportError:
import tomli as tomllib
import appdirs
import tomli_w
try:
from .llm import litellm_completion
except ImportError as e:
except ImportError as e:
from llm import litellm_completion
import random
list_of_messages = [
"'The thing I wish you improved is...'",
"'A feature I really want is...'",
@ -38,35 +39,36 @@ list_of_messages = [
"'I don't like how this works...'",
"'It would help me if you could add...'",
"'This feature doesn't meet my needs because...'",
"'I get frustrated when the product...'",
"'I get frustrated when the product...'",
]
def generate_feedback_box():
box_width = 60
box_width = 60
# Select a random message
message = random.choice(list_of_messages)
# Select a random message
message = random.choice(list_of_messages)
print()
print('\033[1;37m' + '#' + '-' * box_width + '#\033[0m')
print('\033[1;37m' + '#' + ' ' * box_width + '#\033[0m')
print('\033[1;37m' + '# {:^59} #\033[0m'.format(message))
print('\033[1;37m' + '# {:^59} #\033[0m'.format('https://github.com/BerriAI/litellm/issues/new'))
print('\033[1;37m' + '#' + ' ' * box_width + '#\033[0m')
print('\033[1;37m' + '#' + '-' * box_width + '#\033[0m')
print()
print(' Thank you for using LiteLLM! - Krrish & Ishaan')
print()
print()
print()
print('\033[1;37m' + '#' + '-'*box_width + '#\033[0m')
print('\033[1;37m' + '#' + ' '*box_width + '#\033[0m')
print('\033[1;37m' + '# {:^59} #\033[0m'.format(message))
print('\033[1;37m' + '# {:^59} #\033[0m'.format('https://github.com/BerriAI/litellm/issues/new'))
print('\033[1;37m' + '#' + ' '*box_width + '#\033[0m')
print('\033[1;37m' + '#' + '-'*box_width + '#\033[0m')
print()
print(' Thank you for using LiteLLM! - Krrish & Ishaan')
print()
print()
generate_feedback_box()
print()
print("\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m")
print()
print("\033[1;34mDocs: https://docs.litellm.ai/docs/proxy_server\033[0m")
print()
print()
import litellm
from fastapi import FastAPI, Request
@ -102,11 +104,14 @@ config_dir = os.getcwd()
config_dir = appdirs.user_config_dir("litellm")
user_config_path = os.path.join(config_dir, config_filename)
log_file = 'api_log.json'
#### HELPER FUNCTIONS ####
def print_verbose(print_statement):
global user_debug
if user_debug:
print(print_statement)
global user_debug
if user_debug:
print(print_statement)
def find_avatar_url(role):
role = role.replace(" ", "%20")
@ -114,18 +119,21 @@ def find_avatar_url(role):
avatar_url = f"/static/{avatar_filename}"
return avatar_url
def usage_telemetry(feature: str): # helps us know if people are using this feature. Set `litellm --telemetry False` to your cli call to turn this off
if user_telemetry:
def usage_telemetry(
feature: str): # helps us know if people are using this feature. Set `litellm --telemetry False` to your cli call to turn this off
if user_telemetry:
data = {
"feature": feature # "local_proxy_server"
"feature": feature # "local_proxy_server"
}
threading.Thread(target=litellm.utils.litellm_telemetry, args=(data,), daemon=True).start()
def add_keys_to_config(key, value):
# Check if file exists
if os.path.exists(user_config_path):
# Load existing file
with open(user_config_path, "rb") as f:
with open(user_config_path, "rb") as f:
config = tomllib.load(f)
else:
# File doesn't exist, create empty config
@ -138,21 +146,22 @@ def add_keys_to_config(key, value):
with open(user_config_path, 'wb') as f:
tomli_w.dump(config, f)
def save_params_to_config(data: dict):
def save_params_to_config(data: dict):
# Check if file exists
if os.path.exists(user_config_path):
# Load existing file
with open(user_config_path, "rb") as f:
with open(user_config_path, "rb") as f:
config = tomllib.load(f)
else:
# File doesn't exist, create empty config
config = {}
config.setdefault('general', {})
## general config
general_settings = data["general"]
for key, value in general_settings.items():
config["general"][key] = value
@ -170,107 +179,102 @@ def save_params_to_config(data: dict):
with open(user_config_path, 'wb') as f:
tomli_w.dump(config, f)
def print_cost_logs():
with open('costs.json', 'r') as f:
# print this in green
print("\033[1;32m")
print(f.read())
print("\033[0m")
return
def load_config():
try:
global user_config, user_api_base, user_max_tokens, user_temperature, user_model
# As the .env file is typically much simpler in structure, we use load_dotenv here directly
with open(user_config_path, "rb") as f:
user_config = tomllib.load(f)
global user_config, user_api_base, user_max_tokens, user_temperature, user_model
# As the .env file is typically much simpler in structure, we use load_dotenv here directly
with open(user_config_path, "rb") as f:
user_config = tomllib.load(f)
## load keys
if "keys" in user_config:
for key in user_config["keys"]:
os.environ[key] = user_config["keys"][key] # litellm can read keys from the environment
## settings
if "general" in user_config:
litellm.add_function_to_prompt = user_config["general"].get("add_function_to_prompt", True) # by default add function to prompt if unsupported by provider
litellm.drop_params = user_config["general"].get("drop_params", True) # by default drop params if unsupported by provider
litellm.model_fallbacks = user_config["general"].get("fallbacks", None) # fallback models in case initial completion call fails
default_model = user_config["general"].get("default_model", None) # route all requests to this model.
## load keys
if "keys" in user_config:
for key in user_config["keys"]:
os.environ[key] = user_config["keys"][key] # litellm can read keys from the environment
## settings
if "general" in user_config:
litellm.add_function_to_prompt = user_config["general"].get("add_function_to_prompt",
True) # by default add function to prompt if unsupported by provider
litellm.drop_params = user_config["general"].get("drop_params",
True) # by default drop params if unsupported by provider
litellm.model_fallbacks = user_config["general"].get("fallbacks",
None) # fallback models in case initial completion call fails
default_model = user_config["general"].get("default_model", None) # route all requests to this model.
if user_model is None: # `litellm --model <model-name>`` > default_model.
user_model = default_model
if user_model is None: # `litellm --model <model-name>`` > default_model.
user_model = default_model
## load model config - to set this run `litellm --config`
model_config = None
if "model" in user_config:
if user_model in user_config["model"]:
model_config = user_config["model"][user_model]
print_verbose(f"user_config: {user_config}")
print_verbose(f"model_config: {model_config}")
print_verbose(f"user_model: {user_model}")
if model_config is None:
return
## load model config - to set this run `litellm --config`
model_config = None
if "model" in user_config:
if user_model in user_config["model"]:
model_config = user_config["model"][user_model]
user_max_tokens = model_config.get("max_tokens", None)
user_temperature = model_config.get("temperature", None)
user_api_base = model_config.get("api_base", None)
## custom prompt template
if "prompt_template" in model_config:
model_prompt_template = model_config["prompt_template"]
if len(model_prompt_template.keys()) > 0: # if user has initialized this at all
litellm.register_prompt_template(
model=user_model,
initial_prompt_value=model_prompt_template.get("MODEL_PRE_PROMPT", ""),
roles={
"system": {
"pre_message": model_prompt_template.get("MODEL_SYSTEM_MESSAGE_START_TOKEN", ""),
"post_message": model_prompt_template.get("MODEL_SYSTEM_MESSAGE_END_TOKEN", ""),
},
"user": {
"pre_message": model_prompt_template.get("MODEL_USER_MESSAGE_START_TOKEN", ""),
"post_message": model_prompt_template.get("MODEL_USER_MESSAGE_END_TOKEN", ""),
},
"assistant": {
"pre_message": model_prompt_template.get("MODEL_ASSISTANT_MESSAGE_START_TOKEN", ""),
"post_message": model_prompt_template.get("MODEL_ASSISTANT_MESSAGE_END_TOKEN", ""),
}
},
final_prompt_value=model_prompt_template.get("MODEL_POST_PROMPT", ""),
)
except Exception as e:
pass
print_verbose(f"user_config: {user_config}")
print_verbose(f"model_config: {model_config}")
print_verbose(f"user_model: {user_model}")
if model_config is None:
return
def initialize(model, alias, api_base, debug, temperature, max_tokens, max_budget, telemetry, drop_params, add_function_to_prompt, headers, save):
user_max_tokens = model_config.get("max_tokens", None)
user_temperature = model_config.get("temperature", None)
user_api_base = model_config.get("api_base", None)
## custom prompt template
if "prompt_template" in model_config:
model_prompt_template = model_config["prompt_template"]
if len(model_prompt_template.keys()) > 0: # if user has initialized this at all
litellm.register_prompt_template(
model=user_model,
initial_prompt_value=model_prompt_template.get("MODEL_PRE_PROMPT", ""),
roles={
"system": {
"pre_message": model_prompt_template.get("MODEL_SYSTEM_MESSAGE_START_TOKEN", ""),
"post_message": model_prompt_template.get("MODEL_SYSTEM_MESSAGE_END_TOKEN", ""),
},
"user": {
"pre_message": model_prompt_template.get("MODEL_USER_MESSAGE_START_TOKEN", ""),
"post_message": model_prompt_template.get("MODEL_USER_MESSAGE_END_TOKEN", ""),
},
"assistant": {
"pre_message": model_prompt_template.get("MODEL_ASSISTANT_MESSAGE_START_TOKEN", ""),
"post_message": model_prompt_template.get("MODEL_ASSISTANT_MESSAGE_END_TOKEN", ""),
}
},
final_prompt_value=model_prompt_template.get("MODEL_POST_PROMPT", ""),
)
def initialize(model, alias, api_base, debug, temperature, max_tokens, max_budget, telemetry, drop_params,
add_function_to_prompt, headers, save):
global user_model, user_api_base, user_debug, user_max_tokens, user_temperature, user_telemetry, user_headers
user_model = model
user_debug = debug
load_config()
dynamic_config = {"general": {}, user_model: {}}
if headers: # model-specific param
dynamic_config = {"general": {}, user_model: {}}
if headers: # model-specific param
user_headers = headers
dynamic_config[user_model]["headers"] = headers
if api_base: # model-specific param
if api_base: # model-specific param
user_api_base = api_base
dynamic_config[user_model]["api_base"] = api_base
if max_tokens: # model-specific param
if max_tokens: # model-specific param
user_max_tokens = max_tokens
dynamic_config[user_model]["max_tokens"] = max_tokens
if temperature: # model-specific param
if temperature: # model-specific param
user_temperature = temperature
dynamic_config[user_model]["temperature"] = temperature
if alias: # model-specific param
if alias: # model-specific param
dynamic_config[user_model]["alias"] = alias
if drop_params == True: # litellm-specific param
if drop_params == True: # litellm-specific param
litellm.drop_params = True
dynamic_config["general"]["drop_params"] = True
if add_function_to_prompt == True: # litellm-specific param
if add_function_to_prompt == True: # litellm-specific param
litellm.add_function_to_prompt = True
dynamic_config["general"]["add_function_to_prompt"] = True
if max_budget: # litellm-specific param
if max_budget: # litellm-specific param
litellm.max_budget = max_budget
dynamic_config["general"]["max_budget"] = max_budget
if save:
if save:
save_params_to_config(dynamic_config)
with open(user_config_path) as f:
print(f.read())
@ -278,6 +282,7 @@ def initialize(model, alias, api_base, debug, temperature, max_tokens, max_budge
user_telemetry = telemetry
usage_telemetry(feature="local_proxy_server")
def deploy_proxy(model, api_base, debug, temperature, max_tokens, telemetry, deploy):
import requests
# Load .env file
@ -308,8 +313,6 @@ def deploy_proxy(model, api_base, debug, temperature, max_tokens, telemetry, dep
files = {"file": open(".env", "rb")}
# print(files)
response = requests.post(url, data=data, files=files)
# print(response)
# Check the status of the request
@ -324,10 +327,11 @@ def deploy_proxy(model, api_base, debug, temperature, max_tokens, telemetry, dep
return url
def track_cost_callback(
kwargs, # kwargs to completion
completion_response, # response from completion
start_time, end_time # start/end time
kwargs, # kwargs to completion
completion_response, # response from completion
start_time, end_time # start/end time
):
# track cost like this
# {
@ -345,12 +349,12 @@ def track_cost_callback(
# for streaming responses
if "complete_streaming_response" in kwargs:
# for tracking streaming cost we pass the "messages" and the output_text to litellm.completion_cost
completion_response=kwargs["complete_streaming_response"]
completion_response = kwargs["complete_streaming_response"]
input_text = kwargs["messages"]
output_text = completion_response["choices"][0]["message"]["content"]
response_cost = litellm.completion_cost(
model = kwargs["model"],
messages = input_text,
model=kwargs["model"],
messages=input_text,
completion=output_text
)
model = kwargs['model']
@ -368,7 +372,7 @@ def track_cost_callback(
with open("costs.json") as f:
cost_data = json.load(f)
except FileNotFoundError:
cost_data = {}
cost_data = {}
import datetime
date = datetime.datetime.now().strftime("%b-%d-%Y")
if date not in cost_data:
@ -389,47 +393,32 @@ def track_cost_callback(
except:
pass
def logger(
kwargs, # kwargs to completion
completion_response=None, # response from completion
start_time=None,
end_time=None # start/end time
):
log_event_type = kwargs['log_event_type']
try:
if log_event_type == 'pre_api_call':
inference_params = copy.deepcopy(kwargs)
timestamp = inference_params.pop('start_time')
dt_key = timestamp.strftime("%Y%m%d%H%M%S%f")[:23]
log_data = {
dt_key: {
'pre_api_call': inference_params
}
}
try:
with open(log_file, 'r') as f:
existing_data = json.load(f)
except FileNotFoundError:
existing_data = {}
existing_data.update(log_data)
def write_to_log():
with open(log_file, 'w') as f:
json.dump(existing_data, f, indent=2)
thread = threading.Thread(target=write_to_log, daemon=True)
thread.start()
elif log_event_type == 'post_api_call':
if "stream" not in kwargs["optional_params"] or kwargs["optional_params"]["stream"] is False or kwargs.get("complete_streaming_response", False):
def logger(
kwargs, # kwargs to completion
completion_response=None, # response from completion
start_time=None,
end_time=None # start/end time
):
log_event_type = kwargs['log_event_type']
try:
if log_event_type == 'pre_api_call':
inference_params = copy.deepcopy(kwargs)
timestamp = inference_params.pop('start_time')
dt_key = timestamp.strftime("%Y%m%d%H%M%S%f")[:23]
with open(log_file, 'r') as f:
existing_data = json.load(f)
existing_data[dt_key]['post_api_call'] = inference_params
log_data = {
dt_key: {
'pre_api_call': inference_params
}
}
try:
with open(log_file, 'r') as f:
existing_data = json.load(f)
except FileNotFoundError:
existing_data = {}
existing_data.update(log_data)
def write_to_log():
with open(log_file, 'w') as f:
@ -437,15 +426,35 @@ def logger(
thread = threading.Thread(target=write_to_log, daemon=True)
thread.start()
except:
pass
elif log_event_type == 'post_api_call':
if "stream" not in kwargs["optional_params"] or kwargs["optional_params"]["stream"] is False or kwargs.get(
"complete_streaming_response", False):
inference_params = copy.deepcopy(kwargs)
timestamp = inference_params.pop('start_time')
dt_key = timestamp.strftime("%Y%m%d%H%M%S%f")[:23]
with open(log_file, 'r') as f:
existing_data = json.load(f)
existing_data[dt_key]['post_api_call'] = inference_params
def write_to_log():
with open(log_file, 'w') as f:
json.dump(existing_data, f, indent=2)
thread = threading.Thread(target=write_to_log, daemon=True)
thread.start()
except:
pass
litellm.input_callback = [logger]
litellm.success_callback = [logger]
litellm.failure_callback = [logger]
#### API ENDPOINTS ####
@router.get("/models") # if project requires model list
@router.get("/models") # if project requires model list
def model_list():
if user_model != None:
return dict(
@ -455,57 +464,49 @@ def model_list():
else:
all_models = litellm.utils.get_valid_models()
return dict(
data = [{"id": model, "object": "model", "created": 1677610602, "owned_by": "openai"} for model in all_models],
data=[{"id": model, "object": "model", "created": 1677610602, "owned_by": "openai"} for model in
all_models],
object="list",
)
@router.post("/v1/completions")
@router.post("/completions")
async def completion(request: Request):
data = await request.json()
return litellm_completion(data=data, type="completion", user_model=user_model, user_temperature=user_temperature, user_max_tokens=user_max_tokens, user_api_base=user_api_base, user_headers=user_headers, user_debug=user_debug)
return litellm_completion(data=data, type="completion", user_model=user_model, user_temperature=user_temperature,
user_max_tokens=user_max_tokens, user_api_base=user_api_base, user_headers=user_headers,
user_debug=user_debug)
@router.post("/v1/chat/completions")
@router.post("/chat/completions")
async def chat_completion(request: Request):
data = await request.json()
response = litellm_completion(data, type="chat_completion", user_model=user_model, user_temperature=user_temperature, user_max_tokens=user_max_tokens, user_api_base=user_api_base, user_headers=user_headers, user_debug=user_debug)
return response
# V1 Endpoints - some apps expect a v1 endpoint - these call the regular function
@router.post("/v1/completions")
async def v1_completion(request: Request):
data = await request.json()
return litellm_completion(data=data, type="completion")
@router.post("/v1/chat/completions")
async def v1_chat_completion(request: Request):
data = await request.json()
print_verbose(f"data passed in: {data}")
response = litellm_completion(data, type="chat_completion")
return response
return litellm_completion(data, type="chat_completion", user_model=user_model,
user_temperature=user_temperature, user_max_tokens=user_max_tokens,
user_api_base=user_api_base, user_headers=user_headers, user_debug=user_debug)
@router.post("/send_message")
async def send_message(request: Request):
try:
data = await request.json()
role = data.get("role")
text = data.get("text")
avatarUrl = find_avatar_url(role)
def print_cost_logs():
with open('costs.json', 'r') as f:
# print this in green
print("\033[1;32m")
print(f.read())
print("\033[0m")
return
message = {"role": role, "text": text, "avatarUrl": avatarUrl}
messages.append(message)
return jsonable_encoder(messages)
except:
return "An error occurred", 500
@router.get("/ollama_logs")
async def retrieve_server_log(request: Request):
filepath = os.path.expanduser('~/.ollama/logs/server.log')
return FileResponse(filepath)
@router.get("/")
async def home(request: Request):
return "LiteLLM: RUNNING"
app.include_router(router)
app.include_router(router)

3
litellm/proxy/start.sh
View file

@ -1,2 +1,3 @@
python3 proxy_cli.py --config -f ../../secrets_template.toml &
#!/bin/bash
python3 proxy_cli.py --config -f ../../secrets_template.toml
python3 proxy_cli.py

9
requirements.txt
View file

@ -6,4 +6,11 @@ tokenizers
click
jinja2
certifi
appdirs
appdirs
backoff
boto3
uvicorn
fastapi
tomli
appdirs
tomli-w

4
secrets_template.toml
View file

@ -9,8 +9,8 @@
# AWS_SECRET_ACCESS_KEY = "" # Uncomment to save your Bedrock/Sagemaker access keys
[general]
# add_function_to_prompt = True # e.g: Ollama doesn't support functions, so add it to the prompt instead
# drop_params = True # drop any params not supported by the provider (e.g. Ollama)
# add_function_to_prompt = true # e.g: Ollama doesn't support functions, so add it to the prompt instead
# drop_params = true # drop any params not supported by the provider (e.g. Ollama)
# default_model = None # route all requests to this model
# fallbacks = ["gpt-3.5-turbo", "gpt-4"] # models you want to fallback to in case completion call fails (remember: add relevant keys)