From a9ebf1b6abcd2c1470184d766c6ccbf928a288bf Mon Sep 17 00:00:00 2001 From: coconut49 Date: Wed, 18 Oct 2023 13:28:07 +0800 Subject: [PATCH 01/10] Relocate Dockerfile to litellm/proxy directory. --- Dockerfile => litellm/proxy/Dockerfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Dockerfile => litellm/proxy/Dockerfile (100%) diff --git a/Dockerfile b/litellm/proxy/Dockerfile similarity index 100% rename from Dockerfile rename to litellm/proxy/Dockerfile From 52fdfe58195a514ca718df7760969cee610f82c7 Mon Sep 17 00:00:00 2001 From: coconut49 Date: Wed, 18 Oct 2023 13:30:53 +0800 Subject: [PATCH 02/10] Improve code formatting and allow configurable litellm config path via environment variable. --- litellm/proxy/proxy_cli.py | 2 +- litellm/proxy/proxy_server.py | 28 +++++++++++++++------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index f2b29bf5f36..96e089caaa3 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -9,7 +9,7 @@ import operator config_filename = "litellm.secrets.toml" # Using appdirs to determine user-specific config path config_dir = appdirs.user_config_dir("litellm") -user_config_path = os.path.join(config_dir, config_filename) +user_config_path = os.getenv("LITELLM_CONFIG_PATH", os.path.join(config_dir, config_filename)) load_dotenv() from importlib import resources diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5736465840e..0afa6be1f62 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -18,7 +18,8 @@ except ImportError: import subprocess import sys - subprocess.check_call([sys.executable, "-m", "pip", "install", "uvicorn", "fastapi", "tomli", "appdirs", "tomli-w", "backoff"]) + subprocess.check_call( + [sys.executable, "-m", "pip", "install", "uvicorn", "fastapi", "tomli", "appdirs", "tomli-w", "backoff"]) import uvicorn import fastapi import tomli as tomllib @@ -26,9 +27,9 @@ except ImportError: import tomli_w try: - from .llm import litellm_completion + from .llm import litellm_completion except ImportError as e: - from llm import litellm_completion # type: ignore + from llm import litellm_completion # type: ignore import random @@ -105,7 +106,7 @@ model_router = litellm.Router() config_filename = "litellm.secrets.toml" config_dir = os.getcwd() config_dir = appdirs.user_config_dir("litellm") -user_config_path = os.path.join(config_dir, config_filename) +user_config_path = os.getenv("LITELLM_CONFIG_PATH", os.path.join(config_dir, config_filename)) log_file = 'api_log.json' @@ -184,7 +185,7 @@ def save_params_to_config(data: dict): def load_config(): - try: + try: global user_config, user_api_base, user_max_tokens, user_temperature, user_model, local_logging # As the .env file is typically much simpler in structure, we use load_dotenv here directly with open(user_config_path, "rb") as f: @@ -199,9 +200,9 @@ def load_config(): litellm.add_function_to_prompt = user_config["general"].get("add_function_to_prompt", True) # by default add function to prompt if unsupported by provider litellm.drop_params = user_config["general"].get("drop_params", - True) # by default drop params if unsupported by provider + True) # by default drop params if unsupported by provider litellm.model_fallbacks = user_config["general"].get("fallbacks", - None) # fallback models in case initial completion call fails + None) # fallback models in case initial completion call fails default_model = user_config["general"].get("default_model", None) # route all requests to this model. local_logging = user_config["general"].get("local_logging", True) @@ -215,10 +216,10 @@ def load_config(): if user_model in user_config["model"]: model_config = user_config["model"][user_model] model_list = [] - for model in user_config["model"]: + for model in user_config["model"]: if "model_list" in user_config["model"][model]: model_list.extend(user_config["model"][model]["model_list"]) - if len(model_list) > 0: + if len(model_list) > 0: model_router.set_model_list(model_list=model_list) print_verbose(f"user_config: {user_config}") @@ -254,7 +255,7 @@ def load_config(): }, final_prompt_value=model_prompt_template.get("MODEL_POST_PROMPT", ""), ) - except: + except: pass @@ -271,8 +272,8 @@ def initialize(model, alias, api_base, api_version, debug, temperature, max_toke if api_base: # model-specific param user_api_base = api_base dynamic_config[user_model]["api_base"] = api_base - if api_version: - os.environ["AZURE_API_VERSION"] = api_version # set this for azure - litellm can read this from the env + if api_version: + os.environ["AZURE_API_VERSION"] = api_version # set this for azure - litellm can read this from the env if max_tokens: # model-specific param user_max_tokens = max_tokens dynamic_config[user_model]["max_tokens"] = max_tokens @@ -290,7 +291,7 @@ def initialize(model, alias, api_base, api_version, debug, temperature, max_toke if max_budget: # litellm-specific param litellm.max_budget = max_budget dynamic_config["general"]["max_budget"] = max_budget - if debug: # litellm-specific param + if debug: # litellm-specific param litellm.set_verbose = True if save: save_params_to_config(dynamic_config) @@ -300,6 +301,7 @@ def initialize(model, alias, api_base, api_version, debug, temperature, max_toke user_telemetry = telemetry usage_telemetry(feature="local_proxy_server") + def track_cost_callback( kwargs, # kwargs to completion completion_response, # response from completion From be78f2ade955d08ca67ab34f4b40bf476d061965 Mon Sep 17 00:00:00 2001 From: coconut49 Date: Wed, 18 Oct 2023 13:34:00 +0800 Subject: [PATCH 03/10] Move Dockerfile to root and set environment variable for config path --- litellm/proxy/Dockerfile => Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename litellm/proxy/Dockerfile => Dockerfile (79%) diff --git a/litellm/proxy/Dockerfile b/Dockerfile similarity index 79% rename from litellm/proxy/Dockerfile rename to Dockerfile index be162d4511f..cd4f86da414 100644 --- a/litellm/proxy/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ FROM python:3.10 +ENV LITELLM_CONFIG_PATH="/litellm.secrets.toml" COPY . /app WORKDIR /app -RUN mkdir -p /root/.config/litellm/ && cp /app/secrets_template.toml /root/.config/litellm/litellm.secrets.toml RUN pip install -r requirements.txt WORKDIR /app/litellm/proxy From cad02ace21588e5f69e0cde80c5625da835e72ec Mon Sep 17 00:00:00 2001 From: coconut49 Date: Wed, 18 Oct 2023 13:50:37 +0800 Subject: [PATCH 04/10] Add GitHub Actions workflow to build and release Docker images --- .github/workflows/docker.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000000..2c54c26091d --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,35 @@ +name: Build Docker Images +on: + workflow_dispatch: + inputs: + tag: + description: "The tag version you want to build" +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + - name: Build and release Docker images + uses: docker/build-push-action@v5 + with: + platforms: linux/386,linux/amd64,linux/arm64 + tags: | + ${{ steps.tag.outputs.latest }} + ${{ steps.tag.outputs.versioned }} + push: true \ No newline at end of file From e92a68578c3ccf28499649f970b659025f7004e9 Mon Sep 17 00:00:00 2001 From: coconut49 Date: Wed, 18 Oct 2023 14:03:13 +0800 Subject: [PATCH 05/10] Add step to get build tag and include versioned tags in Docker workflow. --- .github/workflows/docker.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2c54c26091d..a76e15010ef 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,6 +25,15 @@ jobs: uses: docker/metadata-action@v5 with: images: ghcr.io/${{ github.repository }} + - name: Get tag to build + id: tag + run: | + echo "latest=ghcr.io/${{ github.repository }}:latest" >> $GITHUB_OUTPUT + if [[ -z "${{ github.event.inputs.tag }}" ]]; then + echo "versioned=ghcr.io/${{ github.repository }}:${{ github.ref_name }}" >> $GITHUB_OUTPUT + else + echo "versioned=ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}" >> $GITHUB_OUTPUT + fi - name: Build and release Docker images uses: docker/build-push-action@v5 with: @@ -32,4 +41,5 @@ jobs: tags: | ${{ steps.tag.outputs.latest }} ${{ steps.tag.outputs.versioned }} + labels: ${{ steps.meta.outputs.labels }} push: true \ No newline at end of file From c19700b0c4fd157dbcce5580c9f6b1e9f2283eed Mon Sep 17 00:00:00 2001 From: coconut49 Date: Wed, 18 Oct 2023 14:07:55 +0800 Subject: [PATCH 06/10] Enable submodules during checkout in Docker workflow --- .github/workflows/docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a76e15010ef..9f731fbed2d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -10,6 +10,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: true - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx From 3850a0ac589c7b956d777d4f10a29c7f6d46adcc Mon Sep 17 00:00:00 2001 From: coconut49 Date: Wed, 18 Oct 2023 14:10:11 +0800 Subject: [PATCH 07/10] Change submodule checkout strategy to recursive in Docker workflow --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9f731fbed2d..623eae997e3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -11,7 +11,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - submodules: true + submodules: recursive - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx From c96e45fbfb50124a2b56b43325207bf5a802a502 Mon Sep 17 00:00:00 2001 From: coconut49 Date: Wed, 18 Oct 2023 14:15:26 +0800 Subject: [PATCH 08/10] Remove submodule checkout and set Docker build context to current directory. --- .github/workflows/docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 623eae997e3..8ab915df533 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -10,8 +10,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - with: - submodules: recursive - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx @@ -39,6 +37,7 @@ jobs: - name: Build and release Docker images uses: docker/build-push-action@v5 with: + context: . platforms: linux/386,linux/amd64,linux/arm64 tags: | ${{ steps.tag.outputs.latest }} From 10737b113f9b734a9d2be6aadec17dafac0e6b4c Mon Sep 17 00:00:00 2001 From: coconut49 Date: Wed, 18 Oct 2023 14:18:31 +0800 Subject: [PATCH 09/10] Update Docker build platforms to exclude linux/386 --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8ab915df533..616ba5c9f9f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -38,7 +38,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . - platforms: linux/386,linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64 tags: | ${{ steps.tag.outputs.latest }} ${{ steps.tag.outputs.versioned }} From f890aa1db5142bad8bfbb40550948893f52778b6 Mon Sep 17 00:00:00 2001 From: coconut49 Date: Wed, 18 Oct 2023 14:31:43 +0800 Subject: [PATCH 10/10] Refactor code for better readability and remove unnecessary comments in Dockerfile. --- Dockerfile | 5 +- litellm/proxy/proxy_server.py | 229 ++++++++++++++++++++++------------ 2 files changed, 151 insertions(+), 83 deletions(-) diff --git a/Dockerfile b/Dockerfile index cd4f86da414..42b223b1fc2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,4 @@ RUN pip install -r requirements.txt WORKDIR /app/litellm/proxy EXPOSE 8000 -ENTRYPOINT [ "python3", "proxy_cli.py" ] -# TODO - Set up a GitHub Action to automatically create the Docker image, -# and then we can quickly deploy the litellm proxy in the following way -# `docker run -p 8000:8000 -v ./secrets_template.toml:/root/.config/litellm/litellm.secrets.toml ghcr.io/BerriAI/litellm:v0.8.4` \ No newline at end of file +ENTRYPOINT [ "python3", "proxy_cli.py" ] \ No newline at end of file diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0afa6be1f62..f82177418c2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -19,7 +19,19 @@ except ImportError: import sys subprocess.check_call( - [sys.executable, "-m", "pip", "install", "uvicorn", "fastapi", "tomli", "appdirs", "tomli-w", "backoff"]) + [ + sys.executable, + "-m", + "pip", + "install", + "uvicorn", + "fastapi", + "tomli", + "appdirs", + "tomli-w", + "backoff", + ] + ) import uvicorn import fastapi import tomli as tomllib @@ -52,14 +64,17 @@ def generate_feedback_box(): message = random.choice(list_of_messages) print() - print('\033[1;37m' + '#' + '-' * box_width + '#\033[0m') - print('\033[1;37m' + '#' + ' ' * box_width + '#\033[0m') - print('\033[1;37m' + '# {:^59} #\033[0m'.format(message)) - print('\033[1;37m' + '# {:^59} #\033[0m'.format('https://github.com/BerriAI/litellm/issues/new')) - print('\033[1;37m' + '#' + ' ' * box_width + '#\033[0m') - print('\033[1;37m' + '#' + '-' * box_width + '#\033[0m') + print("\033[1;37m" + "#" + "-" * box_width + "#\033[0m") + print("\033[1;37m" + "#" + " " * box_width + "#\033[0m") + print("\033[1;37m" + "# {:^59} #\033[0m".format(message)) + print( + "\033[1;37m" + + "# {:^59} #\033[0m".format("https://github.com/BerriAI/litellm/issues/new") + ) + print("\033[1;37m" + "#" + " " * box_width + "#\033[0m") + print("\033[1;37m" + "#" + "-" * box_width + "#\033[0m") print() - print(' Thank you for using LiteLLM! - Krrish & Ishaan') + print(" Thank you for using LiteLLM! - Krrish & Ishaan") print() print() @@ -67,7 +82,9 @@ def generate_feedback_box(): generate_feedback_box() print() -print("\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m") +print( + "\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m" +) print() print("\033[1;34mDocs: https://docs.litellm.ai/docs/proxy_server\033[0m") print() @@ -106,8 +123,10 @@ model_router = litellm.Router() config_filename = "litellm.secrets.toml" config_dir = os.getcwd() config_dir = appdirs.user_config_dir("litellm") -user_config_path = os.getenv("LITELLM_CONFIG_PATH", os.path.join(config_dir, config_filename)) -log_file = 'api_log.json' +user_config_path = os.getenv( + "LITELLM_CONFIG_PATH", os.path.join(config_dir, config_filename) +) +log_file = "api_log.json" #### HELPER FUNCTIONS #### @@ -125,12 +144,13 @@ def find_avatar_url(role): def usage_telemetry( - feature: str): # helps us know if people are using this feature. Set `litellm --telemetry False` to your cli call to turn this off + feature: str, +): # helps us know if people are using this feature. Set `litellm --telemetry False` to your cli call to turn this off if user_telemetry: - data = { - "feature": feature # "local_proxy_server" - } - threading.Thread(target=litellm.utils.litellm_telemetry, args=(data,), daemon=True).start() + data = {"feature": feature} # "local_proxy_server" + threading.Thread( + target=litellm.utils.litellm_telemetry, args=(data,), daemon=True + ).start() def add_keys_to_config(key, value): @@ -143,11 +163,11 @@ def add_keys_to_config(key, value): # File doesn't exist, create empty config config = {} - # Add new key - config.setdefault('keys', {})[key] = value + # Add new key + config.setdefault("keys", {})[key] = value - # Write config to file - with open(user_config_path, 'wb') as f: + # Write config to file + with open(user_config_path, "wb") as f: tomli_w.dump(config, f) @@ -161,15 +181,15 @@ def save_params_to_config(data: dict): # File doesn't exist, create empty config config = {} - config.setdefault('general', {}) + config.setdefault("general", {}) - ## general config + ## general config general_settings = data["general"] for key, value in general_settings.items(): config["general"][key] = value - ## model-specific config + ## model-specific config config.setdefault("model", {}) config["model"].setdefault(user_model, {}) @@ -179,8 +199,8 @@ def save_params_to_config(data: dict): for key, value in user_model_config.items(): config["model"][model_key][key] = value - # Write config to file - with open(user_config_path, 'wb') as f: + # Write config to file + with open(user_config_path, "wb") as f: tomli_w.dump(config, f) @@ -194,16 +214,23 @@ def load_config(): ## load keys if "keys" in user_config: for key in user_config["keys"]: - os.environ[key] = user_config["keys"][key] # litellm can read keys from the environment + os.environ[key] = user_config["keys"][ + key + ] # litellm can read keys from the environment ## settings if "general" in user_config: - litellm.add_function_to_prompt = user_config["general"].get("add_function_to_prompt", - True) # by default add function to prompt if unsupported by provider - litellm.drop_params = user_config["general"].get("drop_params", - True) # by default drop params if unsupported by provider - litellm.model_fallbacks = user_config["general"].get("fallbacks", - None) # fallback models in case initial completion call fails - default_model = user_config["general"].get("default_model", None) # route all requests to this model. + litellm.add_function_to_prompt = user_config["general"].get( + "add_function_to_prompt", True + ) # by default add function to prompt if unsupported by provider + litellm.drop_params = user_config["general"].get( + "drop_params", True + ) # by default drop params if unsupported by provider + litellm.model_fallbacks = user_config["general"].get( + "fallbacks", None + ) # fallback models in case initial completion call fails + default_model = user_config["general"].get( + "default_model", None + ) # route all requests to this model. local_logging = user_config["general"].get("local_logging", True) @@ -235,32 +262,63 @@ def load_config(): ## custom prompt template if "prompt_template" in model_config: model_prompt_template = model_config["prompt_template"] - if len(model_prompt_template.keys()) > 0: # if user has initialized this at all + if ( + len(model_prompt_template.keys()) > 0 + ): # if user has initialized this at all litellm.register_prompt_template( model=user_model, - initial_prompt_value=model_prompt_template.get("MODEL_PRE_PROMPT", ""), + initial_prompt_value=model_prompt_template.get( + "MODEL_PRE_PROMPT", "" + ), roles={ "system": { - "pre_message": model_prompt_template.get("MODEL_SYSTEM_MESSAGE_START_TOKEN", ""), - "post_message": model_prompt_template.get("MODEL_SYSTEM_MESSAGE_END_TOKEN", ""), + "pre_message": model_prompt_template.get( + "MODEL_SYSTEM_MESSAGE_START_TOKEN", "" + ), + "post_message": model_prompt_template.get( + "MODEL_SYSTEM_MESSAGE_END_TOKEN", "" + ), }, "user": { - "pre_message": model_prompt_template.get("MODEL_USER_MESSAGE_START_TOKEN", ""), - "post_message": model_prompt_template.get("MODEL_USER_MESSAGE_END_TOKEN", ""), + "pre_message": model_prompt_template.get( + "MODEL_USER_MESSAGE_START_TOKEN", "" + ), + "post_message": model_prompt_template.get( + "MODEL_USER_MESSAGE_END_TOKEN", "" + ), }, "assistant": { - "pre_message": model_prompt_template.get("MODEL_ASSISTANT_MESSAGE_START_TOKEN", ""), - "post_message": model_prompt_template.get("MODEL_ASSISTANT_MESSAGE_END_TOKEN", ""), - } + "pre_message": model_prompt_template.get( + "MODEL_ASSISTANT_MESSAGE_START_TOKEN", "" + ), + "post_message": model_prompt_template.get( + "MODEL_ASSISTANT_MESSAGE_END_TOKEN", "" + ), + }, }, - final_prompt_value=model_prompt_template.get("MODEL_POST_PROMPT", ""), + final_prompt_value=model_prompt_template.get( + "MODEL_POST_PROMPT", "" + ), ) except: pass -def initialize(model, alias, api_base, api_version, debug, temperature, max_tokens, max_budget, telemetry, drop_params, - add_function_to_prompt, headers, save): +def initialize( + model, + alias, + api_base, + api_version, + debug, + temperature, + max_tokens, + max_budget, + telemetry, + drop_params, + add_function_to_prompt, + headers, + save, +): global user_model, user_api_base, user_debug, user_max_tokens, user_temperature, user_telemetry, user_headers user_model = model user_debug = debug @@ -273,7 +331,9 @@ def initialize(model, alias, api_base, api_version, debug, temperature, max_toke user_api_base = api_base dynamic_config[user_model]["api_base"] = api_base if api_version: - os.environ["AZURE_API_VERSION"] = api_version # set this for azure - litellm can read this from the env + os.environ[ + "AZURE_API_VERSION" + ] = api_version # set this for azure - litellm can read this from the env if max_tokens: # model-specific param user_max_tokens = max_tokens dynamic_config[user_model]["max_tokens"] = max_tokens @@ -303,15 +363,16 @@ def initialize(model, alias, api_base, api_version, debug, temperature, max_toke def track_cost_callback( - kwargs, # kwargs to completion - completion_response, # response from completion - start_time, end_time # start/end time + kwargs, # kwargs to completion + completion_response, # response from completion + start_time, + end_time, # start/end time ): - # track cost like this + # track cost like this # { # "Oct12": { # "gpt-4": 10, - # "claude-2": 12.01, + # "claude-2": 12.01, # }, # "Oct 15": { # "ollama/llama2": 0.0, @@ -319,28 +380,27 @@ def track_cost_callback( # } # } try: - # for streaming responses if "complete_streaming_response" in kwargs: - # for tracking streaming cost we pass the "messages" and the output_text to litellm.completion_cost + # for tracking streaming cost we pass the "messages" and the output_text to litellm.completion_cost completion_response = kwargs["complete_streaming_response"] input_text = kwargs["messages"] output_text = completion_response["choices"][0]["message"]["content"] response_cost = litellm.completion_cost( - model=kwargs["model"], - messages=input_text, - completion=output_text + model=kwargs["model"], messages=input_text, completion=output_text ) - model = kwargs['model'] + model = kwargs["model"] # for non streaming responses else: # we pass the completion_response obj if kwargs["stream"] != True: - response_cost = litellm.completion_cost(completion_response=completion_response) + response_cost = litellm.completion_cost( + completion_response=completion_response + ) model = completion_response["model"] - # read/write from json for storing daily model costs + # read/write from json for storing daily model costs cost_data = {} try: with open("costs.json") as f: @@ -348,6 +408,7 @@ def track_cost_callback( except FileNotFoundError: cost_data = {} import datetime + date = datetime.datetime.now().strftime("%b-%d-%Y") if date not in cost_data: cost_data[date] = {} @@ -358,7 +419,7 @@ def track_cost_callback( else: cost_data[date][kwargs["model"]] = { "cost": response_cost, - "num_requests": 1 + "num_requests": 1, } with open("costs.json", "w") as f: @@ -369,25 +430,21 @@ def track_cost_callback( def logger( - kwargs, # kwargs to completion - completion_response=None, # response from completion - start_time=None, - end_time=None # start/end time + kwargs, # kwargs to completion + completion_response=None, # response from completion + start_time=None, + end_time=None, # start/end time ): - log_event_type = kwargs['log_event_type'] + log_event_type = kwargs["log_event_type"] try: - if log_event_type == 'pre_api_call': + if log_event_type == "pre_api_call": inference_params = copy.deepcopy(kwargs) - timestamp = inference_params.pop('start_time') + timestamp = inference_params.pop("start_time") dt_key = timestamp.strftime("%Y%m%d%H%M%S%f")[:23] - log_data = { - dt_key: { - 'pre_api_call': inference_params - } - } + log_data = {dt_key: {"pre_api_call": inference_params}} try: - with open(log_file, 'r') as f: + with open(log_file, "r") as f: existing_data = json.load(f) except FileNotFoundError: existing_data = {} @@ -395,7 +452,7 @@ def logger( existing_data.update(log_data) def write_to_log(): - with open(log_file, 'w') as f: + with open(log_file, "w") as f: json.dump(existing_data, f, indent=2) thread = threading.Thread(target=write_to_log, daemon=True) @@ -415,14 +472,28 @@ litellm.failure_callback = [logger] def model_list(): if user_model != None: return dict( - data=[{"id": user_model, "object": "model", "created": 1677610602, "owned_by": "openai"}], + data=[ + { + "id": user_model, + "object": "model", + "created": 1677610602, + "owned_by": "openai", + } + ], object="list", ) else: all_models = litellm.utils.get_valid_models() return dict( - data=[{"id": model, "object": "model", "created": 1677610602, "owned_by": "openai"} for model in - all_models], + data=[ + { + "id": model, + "object": "model", + "created": 1677610602, + "owned_by": "openai", + } + for model in all_models + ], object="list", ) @@ -447,7 +518,7 @@ async def chat_completion(request: Request): def print_cost_logs(): - with open('costs.json', 'r') as f: + with open("costs.json", "r") as f: # print this in green print("\033[1;32m") print(f.read()) @@ -457,7 +528,7 @@ def print_cost_logs(): @router.get("/ollama_logs") async def retrieve_server_log(request: Request): - filepath = os.path.expanduser('~/.ollama/logs/server.log') + filepath = os.path.expanduser("~/.ollama/logs/server.log") return FileResponse(filepath)