Merge pull request #1402 from BerriAI/litellm_dockerfile_testing

Litellm dockerfile testing

.circleci/config.yml

@@ -40,6 +40,7 @@ jobs:
             pip install "httpx==0.24.1"
             pip install "anyio==3.7.1"
             pip install "asyncio==3.4.3"
+            pip install "PyGithub==1.59.1"
       - save_cache:
           paths:
             - ./venv
@@ -83,6 +84,103 @@ jobs:
       - store_test_results:
           path: test-results
 
+  build_and_test:
+    machine:
+      image: ubuntu-2204:2023.10.1
+    working_directory: ~/project
+    steps:
+      - checkout
+      - run:
+          name: Install Docker CLI (In case it's not already installed)
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+      - run:
+          name: Build Docker image
+          command: docker build -t my-app:latest -f Dockerfile.database .
+      - run:
+          name: Run Docker container
+          command: |
+            docker run -d \
+              -p 4000:4000 \
+              -e DATABASE_URL=$PROXY_DOCKER_DB_URL \
+              -e AZURE_API_KEY=$AZURE_FRANCE_API_KEY \
+              -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \
+              -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \
+              --name my-app \
+              -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \
+              my-app:latest \
+              --config /app/config.yaml \
+              --port 4000 \
+              --num_workers 8
+      - run:
+          name: Install curl and dockerize
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y curl
+            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
+      - run:
+          name: Start outputting logs
+          command: |
+            while true; do
+              docker logs my-app
+              sleep 10
+            done
+          background: true
+      - run: 
+          name: Wait for app to be ready
+          command: dockerize -wait http://localhost:4000 -timeout 1m
+      - run:
+          name: Test the application
+          command: |
+            mkdir -p /tmp/responses
+            for i in {1..10}; do
+              status_file="/tmp/responses/status_${i}.txt"
+              response_file="/tmp/responses/response_${i}.json"
+
+              (curl --location --request POST 'http://0.0.0.0:4000/key/generate' \
+                --header 'Authorization: Bearer sk-1234' \
+                --header 'Content-Type: application/json' \
+                --data '{"models": ["azure-models"], "aliases": {"mistral-7b": "gpt-3.5-turbo"}, "duration": null}' \
+                --silent --output "${response_file}" --write-out '%{http_code}' > "${status_file}") &
+
+              # Capture PIDs of background processes
+              pids[${i}]=$!
+            done
+
+            # Wait for all background processes to finish
+            for pid in ${pids[*]}; do
+              wait $pid
+            done
+
+            # Check all responses and status codes
+            fail=false
+            for i in {1..10}; do
+              status=$(cat "/tmp/responses/status_${i}.txt")
+              
+              # Here, we need to set the correct response file path for each iteration
+              response_file="/tmp/responses/response_${i}.json" # This was missing in the provided script
+              
+              response=$(cat "${response_file}")
+              echo "Response ${i} (Status code: ${status}):"
+              echo "${response}" # Use echo here to print the contents
+              echo # Additional newline for readability
+
+              if [ "$status" -ne 200 ]; then
+                echo "A request did not return a 200 status code: $status"
+                fail=true
+              fi
+            done
+
+            # If any request did not return status code 200, fail the job
+            if [ "$fail" = true ]; then
+              exit 1
+            fi
+
+            echo "All requests returned a 200 status code."
+
   publish_to_pypi:
     docker:
       - image: cimg/python:3.8
@@ -164,9 +262,16 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - build_and_test:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
       - publish_to_pypi:
           requires:
             - local_testing
+            - build_and_test
           filters:
             branches:
               only:

Dockerfile.database

@@ -53,8 +53,7 @@ RUN chmod +x entrypoint.sh
 
 EXPOSE 4000/tcp
 
-# Set your entrypoint and command
-ENTRYPOINT ["./entrypoint.sh"]
+# # Set your entrypoint and command
 
-# Specify arguments for the entrypoint script
-CMD ["litellm", "--port", "4000"]
+ENTRYPOINT ["litellm"]
+CMD ["--port", "4000"]

litellm/proxy/proxy_cli.py

@@ -5,7 +5,7 @@ import random
 from datetime import datetime
 import importlib
 from dotenv import load_dotenv
-import operator
+
 
 sys.path.append(os.getcwd())
 
@@ -32,29 +32,6 @@ def run_ollama_serve():
         )  # noqa
 
 
-def clone_subfolder(repo_url, subfolder, destination):
-    # Clone the full repo
-    repo_name = repo_url.split("/")[-1]
-    repo_master = os.path.join(destination, "repo_master")
-    subprocess.run(["git", "clone", repo_url, repo_master])
-
-    # Move into the subfolder
-    subfolder_path = os.path.join(repo_master, subfolder)
-
-    # Copy subfolder to destination
-    for file_name in os.listdir(subfolder_path):
-        source = os.path.join(subfolder_path, file_name)
-        if os.path.isfile(source):
-            shutil.copy(source, destination)
-        else:
-            dest_path = os.path.join(destination, file_name)
-            shutil.copytree(source, dest_path)
-
-    # Remove cloned repo folder
-    subprocess.run(["rm", "-rf", os.path.join(destination, "repo_master")])
-    feature_telemetry(feature="create-proxy")
-
-
 def is_port_in_use(port):
     import socket
 
@@ -371,6 +348,20 @@ def run_server(
             raise ImportError(
                 "Uvicorn needs to be imported. Run - `pip install uvicorn`"
             )
+        if os.getenv("DATABASE_URL", None) is not None:
+            # run prisma db push, before starting server
+            # Save the current working directory
+            original_dir = os.getcwd()
+            # set the working directory to where this script is
+            abspath = os.path.abspath(__file__)
+            dname = os.path.dirname(abspath)
+            os.chdir(dname)
+            try:
+                subprocess.run(
+                    ["prisma", "db", "push", "--accept-data-loss"]
+                )  # this looks like a weird edge case when prisma just wont start on render. we need to have the --accept-data-loss
+            finally:
+                os.chdir(original_dir)
         if port == 8000 and is_port_in_use(port):
             port = random.randint(1024, 49152)
         uvicorn.run(

litellm/proxy/utils.py

@@ -253,10 +253,11 @@ class PrismaClient:
         print_verbose(
             "LiteLLM: DATABASE_URL Set in config, trying to 'pip install prisma'"
         )
-         ## init logging object
+        ## init logging object
         self.proxy_logging_obj = proxy_logging_obj
-
-        if os.getenv("DATABASE_URL", None) is None: # setup hasn't taken place
+        try:
+            from prisma import Prisma  # type: ignore
+        except:
             os.environ["DATABASE_URL"] = database_url
             # Save the current working directory
             original_dir = os.getcwd()
@@ -272,8 +273,8 @@ class PrismaClient:
                 )  # this looks like a weird edge case when prisma just wont start on render. we need to have the --accept-data-loss
             finally:
                 os.chdir(original_dir)
-        # Now you can import the Prisma Client
-        from prisma import Prisma  # type: ignore
+            # Now you can import the Prisma Client
+            from prisma import Prisma  # type: ignore
 
         self.db = Prisma(
             http={

proxy_server_config.yaml

@@ -4,18 +4,18 @@ model_list:
       model: azure/chatgpt-v-2
       api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
       api_version: "2023-05-15"
-      api_key: os.environ/AZURE_API_KEY1 # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault
+      api_key: os.environ/AZURE_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault
   - model_name: gpt-4
     litellm_params:
-      model: azure/gpt-4
-      api_key: os.environ/AZURE_API_KEY2
-      api_base: https://openai-gpt-4-test-v-2.openai.azure.com/
+      model: azure/gpt-turbo
+      api_key: os.environ/AZURE_FRANCE_API_KEY
+      api_base: https://openai-france-1234.openai.azure.com/
       rpm: 100
   - model_name: gpt-4
     litellm_params:
-      model: azure/gpt-4
-      api_key: 
-      api_base: https://openai-gpt-4-test-v-2.openai.azure.com/
+      model: azure/gpt-35-turbo
+      api_key: os.environ/AZURE_EUROPE_API_KEY
+      api_base: https://my-endpoint-europe-berri-992.openai.azure.com
       rpm: 10
 
 litellm_settings:
@@ -23,7 +23,7 @@ litellm_settings:
   set_verbose: True
 
 general_settings: 
-  # master_key: sk-1234 # [OPTIONAL] Only use this if you to require all calls to contain this key (Authorization: Bearer sk-1234)
+  master_key: sk-1234 # [OPTIONAL] Only use this if you to require all calls to contain this key (Authorization: Bearer sk-1234)
   # database_url: "postgresql://<user>:<password>@<host>:<port>/<dbname>" # [OPTIONAL] use for token-based auth to proxy
 
 environment_variables: