diff --git a/.circleci/config.yml b/.circleci/config.yml index 56f9a15fe3a..3ea6b7fca9a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,6 +40,7 @@ jobs: pip install "httpx==0.24.1" pip install "anyio==3.7.1" pip install "asyncio==3.4.3" + pip install "PyGithub==1.59.1" - save_cache: paths: - ./venv @@ -83,6 +84,103 @@ jobs: - store_test_results: path: test-results + build_and_test: + machine: + image: ubuntu-2204:2023.10.1 + working_directory: ~/project + steps: + - checkout + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + - run: + name: Build Docker image + command: docker build -t my-app:latest -f Dockerfile.database . + - run: + name: Run Docker container + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=$PROXY_DOCKER_DB_URL \ + -e AZURE_API_KEY=$AZURE_FRANCE_API_KEY \ + -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ + -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ + --name my-app \ + -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \ + my-app:latest \ + --config /app/config.yaml \ + --port 4000 \ + --num_workers 8 + - run: + name: Install curl and dockerize + command: | + sudo apt-get update + sudo apt-get install -y curl + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start outputting logs + command: | + while true; do + docker logs my-app + sleep 10 + done + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 1m + - run: + name: Test the application + command: | + mkdir -p /tmp/responses + for i in {1..10}; do + status_file="/tmp/responses/status_${i}.txt" + response_file="/tmp/responses/response_${i}.json" + + (curl --location --request POST 'http://0.0.0.0:4000/key/generate' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{"models": ["azure-models"], "aliases": {"mistral-7b": "gpt-3.5-turbo"}, "duration": null}' \ + --silent --output "${response_file}" --write-out '%{http_code}' > "${status_file}") & + + # Capture PIDs of background processes + pids[${i}]=$! + done + + # Wait for all background processes to finish + for pid in ${pids[*]}; do + wait $pid + done + + # Check all responses and status codes + fail=false + for i in {1..10}; do + status=$(cat "/tmp/responses/status_${i}.txt") + + # Here, we need to set the correct response file path for each iteration + response_file="/tmp/responses/response_${i}.json" # This was missing in the provided script + + response=$(cat "${response_file}") + echo "Response ${i} (Status code: ${status}):" + echo "${response}" # Use echo here to print the contents + echo # Additional newline for readability + + if [ "$status" -ne 200 ]; then + echo "A request did not return a 200 status code: $status" + fail=true + fi + done + + # If any request did not return status code 200, fail the job + if [ "$fail" = true ]; then + exit 1 + fi + + echo "All requests returned a 200 status code." + publish_to_pypi: docker: - image: cimg/python:3.8 @@ -164,9 +262,16 @@ workflows: only: - main - /litellm_.*/ + - build_and_test: + filters: + branches: + only: + - main + - /litellm_.*/ - publish_to_pypi: requires: - local_testing + - build_and_test filters: branches: only: diff --git a/Dockerfile.database b/Dockerfile.database index d826896d714..5c0c4ad36dc 100644 --- a/Dockerfile.database +++ b/Dockerfile.database @@ -53,8 +53,7 @@ RUN chmod +x entrypoint.sh EXPOSE 4000/tcp -# Set your entrypoint and command -ENTRYPOINT ["./entrypoint.sh"] +# # Set your entrypoint and command -# Specify arguments for the entrypoint script -CMD ["litellm", "--port", "4000"] +ENTRYPOINT ["litellm"] +CMD ["--port", "4000"] diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index b154b21e1ba..0150cfe445a 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -5,7 +5,7 @@ import random from datetime import datetime import importlib from dotenv import load_dotenv -import operator + sys.path.append(os.getcwd()) @@ -32,29 +32,6 @@ def run_ollama_serve(): ) # noqa -def clone_subfolder(repo_url, subfolder, destination): - # Clone the full repo - repo_name = repo_url.split("/")[-1] - repo_master = os.path.join(destination, "repo_master") - subprocess.run(["git", "clone", repo_url, repo_master]) - - # Move into the subfolder - subfolder_path = os.path.join(repo_master, subfolder) - - # Copy subfolder to destination - for file_name in os.listdir(subfolder_path): - source = os.path.join(subfolder_path, file_name) - if os.path.isfile(source): - shutil.copy(source, destination) - else: - dest_path = os.path.join(destination, file_name) - shutil.copytree(source, dest_path) - - # Remove cloned repo folder - subprocess.run(["rm", "-rf", os.path.join(destination, "repo_master")]) - feature_telemetry(feature="create-proxy") - - def is_port_in_use(port): import socket @@ -371,6 +348,20 @@ def run_server( raise ImportError( "Uvicorn needs to be imported. Run - `pip install uvicorn`" ) + if os.getenv("DATABASE_URL", None) is not None: + # run prisma db push, before starting server + # Save the current working directory + original_dir = os.getcwd() + # set the working directory to where this script is + abspath = os.path.abspath(__file__) + dname = os.path.dirname(abspath) + os.chdir(dname) + try: + subprocess.run( + ["prisma", "db", "push", "--accept-data-loss"] + ) # this looks like a weird edge case when prisma just wont start on render. we need to have the --accept-data-loss + finally: + os.chdir(original_dir) if port == 8000 and is_port_in_use(port): port = random.randint(1024, 49152) uvicorn.run( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 798c02b6479..5b21a2d2bea 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -253,10 +253,11 @@ class PrismaClient: print_verbose( "LiteLLM: DATABASE_URL Set in config, trying to 'pip install prisma'" ) - ## init logging object + ## init logging object self.proxy_logging_obj = proxy_logging_obj - - if os.getenv("DATABASE_URL", None) is None: # setup hasn't taken place + try: + from prisma import Prisma # type: ignore + except: os.environ["DATABASE_URL"] = database_url # Save the current working directory original_dir = os.getcwd() @@ -272,8 +273,8 @@ class PrismaClient: ) # this looks like a weird edge case when prisma just wont start on render. we need to have the --accept-data-loss finally: os.chdir(original_dir) - # Now you can import the Prisma Client - from prisma import Prisma # type: ignore + # Now you can import the Prisma Client + from prisma import Prisma # type: ignore self.db = Prisma( http={ diff --git a/proxy_server_config.yaml b/proxy_server_config.yaml index b58f9aea1cf..abe9998586f 100644 --- a/proxy_server_config.yaml +++ b/proxy_server_config.yaml @@ -4,18 +4,18 @@ model_list: model: azure/chatgpt-v-2 api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ api_version: "2023-05-15" - api_key: os.environ/AZURE_API_KEY1 # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault + api_key: os.environ/AZURE_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault - model_name: gpt-4 litellm_params: - model: azure/gpt-4 - api_key: os.environ/AZURE_API_KEY2 - api_base: https://openai-gpt-4-test-v-2.openai.azure.com/ + model: azure/gpt-turbo + api_key: os.environ/AZURE_FRANCE_API_KEY + api_base: https://openai-france-1234.openai.azure.com/ rpm: 100 - model_name: gpt-4 litellm_params: - model: azure/gpt-4 - api_key: - api_base: https://openai-gpt-4-test-v-2.openai.azure.com/ + model: azure/gpt-35-turbo + api_key: os.environ/AZURE_EUROPE_API_KEY + api_base: https://my-endpoint-europe-berri-992.openai.azure.com rpm: 10 litellm_settings: @@ -23,7 +23,7 @@ litellm_settings: set_verbose: True general_settings: - # master_key: sk-1234 # [OPTIONAL] Only use this if you to require all calls to contain this key (Authorization: Bearer sk-1234) + master_key: sk-1234 # [OPTIONAL] Only use this if you to require all calls to contain this key (Authorization: Bearer sk-1234) # database_url: "postgresql://:@:/" # [OPTIONAL] use for token-based auth to proxy environment_variables: