diff --git a/.circleci/config.yml b/.circleci/config.yml index b8084f1df89..440343b45ea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,6 +8,11 @@ jobs: steps: - checkout + - run: + name: Show git commit hash + command: | + echo "Git commit hash: $CIRCLE_SHA1" + - run: name: Check if litellm dir was updated or if pyproject.toml was modified command: | @@ -28,17 +33,20 @@ jobs: pip install "pytest==7.3.1" pip install "pytest-asyncio==0.21.1" pip install mypy - pip install "google-generativeai>=0.3.2" - pip install "google-cloud-aiplatform>=1.38.0" - pip install "boto3>=1.28.57" - pip install "aioboto3>=12.3.0" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install pyarrow + pip install "boto3==1.34.34" + pip install "aioboto3==12.3.0" pip install langchain - pip install "langfuse>=2.0.0" + pip install lunary==0.2.5 + pip install "langfuse==2.7.3" pip install numpydoc pip install traceloop-sdk==0.0.69 pip install openai pip install prisma pip install "httpx==0.24.1" + pip install fastapi pip install "gunicorn==21.2.0" pip install "anyio==3.7.1" pip install "aiodynamo==23.10.1" @@ -48,6 +56,8 @@ jobs: pip install argon2-cffi pip install "pytest-mock==3.12.0" pip install python-multipart + pip install google-cloud-aiplatform + pip install prometheus-client==0.20.0 - save_cache: paths: - ./venv @@ -70,7 +80,7 @@ jobs: name: Linting Testing command: | cd litellm - python -m pip install types-requests types-setuptools types-redis + python -m pip install types-requests types-setuptools types-redis types-PyYAML if ! python -m mypy . --ignore-missing-imports; then echo "mypy detected errors" exit 1 @@ -120,6 +130,7 @@ jobs: build_and_test: machine: image: ubuntu-2204:2023.10.1 + resource_class: xlarge working_directory: ~/project steps: - checkout @@ -152,10 +163,11 @@ jobs: pip install "pytest-mock==3.12.0" pip install "pytest-asyncio==0.21.1" pip install mypy - pip install "google-generativeai>=0.3.2" - pip install "google-cloud-aiplatform>=1.38.0" - pip install "boto3>=1.28.57" - pip install "aioboto3>=12.3.0" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install pyarrow + pip install "boto3==1.34.34" + pip install "aioboto3==12.3.0" pip install langchain pip install "langfuse>=2.0.0" pip install numpydoc @@ -178,12 +190,19 @@ jobs: -p 4000:4000 \ -e DATABASE_URL=$PROXY_DOCKER_DB_URL \ -e AZURE_API_KEY=$AZURE_API_KEY \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ + -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ + -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ + -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ --name my-app \ -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \ my-app:latest \ @@ -288,7 +307,7 @@ jobs: -H "Accept: application/vnd.github.v3+json" \ -H "Authorization: Bearer $GITHUB_TOKEN" \ "https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \ - -d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}\"}}" + -d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}\", \"commit_hash\":\"$CIRCLE_SHA1\"}}" workflows: version: 2 diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt index 4730fc28b15..b505536e27c 100644 --- a/.circleci/requirements.txt +++ b/.circleci/requirements.txt @@ -3,12 +3,10 @@ openai python-dotenv tiktoken importlib_metadata -baseten cohere redis anthropic -boto3 orjson -pydantic -google-cloud-aiplatform +pydantic==1.10.14 +google-cloud-aiplatform==1.43.0 redisvl==0.0.7 # semantic caching \ No newline at end of file diff --git a/.dockerignore b/.dockerignore index efff383d43f..78833aa31d5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,5 @@ -/docs -/cookbook -/.circleci -/.github -/tests \ No newline at end of file +docs +cookbook +.circleci +.github +tests diff --git a/.github/workflows/ghcr_deploy.yml b/.github/workflows/ghcr_deploy.yml index b863fc3fa2e..58cda02c31b 100644 --- a/.github/workflows/ghcr_deploy.yml +++ b/.github/workflows/ghcr_deploy.yml @@ -5,6 +5,13 @@ on: inputs: tag: description: "The tag version you want to build" + release_type: + description: "The release type you want to build. Can be 'latest', 'stable', 'dev'" + type: string + default: "latest" + commit_hash: + description: "Commit hash" + required: true # Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds. env: @@ -43,6 +50,13 @@ jobs: push: true file: Dockerfile.database tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }} + - + name: Build and push litellm-spend-logs image + uses: docker/build-push-action@v5 + with: + push: true + file: ./litellm-js/spend-logs/Dockerfile + tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }} build-and-push-image: runs-on: ubuntu-latest @@ -78,9 +92,9 @@ jobs: - name: Build and push Docker image uses: docker/build-push-action@4976231911ebf5f32aad765192d35f942aa48cb8 with: - context: . + context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}} push: true - tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta.outputs.tags }}-latest # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest' + tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }} # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest' labels: ${{ steps.meta.outputs.labels }} platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 @@ -114,12 +128,49 @@ jobs: - name: Build and push Database Docker image uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: - context: . + context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}} file: Dockerfile.database push: true - tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-latest + tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }} labels: ${{ steps.meta-database.outputs.labels }} platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 + + build-and-push-image-spend-logs: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for spend-logs Dockerfile + id: meta-spend-logs + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-spend_logs + # Configure multi platform Docker builds + - name: Set up QEMU + uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 + + - name: Build and push Database Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}} + file: ./litellm-js/spend-logs/Dockerfile + push: true + tags: ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }} + platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 + build-and-push-helm-chart: runs-on: ubuntu-latest steps: @@ -191,17 +242,20 @@ jobs: with: github-token: "${{ secrets.GITHUB_TOKEN }}" script: | + const commitHash = "${{ github.event.inputs.commit_hash}}"; + console.log("Commit Hash:", commitHash); // Add this line for debugging try { const response = await github.rest.repos.createRelease({ draft: false, generate_release_notes: true, + target_commitish: commitHash, name: process.env.RELEASE_TAG, owner: context.repo.owner, prerelease: false, repo: context.repo.repo, tag_name: process.env.RELEASE_TAG, }); - + core.exportVariable('RELEASE_ID', response.data.id); core.exportVariable('RELEASE_UPLOAD_URL', response.data.upload_url); } catch (error) { @@ -243,4 +297,3 @@ jobs: } ] }' $WEBHOOK_URL - diff --git a/.github/workflows/interpret_load_test.py b/.github/workflows/interpret_load_test.py index b52d4d2b395..9d95c768fcd 100644 --- a/.github/workflows/interpret_load_test.py +++ b/.github/workflows/interpret_load_test.py @@ -77,6 +77,9 @@ if __name__ == "__main__": new_release_body = ( existing_release_body + "\n\n" + + "### Don't want to maintain your internal proxy? get in touch 🎉" + + "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat" + + "\n\n" + "## Load Test LiteLLM Proxy Results" + "\n\n" + markdown_table diff --git a/.github/workflows/locustfile.py b/.github/workflows/locustfile.py index 9e5b62ff09e..5dce0bb02f2 100644 --- a/.github/workflows/locustfile.py +++ b/.github/workflows/locustfile.py @@ -10,7 +10,7 @@ class MyUser(HttpUser): def chat_completion(self): headers = { "Content-Type": "application/json", - "Authorization": f"Bearer sk-gUvTeN9g0sgHBMf9HeCaqA", + "Authorization": f"Bearer sk-S2-EZTUUDY0EmM6-Fy0Fyw", # Include any additional headers you may need for authentication, etc. } diff --git a/.gitignore b/.gitignore index b03bc895bf9..357f3e1bf8e 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,9 @@ deploy/charts/litellm/charts/* deploy/charts/*.tgz litellm/proxy/vertex_key.json **/.vim/ +/node_modules +kub.yaml +loadtest_kub.yaml +litellm/proxy/_new_secret_config.yaml +litellm/proxy/_new_secret_config.yaml +litellm/proxy/_super_secret_config.yaml diff --git a/Dockerfile b/Dockerfile index f560473e25f..c8e9956b29d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ # Base image for building -ARG LITELLM_BUILD_IMAGE=python:3.9 +ARG LITELLM_BUILD_IMAGE=python:3.11.8-slim # Runtime image -ARG LITELLM_RUNTIME_IMAGE=python:3.9-slim +ARG LITELLM_RUNTIME_IMAGE=python:3.11.8-slim # Builder stage FROM $LITELLM_BUILD_IMAGE as builder @@ -70,5 +70,4 @@ EXPOSE 4000/tcp ENTRYPOINT ["litellm"] # Append "--detailed_debug" to the end of CMD to view detailed debug logs -# CMD ["--port", "4000", "--config", "./proxy_server_config.yaml", "--run_gunicorn", "--detailed_debug"] -CMD ["--port", "4000", "--config", "./proxy_server_config.yaml", "--run_gunicorn", "--num_workers", "4"] +CMD ["--port", "4000"] diff --git a/Dockerfile.database b/Dockerfile.database index 92859a61d8e..22084bab89c 100644 --- a/Dockerfile.database +++ b/Dockerfile.database @@ -1,8 +1,8 @@ # Base image for building -ARG LITELLM_BUILD_IMAGE=python:3.9 +ARG LITELLM_BUILD_IMAGE=python:3.11.8-slim # Runtime image -ARG LITELLM_RUNTIME_IMAGE=python:3.9-slim +ARG LITELLM_RUNTIME_IMAGE=python:3.11.8-slim # Builder stage FROM $LITELLM_BUILD_IMAGE as builder @@ -72,5 +72,5 @@ EXPOSE 4000/tcp ENTRYPOINT ["litellm"] # Append "--detailed_debug" to the end of CMD to view detailed debug logs -# CMD ["--port", "4000","--run_gunicorn", "--detailed_debug"] -CMD ["--port", "4000", "--run_gunicorn"] +# CMD ["--port", "4000", "--detailed_debug"] +CMD ["--port", "4000"] diff --git a/README.md b/README.md index 6bdaa9d3755..3caeb830bb6 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@
Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, etc.]