diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b0a6924cce..39492004718 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,40 +16,90 @@ commands: echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf + wait_for_service: + description: "Poll a TCP or HTTP endpoint until it responds (replaces dockerize -wait)" + parameters: + url: + type: string + timeout: + type: string + default: "60" + steps: + - run: + name: "Wait for << parameters.url >>" + command: | + TIMEOUT=<< parameters.timeout >> + URL="<< parameters.url >>" + ELAPSED=0 + echo "Waiting up to ${TIMEOUT}s for ${URL} ..." + if echo "$URL" | grep -q '^tcp://'; then + HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1) + PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2) + while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do + sleep 2; ELAPSED=$((ELAPSED+2)) + if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi + done + else + while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do + sleep 2; ELAPSED=$((ELAPSED+2)) + if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi + done + fi + echo "Service ready after ${ELAPSED}s" + install_helm: + steps: + - run: + name: Install Helm v3.17.3 + command: | + curl -sSLf -o /tmp/helm.tar.gz \ + https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz + echo "ee88b3c851ae6466a3de507f7be73fe94d54cbf2987cbaa3d1a3832ea331f2cd /tmp/helm.tar.gz" | sha256sum -c - + sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/helm.tar.gz linux-amd64/helm + rm -f /tmp/helm.tar.gz + install_kind: + steps: + - run: + name: Install Kind v0.20.0 + command: | + curl -sSLf -o /tmp/kind \ + https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 + echo "513a7213d6d3332dd9ef27c24dab35e5ef10a04fa27274fe1c14d8a246493ded /tmp/kind" | sha256sum -c - + chmod +x /tmp/kind + sudo mv /tmp/kind /usr/local/bin/kind setup_litellm_enterprise_pip: steps: - run: name: "Install local version of litellm-enterprise" command: | - pip install --force-reinstall --no-deps -e enterprise/ + # litellm-enterprise is a uv workspace member and is already installed + # by the main `uv sync --all-groups --all-extras`. Do NOT run + # `uv sync --package litellm-enterprise` here — that overwrites the + # shared .venv and strips out dev/test deps (pytest, prisma, etc.). + uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)" setup_litellm_test_deps: steps: - checkout - setup_google_dns - restore_cache: keys: - - v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} + - v3-litellm-uv-deps-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - # Use uv for the heavy requirements.txt (10-100x faster than pip) - uv pip install --system -r requirements.txt - # Use pip for test deps (small set, avoids uv strict-resolution - # conflicts with transitive dep pins like openai<2 and pydantic>=2.11.5) - pip install "pytest-mock==3.12.0" "pytest==7.3.1" "pytest-retry==1.6.3" \ - "pytest-asyncio==0.21.1" "respx==0.22.0" "hypercorn==0.17.3" \ - "pydantic==2.12.5" "mcp==1.26.0" "requests-mock>=1.12.1" \ - "responses==0.25.7" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" \ - "pytest-cov==5.0.0" "semantic_router==0.1.10" "fastapi-offline==1.7.3" \ - "a2a" "parameterized>=0.9.0" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + uv sync --frozen --all-groups --all-extras --python "$(which python)" - setup_litellm_enterprise_pip - save_cache: paths: - ~/.local/lib - ~/.local/bin - ~/.cache/uv - key: v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} + key: v3-litellm-uv-deps-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} jobs: # Add Windows testing job @@ -71,13 +121,20 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - pip install pytest - pip install . + Invoke-RestMethod https://astral.sh/uv/0.10.9/install.ps1 | Invoke-Expression + $uvBin = Join-Path $HOME ".local\bin" + $env:Path = "$uvBin;$env:Path" + if (!(Test-Path $PROFILE)) { + New-Item -ItemType File -Force -Path $PROFILE | Out-Null + } + if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) { + Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`"" + } + uv sync --frozen --group dev --python (Get-Command python).Source - run: name: Run Windows-specific test command: | - python -m pytest tests/windows_tests/test_litellm_on_windows.py -v + uv run --no-sync python -m pytest tests/windows_tests/test_litellm_on_windows.py -v mypy_linting: docker: @@ -94,16 +151,19 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip uninstall fastuuid -y - pip install "mypy==1.18.2" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + uv sync --frozen --group dev --python "$(which python)" --no-install-package fastuuid - run: name: MyPy Type Checking command: | cd litellm # Use the same approach as GitHub Actions, explicitly exclude fastuuid to avoid segfaults - python -m mypy . + uv run --no-sync python -m mypy . cd .. no_output_timeout: 10m @@ -120,10 +180,19 @@ jobs: - setup_google_dns - run: name: Install Semgrep - command: pip install semgrep + command: | + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" - run: name: Run Semgrep (custom rules only) - command: semgrep scan --config .semgrep/rules . --error + command: | + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + uv tool run --from 'semgrep==1.157.0' semgrep scan --config .semgrep/rules . --error local_testing_part1: docker: @@ -143,31 +212,27 @@ jobs: - restore_cache: keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \ - "mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.133.0" pyarrow \ - "boto3==1.42.80" langchain lunary==0.2.5 \ - "azure-identity==1.25.3" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \ - traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \ - "detect_secrets==1.5.0" "respx==0.22.0" fastapi \ - "gunicorn==23.0.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \ - "apscheduler==3.11.2" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \ - python-multipart prometheus-client==0.20.0 "pydantic==2.12.5" \ - "diskcache==5.6.1" "Pillow==12.1.1" "jsonschema==4.23.0" \ - "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==15.0.1" - pip install semantic_router --no-deps - pip install aurelio_sdk --no-deps - pip uninstall posthog -y + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - setup_litellm_enterprise_pip - save_cache: paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - ./.venv + key: v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Run prisma ./docker/entrypoint.sh command: | @@ -179,8 +244,7 @@ jobs: name: Black Formatting command: | cd litellm - python -m pip install black - python -m black . + uv run --no-sync python -m black . cd .. # Run pytest and generate JUnit XML report @@ -195,7 +259,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ - --command="xargs python -m pytest \ + --command="xargs uv run --no-sync python -m pytest \ -vv \ --cov=litellm \ --cov-report=xml \ @@ -238,31 +302,27 @@ jobs: - restore_cache: keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \ - "mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.133.0" pyarrow \ - "boto3==1.42.80" langchain lunary==0.2.5 \ - "azure-identity==1.25.3" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \ - traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \ - "detect_secrets==1.5.0" "respx==0.22.0" fastapi \ - "gunicorn==23.0.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \ - "apscheduler==3.11.2" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \ - python-multipart prometheus-client==0.20.0 "pydantic==2.12.5" \ - "diskcache==5.6.1" "Pillow==12.1.1" "jsonschema==4.23.0" \ - "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==15.0.1" - pip install semantic_router --no-deps - pip install aurelio_sdk --no-deps - pip uninstall posthog -y + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - setup_litellm_enterprise_pip - save_cache: paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - ./.venv + key: v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Run prisma ./docker/entrypoint.sh command: | @@ -274,8 +334,7 @@ jobs: name: Black Formatting command: | cd litellm - python -m pip install black - python -m black . + uv run --no-sync python -m black . cd .. # Run pytest and generate JUnit XML report @@ -290,7 +349,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ - --command="xargs python -m pytest \ + --command="xargs uv run --no-sync python -m pytest \ -vv \ --cov=litellm \ --cov-report=xml \ @@ -334,58 +393,27 @@ jobs: - restore_cache: keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.133.0" - pip install pyarrow - pip install "boto3==1.42.80" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.25.3" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.28.0 - pip install opentelemetry-sdk==1.28.0 - pip install opentelemetry-exporter-otlp==1.28.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.28.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==23.0.0" - pip install "anyio==4.8.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.11.2" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.12.5" - pip install "diskcache==5.6.1" - pip install "Pillow==12.1.1" - pip install "jsonschema==4.23.0" - pip install "websockets==15.0.1" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - setup_litellm_enterprise_pip - save_cache: paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - ./.venv + key: v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Run prisma ./docker/entrypoint.sh command: | @@ -400,7 +428,7 @@ jobs: command: | pwd ls - python -m pytest -v tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse" + uv run --no-sync python -m pytest -v tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse" no_output_timeout: 15m # Store test results - store_test_results: @@ -419,16 +447,22 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-xdist==3.6.1" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - save_cache: paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - ./.venv + key: v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Run prisma ./docker/entrypoint.sh command: | @@ -436,13 +470,16 @@ jobs: chmod +x docker/entrypoint.sh ./docker/entrypoint.sh set -e + - run: + name: Generate Prisma Client + command: uv run --no-sync python -m prisma generate # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -v tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 -n 2 + uv run --no-sync python -m pytest -v tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 -n 2 no_output_timeout: 15m # Store test results @@ -463,25 +500,27 @@ jobs: - setup_google_dns - restore_cache: keys: - - v1-router-testing-deps-{{ checksum "requirements.txt" }} + - v1-router-testing-deps-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "respx==0.22.0" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-xdist==3.6.1" - pip install "pytest-timeout==2.2.0" - pip install semantic_router --no-deps - pip install aurelio_sdk --no-deps + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - save_cache: paths: - /home/circleci/.pyenv - /home/circleci/.local - key: v1-router-testing-deps-{{ checksum "requirements.txt" }} + key: v1-router-testing-deps-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: @@ -494,7 +533,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ - --command="xargs python -m pytest \ + --command="xargs uv run --no-sync python -m pytest \ -v \ -k 'router' \ -n 4 \ @@ -520,24 +559,27 @@ jobs: - setup_google_dns - restore_cache: keys: - - v1-router-unit-deps-{{ checksum "requirements.txt" }} + - v1-router-unit-deps-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "respx==0.22.0" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install semantic_router --no-deps - pip install aurelio_sdk --no-deps - pip install "pytest-xdist==3.6.1" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - save_cache: paths: - /home/circleci/.pyenv - /home/circleci/.local - key: v1-router-unit-deps-{{ checksum "requirements.txt" }} + key: v1-router-unit-deps-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: @@ -545,7 +587,7 @@ jobs: command: | pwd ls - python -m pytest -v tests/router_unit_tests -x --junitxml=test-results/junit.xml --durations=5 -n 4 + uv run --no-sync python -m pytest -v tests/router_unit_tests -x --junitxml=test-results/junit.xml --durations=5 -n 4 no_output_timeout: 15m # Store test results - store_test_results: @@ -565,13 +607,18 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - pip install wheel setuptools - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "respx==0.22.0" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: @@ -579,7 +626,7 @@ jobs: command: | pwd ls - python -m pytest tests/local_testing/ -v -k "assistants" -x --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest tests/local_testing/ -v -k "assistants" -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Store test results - store_test_results: @@ -598,23 +645,27 @@ jobs: - setup_google_dns - restore_cache: keys: - - v1-llm-translation-deps-{{ checksum "requirements.txt" }} + - v1-llm-translation-deps-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "pytest-xdist==3.6.1" - pip install "pytest-timeout==2.2.0" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - save_cache: paths: - /home/circleci/.pyenv - /home/circleci/.local - key: v1-llm-translation-deps-{{ checksum "requirements.txt" }} + key: v1-llm-translation-deps-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - run: name: Run tests @@ -631,7 +682,7 @@ jobs: for dir in "${IGNORE_DIRS[@]}"; do IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir" done - python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 + uv run --no-sync python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 no_output_timeout: 15m # Store test results @@ -651,9 +702,18 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run realtime tests @@ -662,7 +722,7 @@ jobs: ls # Add --timeout to kill hanging tests after 120s (2 min) # Add --durations=20 to show 20 slowest tests for debugging - python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread + uv run --no-sync python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread no_output_timeout: 15m - run: name: Rename the coverage files @@ -692,23 +752,25 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "pydantic==2.12.5" - pip install "mcp==1.26.0" - pip install "pytest-xdist==3.6.1" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/mcp_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 + uv run --no-sync python -m pytest -vv tests/mcp_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 no_output_timeout: 15m - run: name: Rename the coverage files @@ -738,22 +800,25 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "pydantic==2.12.5" - pip install "a2a-sdk" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m - run: name: Rename the coverage files @@ -783,26 +848,25 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "pydantic==2.12.5" - pip install "boto3==1.42.80" - pip install "semantic_router==0.1.10" --no-deps - pip install aurelio_sdk - pip install "pytest-xdist==3.6.1" - pip install "pytest-timeout==2.2.0" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - LITELLM_LOG=WARNING python -m pytest tests/guardrails_tests -vv --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -n 2 --timeout=120 --timeout_method=thread + LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/guardrails_tests -vv --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -n 2 --timeout=120 --timeout_method=thread no_output_timeout: 15m - run: name: Rename the coverage files @@ -833,21 +897,25 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "pydantic==2.12.5" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5 + uv run --no-sync python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5 no_output_timeout: 15m - run: name: Rename the coverage files @@ -878,29 +946,34 @@ jobs: - setup_google_dns - restore_cache: keys: - - v1-llm-responses-deps-{{ checksum "requirements.txt" }} + - v1-llm-responses-deps-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "pytest-xdist==3.6.1" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - save_cache: paths: - /home/circleci/.pyenv - /home/circleci/.local - key: v1-llm-responses-deps-{{ checksum "requirements.txt" }} + key: v1-llm-responses-deps-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -v tests/llm_responses_api_testing -x --junitxml=test-results/junit.xml --durations=5 -n 8 + uv run --no-sync python -m pytest -v tests/llm_responses_api_testing -x --junitxml=test-results/junit.xml --durations=5 -n 8 no_output_timeout: 15m # Store test results @@ -920,16 +993,25 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 + uv run --no-sync python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 no_output_timeout: 15m - run: name: Rename the coverage files @@ -959,16 +1041,25 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 + uv run --no-sync python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 no_output_timeout: 15m - run: name: Rename the coverage files @@ -998,9 +1089,9 @@ jobs: - run: name: Run proxy tests part 1 (high-volume directories) command: | - prisma generate + uv run --no-sync python -m prisma generate export PYTHONUNBUFFERED=1 - python -m pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/client tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part1.xml --durations=10 -n 4 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A + uv run --no-sync python -m pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/client tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part1.xml --durations=10 -n 4 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A no_output_timeout: 15m - store_test_results: path: test-results @@ -1017,9 +1108,9 @@ jobs: - run: name: Run proxy tests part 2 (all other tests) command: | - prisma generate + uv run --no-sync python -m prisma generate export PYTHONUNBUFFERED=1 - python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 4 --maxfail=5 --timeout=120 -vv --log-cli-level=WARNING -r A + uv run --no-sync python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 4 --maxfail=5 --timeout=120 -vv --log-cli-level=WARNING -r A no_output_timeout: 15m - store_test_results: path: test-results @@ -1038,31 +1129,26 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest-mock==3.12.0" - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "hypercorn==0.17.3" - pip install "pydantic==2.12.5" - pip install "mcp==1.26.0" - pip install "requests-mock>=1.12.1" - pip install "responses==0.25.7" - pip install "pytest-xdist==3.6.1" - pip install "semantic_router==0.1.10" --no-deps - pip install aurelio_sdk - pip install "fastapi-offline==1.7.3" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - setup_litellm_enterprise_pip - run: name: Run enterprise tests command: | pwd ls - prisma generate - python -m pytest -v tests/enterprise -x --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4 + uv run --no-sync python -m prisma generate + uv run --no-sync python -m pytest -v tests/enterprise -x --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4 no_output_timeout: 15m # Store test results - store_test_results: @@ -1081,23 +1167,25 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "respx==0.22.0" - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.133.0" - pip install "pytest-xdist==3.6.1" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 + uv run --no-sync python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 no_output_timeout: 15m - run: name: Rename the coverage files @@ -1127,25 +1215,25 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install numpydoc - pip install "respx==0.22.0" - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.133.0" - pip install pytest-mock - pip install "pytest-xdist==3.6.1" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 + uv run --no-sync python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 no_output_timeout: 15m - run: name: Rename the coverage files @@ -1176,16 +1264,25 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 + uv run --no-sync python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 no_output_timeout: 15m - run: name: Rename the coverage files @@ -1216,21 +1313,25 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "pytest-xdist==3.6.1" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -v tests/image_gen_tests -n 4 -x --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -v tests/image_gen_tests -n 4 -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Store test results - store_test_results: @@ -1249,21 +1350,18 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install pytest-mock - pip install "respx==0.22.0" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.133.0" - pip install "mlflow==2.17.2" - pip install "anthropic==0.54.0" - pip install "blockbuster==1.5.24" - pip install "pytest-xdist==3.6.1" - pip install "pytest-timeout==2.2.0" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: @@ -1271,7 +1369,7 @@ jobs: command: | pwd ls - LITELLM_LOG=WARNING python -m pytest tests/logging_callback_tests -vv --cov=litellm --cov-report=xml -n 4 --junitxml=test-results/junit.xml --durations=5 --timeout=120 --timeout_method=thread + LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/logging_callback_tests -vv --cov=litellm --cov-report=xml -n 4 --junitxml=test-results/junit.xml --durations=5 --timeout=120 --timeout_method=thread no_output_timeout: 15m - run: name: Rename the coverage files @@ -1301,20 +1399,25 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m - run: name: Rename the coverage files @@ -1341,24 +1444,28 @@ jobs: steps: - checkout - setup_google_dns + - restore_cache: + keys: + - v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-xdist==3.6.1" - pip install "pytest-rerunfailures==14.0" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + uv sync --frozen --all-groups --all-extras --python "$(which python)" + - save_cache: + paths: + - ./.venv + key: v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} # Run pytest and generate JUnit XML report - run: name: Run tests command: | - pwd - ls - python -m pytest -vv \ + uv run --no-sync python -m pytest -vv \ tests/local_testing/test_dual_cache.py \ tests/local_testing/test_redis_batch_optimizations.py \ tests/local_testing/test_router_utils.py \ @@ -1395,26 +1502,25 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - pip install python-dotenv - pip install pytest - pip install tiktoken - pip install aiohttp - pip install openai - pip install click - pip install "boto3==1.42.80" - pip install jinja2 - pip install "tokenizers==0.22.2" - pip install "uvloop==0.21.0" - pip install "fastuuid==0.14.0" - pip install jsonschema + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - setup_litellm_enterprise_pip - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/local_testing/test_basic_python_version.py + uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py installing_litellm_on_python_3_13: docker: @@ -1431,21 +1537,24 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - pip install wheel setuptools - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "tomli==2.2.1" - pip install "mcp==1.26.0" + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Run tests command: | pwd ls - python -m pytest -v tests/local_testing/test_basic_python_version.py + uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py helm_chart_testing: machine: image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker @@ -1457,27 +1566,21 @@ jobs: - attach_workspace: at: ~/project - setup_google_dns - # Install Helm - - run: - name: Install Helm - command: | - curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + - install_helm + - install_kind - # Install kind + # Install kubectl (pinned version with official checksum verification) - run: - name: Install Kind + name: Install kubectl v1.31.4 command: | - curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 - chmod +x ./kind - sudo mv ./kind /usr/local/bin/kind - - # Install kubectl - - run: - name: Install kubectl - command: | - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - chmod +x kubectl - sudo mv kubectl /usr/local/bin/ + curl -sSLf -o /tmp/kubectl \ + https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl + curl -sSLf -o /tmp/kubectl.sha256 \ + https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl.sha256 + echo "$(cat /tmp/kubectl.sha256) /tmp/kubectl" | sha256sum -c - + chmod +x /tmp/kubectl + sudo mv /tmp/kubectl /usr/local/bin/ + rm -f /tmp/kubectl.sha256 # Create kind cluster - run: @@ -1546,42 +1649,47 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - pip install ruff - pip install pylint - pip install pyright - pip install beautifulsoup4 - pip install . - curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - - run: python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) - - run: ruff check ./litellm + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" + - run: uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) + - run: uv run --no-sync ruff check ./litellm # - run: python ./tests/documentation_tests/test_general_setting_keys.py - - run: python ./tests/code_coverage_tests/check_licenses.py - - run: python ./tests/code_coverage_tests/check_provider_folders_documented.py - - run: python ./tests/code_coverage_tests/router_code_coverage.py - - run: python ./tests/code_coverage_tests/test_chat_completion_imports.py - - run: python ./tests/code_coverage_tests/info_log_check.py - - run: python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py - - run: python ./tests/code_coverage_tests/test_ban_set_verbose.py - - run: python ./tests/code_coverage_tests/code_qa_check_tests.py - - run: python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py - - run: python ./tests/code_coverage_tests/test_proxy_types_import.py - - run: python ./tests/code_coverage_tests/callback_manager_test.py - - run: python ./tests/code_coverage_tests/recursive_detector.py - - run: python ./tests/code_coverage_tests/test_router_strategy_async.py - - run: python ./tests/code_coverage_tests/litellm_logging_code_coverage.py - - run: python ./tests/documentation_tests/test_env_keys.py - - run: python ./tests/documentation_tests/test_router_settings.py - - run: python ./tests/documentation_tests/test_api_docs.py - - run: python ./tests/code_coverage_tests/ensure_async_clients_test.py - - run: python ./tests/code_coverage_tests/enforce_llms_folder_style.py - - run: python ./tests/documentation_tests/test_circular_imports.py - - run: python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py - - run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py - - run: python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py - - run: python ./tests/code_coverage_tests/check_fastuuid_usage.py - - run: python ./tests/code_coverage_tests/memory_test.py - - run: helm lint ./deploy/charts/litellm-helm + - run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py + - run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py + - run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py + - run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py + - run: uv run --no-sync python ./tests/code_coverage_tests/info_log_check.py + - run: uv run --no-sync python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py + - run: uv run --no-sync python ./tests/code_coverage_tests/test_ban_set_verbose.py + - run: uv run --no-sync python ./tests/code_coverage_tests/code_qa_check_tests.py + - run: uv run --no-sync python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py + - run: uv run --no-sync python ./tests/code_coverage_tests/test_proxy_types_import.py + - run: uv run --no-sync python ./tests/code_coverage_tests/callback_manager_test.py + - run: uv run --no-sync python ./tests/code_coverage_tests/recursive_detector.py + - run: uv run --no-sync python ./tests/code_coverage_tests/test_router_strategy_async.py + - run: uv run --no-sync python ./tests/code_coverage_tests/litellm_logging_code_coverage.py + - run: uv run --no-sync python ./tests/documentation_tests/test_env_keys.py + - run: uv run --no-sync python ./tests/documentation_tests/test_router_settings.py + - run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py + - run: uv run --no-sync python ./tests/code_coverage_tests/ensure_async_clients_test.py + - run: uv run --no-sync python ./tests/code_coverage_tests/enforce_llms_folder_style.py + - run: uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py + - run: uv run --no-sync python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py + - run: uv run --no-sync python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py + - run: uv run --no-sync python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py + - run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py + - run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py + # helm lint is handled by the dedicated helm_chart_testing job db_migration_disable_update_check: machine: @@ -1605,16 +1713,18 @@ jobs: - run: name: Install Dependencies command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - pip install apscheduler - - run: - name: Install dockerize - command: | - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Start PostgreSQL Database command: | @@ -1625,9 +1735,9 @@ jobs: -e POSTGRES_DB=litellm_test \ -p 5432:5432 \ postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" - attach_workspace: at: ~/project - run: @@ -1649,9 +1759,9 @@ jobs: --config /app/config.yaml \ --port 4000 \ --use_prisma_db_push - - run: - name: Wait for schema seed to complete - command: dockerize -wait http://localhost:4001 -timeout 5m + - wait_for_service: + url: http://localhost:4001 + timeout: "300" - run: name: Stop schema seed container command: docker stop schema-seed && docker rm schema-seed @@ -1671,9 +1781,9 @@ jobs: litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - - run: - name: Wait for container to be ready - command: dockerize -wait http://localhost:4000 -timeout 1m + - wait_for_service: + url: http://localhost:4000 + timeout: "60" - run: name: Check container logs for expected message command: | @@ -1691,7 +1801,7 @@ jobs: - run: name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) command: | - python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 no_output_timeout: 15m build_and_test: @@ -1718,42 +1828,18 @@ jobs: - run: name: Install Dependencies command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.133.0" - pip install pyarrow - pip install "boto3==1.42.80" - pip install langchain - pip install "langfuse>=2.0.0" - pip install "logfire==0.29.0" - pip install numpydoc - pip install prisma - pip install fastapi - pip install jsonschema - pip install "httpx==0.28.1" - pip install "gunicorn==23.0.0" - pip install "anyio==4.8.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "PyGithub==1.59.1" - pip install "openai==1.100.1" - pip install "litellm[proxy]" - pip install "pytest-xdist==3.6.1" - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Start PostgreSQL Database command: | @@ -1764,9 +1850,9 @@ jobs: -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" - run: name: Load Docker Database Image command: | @@ -1818,15 +1904,15 @@ jobs: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Run tests command: | pwd ls - python -m pytest -s -v tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests + uv run --no-sync python -m pytest -s -v tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests no_output_timeout: 15m # Store test results @@ -1841,10 +1927,8 @@ jobs: - checkout - setup_google_dns - run: - name: Install Docker CLI (In case it's not already installed) + name: Verify Docker is available command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER docker version - run: name: Install Python 3.10 @@ -1860,43 +1944,18 @@ jobs: - run: name: Install Dependencies command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - pip install "mypy==1.18.2" - pip install "jsonlines==4.0.0" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.133.0" - pip install pyarrow - pip install "boto3==1.42.80" - pip install langchain - pip install "langchain_mcp_adapters==0.0.5" - pip install "langfuse>=2.0.0" - pip install "logfire==0.29.0" - pip install numpydoc - pip install prisma - pip install fastapi - pip install jsonschema - pip install "httpx==0.28.1" - pip install "gunicorn==23.0.0" - pip install "anyio==4.8.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "PyGithub==1.59.1" - pip install "openai==1.100.1" - # Run pytest and generate JUnit XML report - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Start PostgreSQL Database command: | @@ -1907,9 +1966,9 @@ jobs: -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" - attach_workspace: at: ~/project - run: @@ -1964,15 +2023,15 @@ jobs: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Run tests command: | pwd ls - python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Store test results @@ -1987,10 +2046,8 @@ jobs: - checkout - setup_google_dns - run: - name: Install Docker CLI (In case it's not already installed) + name: Verify Docker is available command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER docker version - run: name: Install Python 3.9 @@ -2006,40 +2063,18 @@ jobs: - run: name: Install Dependencies command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.133.0" - pip install pyarrow - pip install "boto3==1.42.80" - pip install langchain - pip install "langfuse>=2.0.0" - pip install "logfire==0.29.0" - pip install numpydoc - pip install prisma - pip install fastapi - pip install jsonschema - pip install "httpx==0.28.1" - pip install "gunicorn==23.0.0" - pip install "anyio==4.8.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "PyGithub==1.59.1" - pip install "openai==1.100.1" - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Start PostgreSQL Database command: | @@ -2050,9 +2085,9 @@ jobs: -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" - attach_workspace: at: ~/project - run: @@ -2092,27 +2127,19 @@ jobs: --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Run tests command: | pwd ls - python -m pytest -v tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -v tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Clean up first container - run: @@ -2145,17 +2172,17 @@ jobs: - run: name: Start outputting logs for second container - command: docker logs -f my-app-2 + command: docker logs -f my-app-3 background: true - - run: - name: Wait for second app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Run second round of tests command: | - python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 no_output_timeout: 15m # Store test results @@ -2170,10 +2197,8 @@ jobs: - checkout - setup_google_dns - run: - name: Install Docker CLI (In case it's not already installed) + name: Verify Docker is available command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER docker version - run: name: Install Python 3.9 @@ -2189,17 +2214,18 @@ jobs: - run: name: Install Dependencies command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Start PostgreSQL Database command: | @@ -2210,9 +2236,9 @@ jobs: -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" - attach_workspace: at: ~/project - run: @@ -2256,15 +2282,15 @@ jobs: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Clean up first container - run: @@ -2282,10 +2308,8 @@ jobs: - checkout - setup_google_dns - run: - name: Install Docker CLI (In case it's not already installed) + name: Verify Docker is available command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER docker version - run: name: Install Python 3.9 @@ -2301,21 +2325,18 @@ jobs: - run: name: Install Dependencies command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Start PostgreSQL Database command: | @@ -2326,9 +2347,9 @@ jobs: -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" - attach_workspace: at: ~/project - run: @@ -2380,30 +2401,22 @@ jobs: --config /app/config.yaml \ --port 4001 \ --detailed_debug - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for instance 1 to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m - - run: - name: Wait for instance 2 to be ready - command: dockerize -wait http://localhost:4001 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" + - wait_for_service: + url: http://localhost:4001 + timeout: "300" - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Clean up first container # Store test results @@ -2419,10 +2432,8 @@ jobs: - checkout - setup_google_dns - run: - name: Install Docker CLI (In case it's not already installed) + name: Verify Docker is available command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER docker version sudo systemctl restart docker - run: @@ -2439,22 +2450,18 @@ jobs: - run: name: Install Dependencies command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - pip install "assemblyai==0.37.0" - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Start PostgreSQL Database command: | @@ -2465,9 +2472,9 @@ jobs: -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" - attach_workspace: at: ~/project - run: @@ -2493,27 +2500,19 @@ jobs: --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m - run: name: Stop and remove containers @@ -2550,9 +2549,18 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - pip install "pytest==7.3.1" "pytest-asyncio==0.21.1" "pytest-retry==1.6.3" \ - "pytest-mock==3.12.0" "mypy==1.18.2" aiohttp apscheduler + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Build Docker image command: | @@ -2601,25 +2609,17 @@ jobs: --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Run tests command: | - python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + uv run --no-sync python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 no_output_timeout: 15m # Clean up first container - run: @@ -2652,45 +2652,18 @@ jobs: - run: name: Install Dependencies command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "google-cloud-aiplatform==1.133.0" - pip install aiohttp - pip install "openai==1.100.1" - pip install "assemblyai==0.37.0" - python -m pip install --upgrade pip - pip install "pydantic==2.12.5" - pip install "pytest==7.3.1" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - pip install "boto3==1.42.80" - pip install "mypy==1.18.2" - pip install pyarrow - pip install numpydoc - pip install prisma - pip install fastapi - pip install jsonschema - pip install "httpx==0.27.0" - pip install "anyio==4.8.0" - pip install "asyncio==3.4.3" - pip install "PyGithub==1.59.1" - pip install "google-cloud-aiplatform==1.59.0" - pip install "anthropic==0.54.0" - pip install "langchain_mcp_adapters==0.0.5" - pip install "langchain_openai==0.2.1" - pip install "langgraph==0.3.18" - pip install "fastuuid==0.13.5" - pip install -r requirements.txt - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Start PostgreSQL Database command: | @@ -2701,9 +2674,9 @@ jobs: -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" - attach_workspace: at: ~/project - run: @@ -2741,9 +2714,9 @@ jobs: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" # Add Ruby installation and testing before the existing Node.js and Python tests - run: name: Install Ruby and Bundler @@ -2806,7 +2779,7 @@ jobs: conda activate myenv pwd ls - python -m pytest -v tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -v tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Store test results @@ -2822,10 +2795,8 @@ jobs: - checkout - setup_google_dns - run: - name: Install Docker CLI (In case it's not already installed) + name: Verify Docker is available command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER docker version - run: name: Install Python 3.10 @@ -2841,21 +2812,18 @@ jobs: - run: name: Install Dependencies command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install "boto3==1.42.80" - pip install "httpx==0.27.0" - pip install "claude-agent-sdk" - pip install -r requirements.txt - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then + export PATH="$HOME/miniconda/bin:$PATH" + source "$HOME/miniconda/etc/profile.d/conda.sh" + conda activate myenv + fi + uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Start PostgreSQL Database command: | @@ -2866,9 +2834,9 @@ jobs: -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" - attach_workspace: at: ~/project - run: @@ -2899,9 +2867,9 @@ jobs: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Run Claude Agent SDK E2E Tests command: | @@ -2912,7 +2880,7 @@ jobs: export LITELLM_API_KEY="sk-1234" pwd ls - python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Store test results @@ -2937,17 +2905,20 @@ jobs: - run: name: Combine Coverage command: | - python -m venv venv - . venv/bin/activate - pip install coverage - coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage - coverage xml + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage + uv tool run --from 'coverage[toml]==7.10.6' coverage xml - codecov/upload: file: ./coverage.xml publish_proxy_extras: docker: - - image: cimg/python:3.8 + - image: cimg/python:3.12 working_directory: ~/project/litellm-proxy-extras environment: TWINE_USERNAME: __token__ @@ -2959,10 +2930,15 @@ jobs: - run: name: Check if litellm-proxy-extras dir or pyproject.toml was modified command: | - echo "Install TOML package." - python -m pip install toml + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" # Get current version from pyproject.toml - CURRENT_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") + CURRENT_VERSION=$(python -c 'import tomllib; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); print(data["project"]["version"])') # Get last published version from PyPI LAST_VERSION=$(curl -s https://pypi.org/pypi/litellm-proxy-extras/json | python -c "import json, sys; print(json.load(sys.stdin)['info']['version'])") @@ -2971,7 +2947,7 @@ jobs: echo "Last published version: $LAST_VERSION" # Compare versions using Python's packaging.version - VERSION_COMPARE=$(python -c "from packaging import version; print(1 if version.parse('$CURRENT_VERSION') < version.parse('$LAST_VERSION') else 0)") + VERSION_COMPARE=$(uv run --with 'packaging==25.0' python -c "from packaging import version; print(1 if version.parse('$CURRENT_VERSION') < version.parse('$LAST_VERSION') else 0)") echo "Version compare: $VERSION_COMPARE" if [ "$VERSION_COMPARE" = "1" ]; then @@ -2979,38 +2955,17 @@ jobs: exit 1 fi - # If versions are equal or current is greater, check contents - pip download --no-deps litellm-proxy-extras==$LAST_VERSION -d /tmp - - echo "Contents of /tmp directory:" - ls -la /tmp - - # Find the downloaded file (could be .whl or .tar.gz) - DOWNLOADED_FILE=$(ls /tmp/litellm_proxy_extras-*) - echo "Downloaded file: $DOWNLOADED_FILE" - - # Extract based on file extension - if [[ "$DOWNLOADED_FILE" == *.whl ]]; then - echo "Extracting wheel file..." - unzip -q "$DOWNLOADED_FILE" -d /tmp/extracted - EXTRACTED_DIR="/tmp/extracted" - else - echo "Extracting tar.gz file..." - tar -xzf "$DOWNLOADED_FILE" -C /tmp - EXTRACTED_DIR="/tmp/litellm_proxy_extras-$LAST_VERSION" - fi - - echo "Contents of extracted package:" - ls -R "$EXTRACTED_DIR" + # If versions are equal or current is greater, compare against the published package contents. + EXTRACTED_DIR=$(uv run --with "litellm-proxy-extras==$LAST_VERSION" python -c 'import importlib.util; from pathlib import Path; spec = importlib.util.find_spec("litellm_proxy_extras"); assert spec is not None and spec.origin is not None, "litellm_proxy_extras not found in uv-run environment"; print(Path(spec.origin).resolve().parent)') # Compare contents - if ! diff -r "$EXTRACTED_DIR/litellm_proxy_extras" ./litellm_proxy_extras; then + if ! diff -r "$EXTRACTED_DIR" ./litellm_proxy_extras; then if [ "$CURRENT_VERSION" = "$LAST_VERSION" ]; then echo "Error: Changes detected in litellm-proxy-extras but version was not bumped" echo "Current version: $CURRENT_VERSION" echo "Last published version: $LAST_VERSION" echo "Changes:" - diff -r "$EXTRACTED_DIR/litellm_proxy_extras" ./litellm_proxy_extras + diff -r "$EXTRACTED_DIR" ./litellm_proxy_extras exit 1 fi else @@ -3021,7 +2976,7 @@ jobs: - run: name: Get new version command: | - NEW_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") + NEW_VERSION=$(python -c 'import tomllib; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); print(data["project"]["version"])') echo "export NEW_VERSION=$NEW_VERSION" >> $BASH_ENV - run: @@ -3029,27 +2984,21 @@ jobs: command: | cd ~/project # Check pyproject.toml - CURRENT_VERSION=$(python -c "import toml; dep = toml.load('pyproject.toml')['tool']['poetry']['dependencies']['litellm-proxy-extras']; print(dep['version'] if isinstance(dep, dict) else dep)") + CURRENT_VERSION=$(uv run --with 'packaging==25.0' python -c 'import tomllib; from packaging.requirements import Requirement; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); matches = [spec.version for requirement in data["project"]["optional-dependencies"]["proxy"] for parsed in [Requirement(requirement)] if parsed.name == "litellm-proxy-extras" and parsed.specifier for spec in parsed.specifier if spec.operator == "=="]; print(matches[0] if matches else (_ for _ in ()).throw(SystemExit("Could not find exact litellm-proxy-extras pin in project.optional-dependencies.proxy")))') if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then echo "Error: Version in pyproject.toml ($CURRENT_VERSION) doesn't match new version ($NEW_VERSION)" exit 1 fi - # Check requirements.txt - REQ_VERSION=$(grep -oP 'litellm-proxy-extras==\K[0-9.]+' requirements.txt) - if [ "$REQ_VERSION" != "$NEW_VERSION" ]; then - echo "Error: Version in requirements.txt ($REQ_VERSION) doesn't match new version ($NEW_VERSION)" - exit 1 - fi - - run: name: Publish to PyPI command: | echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc - python -m pip install --upgrade pip build twine setuptools wheel + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" rm -rf build dist - python -m build - twine upload --verbose dist/* + uv build + uv tool run --from 'twine==6.2.0' twine upload --verbose dist/* ui_build: docker: @@ -3146,19 +3095,22 @@ jobs: - setup_google_dns - restore_cache: keys: - - ui-e2e-py-deps-v1-{{ checksum "requirements.txt" }} + - ui-e2e-py-deps-v2-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Install Python dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "prisma==0.11.0" - prisma generate --schema litellm/proxy/schema.prisma + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + uv sync --frozen --all-groups --all-extras --python "$(which python)" + uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma - save_cache: - key: ui-e2e-py-deps-v1-{{ checksum "requirements.txt" }} + key: ui-e2e-py-deps-v2-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} paths: - - ~/.local/lib - - ~/.local/bin + - ./.venv - restore_cache: keys: - ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} @@ -3182,12 +3134,12 @@ jobs: find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" done - - run: - name: Wait for PostgreSQL - command: dockerize -wait tcp://localhost:5432 -timeout 30s + - wait_for_service: + url: tcp://localhost:5432 + timeout: "30" - run: name: Push Prisma schema - command: prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - run: name: Seed database command: | @@ -3195,7 +3147,7 @@ jobs: -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql - run: name: Start mock LLM server - command: python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py + command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py background: true - run: name: Start LiteLLM proxy @@ -3206,7 +3158,7 @@ jobs: SERVER_ROOT_PATH: "" PROXY_LOGOUT_URL: "" command: | - python -m litellm.proxy.proxy_cli \ + uv run --no-sync python -m litellm.proxy.proxy_cli \ --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \ --port 4000 background: true @@ -3272,12 +3224,6 @@ jobs: - setup_google_dns - attach_workspace: at: ~/project - - run: - name: Install dockerize - command: | - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start PostgreSQL Database command: | @@ -3288,9 +3234,9 @@ jobs: -e POSTGRES_DB=litellm_schema_sync \ -p 5432:5432 \ postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" - run: name: Load Docker Database Image command: | @@ -3314,9 +3260,9 @@ jobs: name: Start outputting logs command: docker logs -f schema-sync background: true - - run: - name: Wait for proxy to be ready (schema sync complete) - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Stop schema sync container command: docker stop schema-sync @@ -3332,12 +3278,6 @@ jobs: - attach_workspace: at: ~/project - setup_google_dns - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start PostgreSQL Database command: | @@ -3348,9 +3288,9 @@ jobs: -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" - run: name: Load Docker Database Image command: | diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt deleted file mode 100644 index be12ab2d0f1..00000000000 --- a/.circleci/requirements.txt +++ /dev/null @@ -1,21 +0,0 @@ -# used by CI/CD testing -openai==1.100.1 -python-dotenv -tiktoken -importlib_metadata -cohere -redis==5.2.1 -redisvl==0.4.1 -anthropic -orjson==3.10.15 # fast /embedding responses -pydantic==2.12.5 -google-cloud-aiplatform==1.133.0 -google-cloud-iam==2.19.1 -fastapi-sso==0.16.0 -uvloop==0.21.0 -mcp==1.26.0 # for MCP server -semantic_router==0.1.10 # for auto-routing with litellm -fastuuid==0.14.0 -responses==0.25.7 # for proxy client tests -pytest-retry==1.6.3 # for automatic test retries -litellm-proxy-extras # for prisma migrations \ No newline at end of file diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index 484baa9041d..78f857d55d6 100644 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -1,17 +1,17 @@ #!/usr/bin/env bash set -e -echo "[post-create] Installing poetry via pip" -python -m pip install --upgrade pip -python -m pip install poetry +echo "[post-create] Installing uv" +curl -LsSf https://astral.sh/uv/0.10.9/install.sh | env UV_NO_MODIFY_PATH=1 sh +export PATH="$HOME/.local/bin:$PATH" -echo "[post-create] Installing Python dependencies (poetry)" -poetry install --with dev --extras proxy +echo "[post-create] Installing Python dependencies (uv)" +uv sync --frozen --group proxy-dev --extra proxy echo "[post-create] Generating Prisma client" -poetry run prisma generate +uv run --no-sync prisma generate echo "[post-create] Installing npm dependencies" cd ui/litellm-dashboard && npm ci -echo "[post-create] Done" \ No newline at end of file +echo "[post-create] Done" diff --git a/.gitguardian.yaml b/.gitguardian.yaml index 1eeec0677af..2a16ffe0c52 100644 --- a/.gitguardian.yaml +++ b/.gitguardian.yaml @@ -37,7 +37,7 @@ secret: - "docs/**" - "**/*.md" - "**/*.lock" - - "poetry.lock" + - "uv.lock" - "package-lock.json" # Ignore security incidents with the SHA256 of the occurrence (false positives) diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index d8dec73c428..9377cbeb0ca 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -51,37 +51,30 @@ jobs: with: python-version: "3.12" - - name: Install Poetry - run: pip install 'poetry==2.3.2' + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - - name: Cache Poetry dependencies + - name: Cache uv dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | - ~/.cache/pypoetry - ~/.cache/pip + ~/.cache/uv .venv - key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} restore-keys: | - ${{ runner.os }}-poetry- + ${{ runner.os }}-uv- - name: Install dependencies run: | - poetry config virtualenvs.in-project true - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install google-genai==1.22.0 \ - google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 - - - name: Setup litellm-enterprise - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ + uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router - name: Generate Prisma client env: PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | - poetry run pip install nodejs-wheel-binaries==24.13.1 - poetry run prisma generate --schema litellm/proxy/schema.prisma + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - name: Run tests env: @@ -90,7 +83,7 @@ jobs: WORKERS: ${{ inputs.workers }} RERUNS: ${{ inputs.reruns }} run: | - poetry run pytest ${TEST_PATH:?} \ + uv run --no-sync pytest ${TEST_PATH:?} \ --tb=short -vv \ --maxfail="${MAX_FAILURES}" \ -n "${WORKERS}" \ diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml index ce4c048c624..8e0b3568aea 100644 --- a/.github/workflows/_test-unit-services-base.yml +++ b/.github/workflows/_test-unit-services-base.yml @@ -86,44 +86,37 @@ jobs: with: python-version: "3.12" - - name: Install Poetry - run: pip install 'poetry==2.3.2' + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - - name: Cache Poetry dependencies + - name: Cache uv dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | - ~/.cache/pypoetry - ~/.cache/pip + ~/.cache/uv .venv - key: ${{ runner.os }}-poetry-services-${{ hashFiles('poetry.lock') }} + key: ${{ runner.os }}-uv-services-${{ hashFiles('uv.lock') }} restore-keys: | - ${{ runner.os }}-poetry-services- + ${{ runner.os }}-uv-services- - name: Install dependencies run: | - poetry config virtualenvs.in-project true - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install google-genai==1.22.0 \ - google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 - - - name: Setup litellm-enterprise - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ + uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router - name: Generate Prisma client env: PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | - poetry run pip install nodejs-wheel-binaries==24.13.1 - poetry run prisma generate --schema litellm/proxy/schema.prisma + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - name: Run Prisma migrations if: ${{ inputs.enable-postgres }} env: DATABASE_URL: ${{ secrets.DATABASE_URL }} run: | - poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - name: Run tests env: @@ -134,7 +127,7 @@ jobs: DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }} run: | if [ "${WORKERS}" = "0" ]; then - poetry run pytest ${TEST_PATH:?} \ + uv run --no-sync pytest ${TEST_PATH:?} \ --tb=short -vv \ --maxfail="${MAX_FAILURES}" \ --reruns "${RERUNS}" \ @@ -144,7 +137,7 @@ jobs: --cov-report=xml:coverage.xml \ --cov-config=pyproject.toml else - poetry run pytest ${TEST_PATH:?} \ + uv run --no-sync pytest ${TEST_PATH:?} \ --tb=short -vv \ --maxfail="${MAX_FAILURES}" \ -n "${WORKERS}" \ diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml index 60e89936219..1c6c318c717 100644 --- a/.github/workflows/auto_update_price_and_context_window.yml +++ b/.github/workflows/auto_update_price_and_context_window.yml @@ -17,12 +17,13 @@ jobs: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: persist-credentials: false - - name: Install Dependencies - run: | - pip install 'aiohttp==3.13.3' + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - name: Update JSON Data run: | - python ".github/workflows/auto_update_price_and_context_window_file.py" + uv run --frozen --with 'aiohttp==3.13.3' python ".github/workflows/auto_update_price_and_context_window_file.py" - name: Create Pull Request run: | git add model_prices_and_context_window.json diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 52d64addea9..17efbf90339 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -34,13 +34,21 @@ jobs: with: python-version: "3.12" - - name: Install dependencies - run: | - pip install -e "." - pip install pytest pytest-codspeed==4.3.0 + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - name: Run benchmarks uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1 with: mode: simulation - run: pytest tests/benchmarks/ --codspeed + run: > + env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 + uv run --frozen --no-default-groups + --with pytest==8.3.5 + --with pytest-codspeed==4.3.0 + pytest + -p pytest_codspeed.plugin + tests/benchmarks/ + --codspeed diff --git a/.github/workflows/guard-main-branch.yml b/.github/workflows/guard-main-branch.yml new file mode 100644 index 00000000000..1c1ce0de079 --- /dev/null +++ b/.github/workflows/guard-main-branch.yml @@ -0,0 +1,42 @@ +name: Guard main branch + +on: + pull_request: + branches: + - main + merge_group: + +permissions: {} + +# DO NOT RENAME the job's `name:` — it is referenced by GitHub branch +# protection as a required status check on `main`. Renaming silently +# breaks the gate. +jobs: + guard: + name: Verify PR source branch + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - name: Reject merge_group events + if: github.event_name == 'merge_group' + run: | + echo "::error::Merge queue is not supported for main. Disable merge queue or update this guard." + exit 1 + - name: Check head branch name + env: + HEAD_REF: ${{ github.head_ref }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + BASE_REPO: ${{ github.repository }} + run: | + echo "PR head repo: $HEAD_REPO" + echo "PR head branch: $HEAD_REF" + if [ "$HEAD_REPO" != "$BASE_REPO" ]; then + echo "::error::PRs to main must originate from the canonical repository ($BASE_REPO), not a fork ($HEAD_REPO). External contributors should open PRs against the 'litellm_oss_branch' branch instead." + exit 1 + fi + if [ "$HEAD_REF" = "litellm_internal_staging" ] || [[ "$HEAD_REF" == litellm_hotfix_?* ]]; then + echo "Allowed source branch." + exit 0 + fi + echo "::error::PRs to main must originate from 'litellm_internal_staging' or a 'litellm_hotfix_*' branch. Got: '$HEAD_REF'. If this is a contribution, retarget the PR against 'litellm_oss_branch' instead." + exit 1 diff --git a/.github/workflows/llm-translation-testing.yml b/.github/workflows/llm-translation-testing.yml index 922013c4b54..93b69e5c6a9 100644 --- a/.github/workflows/llm-translation-testing.yml +++ b/.github/workflows/llm-translation-testing.yml @@ -31,26 +31,25 @@ jobs: with: python-version: "3.11" - - name: Install Poetry - run: | - pip install 'poetry==2.3.2' - poetry config virtualenvs.create true - poetry config virtualenvs.in-project true + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + enable-cache: false - - name: Restore Poetry dependencies cache - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 + - name: Restore uv dependencies cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | - ~/.cache/pypoetry + ~/.cache/uv .venv - key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} restore-keys: | - ${{ runner.os }}-poetry- + ${{ runner.os }}-uv- - name: Install dependencies run: | - poetry install --with dev - poetry run pip install 'pytest-xdist==3.8.0' 'pytest-timeout==2.4.0' + uv sync --frozen - name: Create test results directory run: mkdir -p test-results diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml index 8f675bb3075..d60254a0ac5 100644 --- a/.github/workflows/publish_to_pypi.yml +++ b/.github/workflows/publish_to_pypi.yml @@ -24,10 +24,22 @@ jobs: with: python-version: "3.12" + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + enable-cache: false + - name: Check litellm version on PyPI id: check-litellm run: | - VERSION=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/') + VERSION=$(python - <<'PY' + import tomllib + + with open("pyproject.toml", "rb") as f: + print(tomllib.load(f)["project"]["version"]) + PY + ) echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "Checking if litellm $VERSION exists on PyPI..." @@ -42,43 +54,46 @@ jobs: - name: Sanity check proxy-extras version run: | - # Read pinned version from requirements.txt - REQ_VERSION=$(grep -oP 'litellm-proxy-extras==\K[0-9.]+' requirements.txt) - if [ -z "$REQ_VERSION" ]; then - echo "::error::Could not find litellm-proxy-extras version in requirements.txt" - exit 1 - fi - echo "requirements.txt pins litellm-proxy-extras==$REQ_VERSION" + # Read pinned version from project optional dependencies + PYPROJECT_VERSION=$(python3 - <<'PY' + import sys + import tomllib - # Read pinned version from pyproject.toml dependency - PYPROJECT_VERSION=$(python3 -c " - import re - with open('pyproject.toml') as f: - content = f.read() - match = re.search(r'litellm-proxy-extras\s*=\s*\{version\s*=\s*\"([^\"]+)\"', content) - if match: - print(match.group(1).lstrip('^~>=')) - else: - import sys - print('::error::Could not find litellm-proxy-extras dependency in pyproject.toml', file=sys.stderr) + with open("pyproject.toml", "rb") as f: + proxy_requirements = tomllib.load(f)["project"]["optional-dependencies"]["proxy"] + + version = None + for requirement in proxy_requirements: + normalized = requirement.split(";", 1)[0].strip() + if not normalized.startswith("litellm-proxy-extras"): + continue + parts = normalized.split("==", 1) + if len(parts) == 2 and parts[0].strip() == "litellm-proxy-extras": + candidate = parts[1].strip() + if candidate: + version = candidate + break + + if version is None: + print( + "::error::Could not find an exact litellm-proxy-extras pin in project.optional-dependencies.proxy", + file=sys.stderr, + ) sys.exit(1) - ") + + print(version) + PY + ) echo "pyproject.toml pins litellm-proxy-extras version: $PYPROJECT_VERSION" - # Check that both pinned versions match - if [ "$REQ_VERSION" != "$PYPROJECT_VERSION" ]; then - echo "::error::Version mismatch: requirements.txt has $REQ_VERSION but pyproject.toml has $PYPROJECT_VERSION" - exit 1 - fi - # Check that the pinned version exists on PyPI - echo "Checking if litellm-proxy-extras $REQ_VERSION exists on PyPI..." - HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm-proxy-extras/$REQ_VERSION/json") + echo "Checking if litellm-proxy-extras $PYPROJECT_VERSION exists on PyPI..." + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm-proxy-extras/$PYPROJECT_VERSION/json") if [ "$HTTP_STATUS" != "200" ]; then - echo "::error::litellm-proxy-extras $REQ_VERSION is not published on PyPI yet. Publish it before releasing litellm." + echo "::error::litellm-proxy-extras $PYPROJECT_VERSION is not published on PyPI yet. Publish it before releasing litellm." exit 1 fi - echo "litellm-proxy-extras $REQ_VERSION exists on PyPI. Sanity check passed." + echo "litellm-proxy-extras $PYPROJECT_VERSION exists on PyPI. Sanity check passed." publish-litellm: name: Publish litellm to PyPI @@ -100,16 +115,19 @@ jobs: with: python-version: "3.12" + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + enable-cache: false + - name: Copy model prices backup run: cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json - - name: Install build tools - run: python -m pip install --upgrade pip build==1.4.2 - - name: Build package run: | rm -rf build dist - python -m build + uv build - name: Verify build artifacts env: @@ -129,8 +147,7 @@ jobs: - name: Validate package metadata run: | - pip install twine==6.2.0 - twine check dist/* + uv tool run --from 'twine==6.2.0' twine check dist/* - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 diff --git a/.github/workflows/run_llm_translation_tests.py b/.github/workflows/run_llm_translation_tests.py index 5b3a4817ecb..3f3a70efe92 100644 --- a/.github/workflows/run_llm_translation_tests.py +++ b/.github/workflows/run_llm_translation_tests.py @@ -325,7 +325,7 @@ def run_tests(test_path: str = "tests/llm_translation/", # Run pytest cmd = [ - "poetry", "run", "pytest", test_path, + "uv", "run", "--no-sync", "pytest", test_path, f"--junitxml={junit_xml}", "-v", "--tb=short", @@ -335,7 +335,7 @@ def run_tests(test_path: str = "tests/llm_translation/", # Add timeout if pytest-timeout is installed try: - subprocess.run(["poetry", "run", "python", "-c", "import pytest_timeout"], + subprocess.run(["uv", "run", "--no-sync", "python", "-c", "import pytest_timeout"], capture_output=True, check=True) cmd.extend(["--timeout=300"]) except: @@ -436,4 +436,4 @@ if __name__ == "__main__": commit=args.commit ) - sys.exit(exit_code) \ No newline at end of file + sys.exit(exit_code) diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 5bb85716a17..eefa42e7fa2 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -24,26 +24,28 @@ jobs: with: python-version: "3.12" - - name: Install Poetry - run: pip install 'poetry==2.3.2' + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - name: Clean Python cache run: | find . -type d -name "__pycache__" -exec rm -rf {} + || true find . -name "*.pyc" -delete || true - - name: Check poetry.lock is up to date + - name: Check uv.lock is up to date run: | - poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1) + uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1) - name: Install dependencies run: | - poetry install --with dev + uv sync --frozen - name: Check Black formatting run: | cd litellm - poetry run black --check --exclude '/enterprise/' . + uv run --no-sync black --check --exclude '/enterprise/' . cd .. - name: Debug - Check file state @@ -58,28 +60,28 @@ jobs: - name: Run Ruff linting run: | cd litellm - poetry run ruff check . + uv run --no-sync ruff check . cd .. - name: Print OpenAI version run: | - poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')" + uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')" - name: Run MyPy type checking run: | cd litellm - poetry run mypy . + uv run --no-sync mypy . cd .. - name: Check for circular imports run: | cd litellm - poetry run python ../tests/documentation_tests/test_circular_imports.py + uv run --no-sync python ../tests/documentation_tests/test_circular_imports.py cd .. - name: Check import safety run: | - poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) + uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) secret-scan: runs-on: ubuntu-latest @@ -98,18 +100,21 @@ jobs: with: python-version: "3.12" + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + - name: Run secret scan test run: | - pip install 'pytest==9.0.2' - pytest tests/litellm/test_no_hardcoded_secrets.py -v + uv run --frozen --with 'pytest==9.0.2' pytest tests/litellm/test_no_hardcoded_secrets.py -v - name: Run ggshield secret scan env: GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} run: | if [ -n "$GITGUARDIAN_API_KEY" ]; then - pip install 'ggshield==1.48.0' - ggshield secret scan repo . + uv tool run --from 'ggshield==1.48.0' ggshield secret scan repo . else echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan" fi diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml index 0c040b3ebe7..938647f5d0c 100644 --- a/.github/workflows/test-litellm.yml +++ b/.github/workflows/test-litellm.yml @@ -31,23 +31,15 @@ jobs: with: python-version: "3.12" - - name: Install Poetry - run: pip install 'poetry==2.3.2' + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - name: Install dependencies run: | - poetry lock - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install "pytest-retry==1.6.3" - poetry run pip install 'pytest-xdist==3.8.0' - poetry run pip install "google-genai==1.22.0" - poetry run pip install "google-cloud-aiplatform==1.115.0" - poetry run pip install "fastapi-offline==1.7.3" - poetry run pip install "python-multipart==0.0.22" - poetry run pip install "openapi-core==0.23.0" - - name: Setup litellm-enterprise as local package - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ + uv lock --check + uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router - name: Run tests run: | - poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50 + uv run --no-sync pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50 diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml index 1b228ab76bb..11c5441bf9c 100644 --- a/.github/workflows/test-mcp.yml +++ b/.github/workflows/test-mcp.yml @@ -27,26 +27,16 @@ jobs: with: python-version: "3.12" - - name: Install Poetry - run: pip install 'poetry==2.3.2' + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - name: Install dependencies run: | - poetry lock - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install "pytest==7.3.1" - poetry run pip install "pytest-retry==1.6.3" - poetry run pip install "pytest-cov==5.0.0" - poetry run pip install "pytest-asyncio==0.21.1" - poetry run pip install "respx==0.22.0" - poetry run pip install "pydantic==2.11.0" - poetry run pip install "mcp==1.25.0" - poetry run pip install 'pytest-xdist==3.8.0' - - - name: Setup litellm-enterprise as local package - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ + uv lock --check + uv sync --frozen --group proxy-dev --extra proxy --extra semantic-router - name: Run MCP tests run: | - poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5 + uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5 diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml index d8b30de6844..8440c53f9f5 100644 --- a/.github/workflows/test-unit-documentation.yml +++ b/.github/workflows/test-unit-documentation.yml @@ -26,42 +26,35 @@ jobs: with: python-version: "3.12" - - name: Install Poetry - run: pip install 'poetry==2.3.2' + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - - name: Cache Poetry dependencies + - name: Cache uv dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | - ~/.cache/pypoetry - ~/.cache/pip + ~/.cache/uv .venv - key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} restore-keys: | - ${{ runner.os }}-poetry- + ${{ runner.os }}-uv- - name: Install dependencies run: | - poetry config virtualenvs.in-project true - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install google-genai==1.22.0 \ - google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 - - - name: Setup litellm-enterprise - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ + uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router - name: Generate Prisma client env: PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | - poetry run pip install nodejs-wheel-binaries==24.13.1 - poetry run prisma generate --schema litellm/proxy/schema.prisma + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma # Run the same documentation tests that CircleCI ran (as direct Python scripts) - name: Run documentation validation tests run: | - poetry run python ./tests/documentation_tests/test_env_keys.py - poetry run python ./tests/documentation_tests/test_router_settings.py - poetry run python ./tests/documentation_tests/test_api_docs.py - poetry run python ./tests/documentation_tests/test_circular_imports.py + uv run --no-sync python ./tests/documentation_tests/test_env_keys.py + uv run --no-sync python ./tests/documentation_tests/test_router_settings.py + uv run --no-sync python ./tests/documentation_tests/test_api_docs.py + uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml index 49d399e8741..35d3c018a49 100644 --- a/.github/workflows/test-unit-proxy-db.yml +++ b/.github/workflows/test-unit-proxy-db.yml @@ -34,7 +34,7 @@ jobs: - test-group: remaining test-path: "tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py" workers: 8 - timeout: 20 + timeout: 30 uses: ./.github/workflows/_test-unit-services-base.yml with: test-path: ${{ matrix.test-path }} diff --git a/.github/workflows/test-unit-proxy-legacy.yml b/.github/workflows/test-unit-proxy-legacy.yml index a9391137263..d4f5c38a61c 100644 --- a/.github/workflows/test-unit-proxy-legacy.yml +++ b/.github/workflows/test-unit-proxy-legacy.yml @@ -50,43 +50,36 @@ jobs: with: python-version: "3.12" - - name: Install Poetry - run: pip install 'poetry==2.3.2' + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - - name: Cache Poetry dependencies + - name: Cache uv dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | - ~/.cache/pypoetry - ~/.cache/pip + ~/.cache/uv .venv - key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} restore-keys: | - ${{ runner.os }}-poetry- + ${{ runner.os }}-uv- - name: Install dependencies run: | - poetry config virtualenvs.in-project true - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install google-genai==1.22.0 \ - google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 - - - name: Setup litellm-enterprise - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ + uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router - name: Generate Prisma client env: PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | - poetry run pip install nodejs-wheel-binaries==24.13.1 - poetry run prisma generate --schema litellm/proxy/schema.prisma + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - name: Run tests - ${{ matrix.test-group.name }} env: TEST_PATH: ${{ matrix.test-group.path }} run: | - poetry run pytest ${TEST_PATH} \ + uv run --no-sync pytest ${TEST_PATH} \ --tb=short -vv \ --maxfail=10 \ -n 2 \ diff --git a/.github/workflows/test_server_root_path.yml b/.github/workflows/test_server_root_path.yml index 47636ce8e92..58e3a417091 100644 --- a/.github/workflows/test_server_root_path.yml +++ b/.github/workflows/test_server_root_path.yml @@ -9,7 +9,7 @@ on: jobs: test-server-root-path: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 30 strategy: matrix: @@ -21,6 +21,12 @@ jobs: with: persist-credentials: false + - name: Free up disk space + run: | + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/share/boost + sudo apt-get clean + df -h / + - name: Set up Docker Buildx uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12 diff --git a/AGENTS.md b/AGENTS.md index 37411938e2f..0d898fc6d56 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,7 +51,9 @@ LiteLLM is a unified interface for 100+ LLMs that: ### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND) -1. **Tremor is DEPRECATED, do not use Tremor components in new features/changes** +1. **Always use `antd` for new UI components — Tremor is DEPRECATED** + - We are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. + - Use `antd` equivalents: `Tag` for labels, plain ``/`
` with Tailwind classes (or `Typography.Text`) for text, `Card` from `antd`, etc. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow. - The only exception is the Tremor Table component and its required Tremor Table sub components. 2. **Use Common Components as much as possible**: @@ -121,7 +123,7 @@ LiteLLM supports MCP for agent workflows: ## RUNNING SCRIPTS -Use `poetry run python script.py` to run Python scripts in the project environment (for non-test files). +Use `uv run python script.py` to run Python scripts in the project environment (for non-test files). ## GITHUB TEMPLATES @@ -232,16 +234,16 @@ When opening issues or pull requests, follow these templates: ### Environment -- Poetry is installed in `~/.local/bin`; the update script ensures it is on `PATH`. +- uv is installed in `~/.local/bin`; the update script ensures it is on `PATH`. - Python 3.12, Node 22 are pre-installed. -- The virtual environment lives under `~/.cache/pypoetry/virtualenvs/`. +- The project virtual environment lives under `.venv/`. ### Running the proxy server Start the proxy with a config file: ```bash -poetry run litellm --config dev_config.yaml --port 4000 +uv run litellm --config dev_config.yaml --port 4000 ``` The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package. @@ -250,17 +252,16 @@ The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot See `CLAUDE.md` and the `Makefile` for standard commands. Key notes: -- `psycopg-binary` must be installed (`poetry run pip install psycopg-binary`) because the pytest-postgresql plugin requires it and the lock file only includes `psycopg` (no binary). -- `openapi-core` must be installed (`poetry run pip install openapi-core`) for the OpenAPI compliance tests in `tests/test_litellm/interactions/`. +- `uv sync --group proxy-dev --extra proxy` installs the Prisma and proxy-side test dependencies used by the standard local workflow. - The `--timeout` pytest flag is NOT available; don't pass it. -- Unit tests: `poetry run pytest tests/test_litellm/ -x -vv -n 4` -- **Before committing, always run `poetry run black .` to format your code.** Black formatting is enforced in CI. -- If `poetry install` fails with "pyproject.toml changed significantly since poetry.lock was last generated", run `poetry lock` first to regenerate the lock file. +- Unit tests: `uv run pytest tests/test_litellm/ -x -vv -n 4` +- **Before committing, always run `uv run black .` to format your code.** Black formatting is enforced in CI. +- If `uv sync` fails because the lockfile is outdated, run `uv lock` and retry. ### Lint ```bash -cd litellm && poetry run ruff check . +cd litellm && uv run ruff check . ``` Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`. @@ -271,4 +272,4 @@ Ruff is the primary fast linter. For the full lint suite (including mypy, black, - The proxy at port 4000 serves a **pre-built** static UI from `litellm/proxy/_experimental/out/`. After making UI code changes, you must run `npm run build` in the dashboard directory and copy the output: `cp -r ui/litellm-dashboard/out/* litellm/proxy/_experimental/out/` for the proxy to serve the updated UI. - SVGs used as provider logos (loaded via `` tags) must NOT use `fill="currentColor"` — replace with an explicit color like `#000000` or use the `-color` variant from lobehub icons, since CSS color inheritance does not work inside `` elements. - Provider logos live in `ui/litellm-dashboard/public/assets/logos/` (source) and `litellm/proxy/_experimental/out/assets/logos/` (pre-built). Both locations must have the file for it to work in dev and proxy-served modes. -- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run` \ No newline at end of file +- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run` diff --git a/CLAUDE.md b/CLAUDE.md index a8800ff8884..043055408c2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,7 +7,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ### Installation - `make install-dev` - Install core development dependencies - `make install-proxy-dev` - Install proxy development dependencies with full feature set -- `make install-test-deps` - Install all test dependencies +- `make install-test-deps` - Install the full local test environment and generate the Prisma client ### Testing - `make test` - Run all tests @@ -20,14 +20,14 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co - `make format` - Apply Black code formatting - `make lint-ruff` - Run Ruff linting only - `make lint-mypy` - Run MyPy type checking only -- **Before committing, always run `poetry run black .` to format your code.** Black formatting is enforced in CI. +- **Before committing, always run `uv run black .` to format your code.** Black formatting is enforced in CI. ### Single Test Files -- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file -- `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test +- `uv run pytest tests/path/to/test_file.py -v` - Run specific test file +- `uv run pytest tests/path/to/test_file.py::test_function -v` - Run specific test ### Running Scripts -- `poetry run python script.py` - Run Python scripts (use for non-test files) +- `uv run python script.py` - Run Python scripts (use for non-test files) ### GitHub Issue & PR Templates When contributing to the project, use the appropriate templates: @@ -109,6 +109,9 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components: ### UI / Backend Consistency - When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select +### UI Component Library +- **Always use `antd` for new UI components** — we are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. Use `antd` equivalents: `Tag` for labels, plain ``/`
` with Tailwind classes (or `Typography.Text`) for text, `Card` from `antd`, etc. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow. + ### MCP OAuth / OpenAPI Transport Mapping - `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls). - FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback. @@ -151,6 +154,14 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components: - Optional features enabled via environment variables - Separate licensing and authentication for enterprise features +### CI Supply-Chain Safety +- **Never pipe a remote script into a shell** (`curl ... | bash`, `wget ... | sh`). Download the artifact to a file, verify its SHA-256 checksum, then install. +- **Pin every external tool to a specific version** with a full URL (not `latest` or `stable`). Unversioned downloads silently change under you. +- **Verify checksums for all downloaded binaries.** Use the provider's official `.sha256` / `.sha256sum` sidecar file when available; otherwise compute and hardcode the digest. +- **Prefer reusable CircleCI commands** (`commands:` section) so a tool is installed and verified in exactly one place, then referenced everywhere with `- install_` or `- wait_for_service`. +- **Don't add tools just because they were there before.** Audit whether an external dependency is still needed. If it can be replaced with a shell one-liner or a tool already in the image, remove it. +- These rules apply to every download in CI: binaries, install scripts, language version managers, package repos. No exceptions. + ### HTTP Client Cache Safety - **Never close HTTP/SDK clients on cache eviction.** `LLMClientCache._remove_key()` must not call `close()`/`aclose()` on evicted clients — they may still be used by in-flight requests. Doing so causes `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expires. Cleanup happens at shutdown via `close_litellm_async_clients()`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c029ccce1ab..8ac83341f64 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -122,9 +122,17 @@ Run all unit tests (uses parallel execution for speed): make test-unit ``` +If you're running broader test suites, proxy tests, or anything that touches PostgreSQL-backed fixtures/plugins, install the full local test environment first: + +```bash +make install-test-deps +``` + +This syncs the locked test environment used across the repo, including `psycopg` v3 plus `psycopg-binary` (used by `pytest-postgresql`), `psycopg2-binary` (used by some proxy E2E tests), and a generated Prisma client for DB-backed proxy tests, so pytest startup matches CI without manual package installs. + Run specific test files: ```bash -poetry run pytest tests/test_litellm/test_your_file.py -v +uv run pytest tests/test_litellm/test_your_file.py -v ``` ### Running Linting and Formatting Checks @@ -185,7 +193,7 @@ Run `make help` to see all available commands: make help # Show all available commands make install-dev # Install development dependencies make install-proxy-dev # Install proxy development dependencies -make install-test-deps # Install test dependencies (for running tests) +make install-test-deps # Install the full local test environment make format # Apply Black code formatting make format-check # Check Black formatting (matches CI) make lint # Run all linting checks @@ -247,7 +255,7 @@ To run the proxy server locally: make install-proxy-dev # Start the proxy server -poetry run litellm --config your_config.yaml +uv run litellm --config your_config.yaml ``` ### Docker Development @@ -332,4 +340,4 @@ Looking for ideas? Check out: - 🧪 Test coverage improvements - 🔌 New LLM provider integrations -Thank you for contributing to LiteLLM! 🚀 \ No newline at end of file +Thank you for contributing to LiteLLM! 🚀 diff --git a/Dockerfile b/Dockerfile index f4cb501ad8b..a2cd1cb3ed2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,57 +3,75 @@ ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92 # Runtime image ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502 +ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9@sha256:10902f58a1606787602f303954cea099626a4adb02acbac4c69920fe9d278f82 + +FROM $UV_IMAGE AS uvbin # Builder stage FROM $LITELLM_BUILD_IMAGE AS builder -# Set the working directory to /app WORKDIR /app - USER root -# Install build dependencies -RUN apk add --no-cache bash gcc py3-pip python3 python3-dev openssl openssl-dev +COPY --from=uvbin /uv /usr/local/bin/uv +COPY --from=uvbin /uvx /usr/local/bin/uvx -RUN python -m pip install build==1.4.2 +RUN apk add --no-cache \ + bash \ + gcc \ + python3 \ + python3-dev \ + openssl \ + openssl-dev \ + nodejs \ + npm \ + libsndfile -# Copy the current directory contents into the container at /app +ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ + UV_PROJECT_ENVIRONMENT=/app/.venv \ + UV_LINK_MODE=copy \ + XDG_CACHE_HOME=/app/.cache \ + PATH="/app/.venv/bin:${PATH}" + +# Copy dependency metadata first for layer caching +COPY pyproject.toml uv.lock ./ +COPY enterprise/pyproject.toml enterprise/ +COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ + +# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) +RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ + --extra proxy \ + --extra proxy-runtime \ + --extra extra_proxy \ + --extra semantic-router \ + --python python3 + +# Copy full source tree COPY . . -# Build Admin UI -# Convert Windows line endings to Unix and make executable +# Build Admin UI before final sync RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh -# Build the package -RUN rm -rf dist/* && python -m build +# Install project and workspace packages (fast - deps already cached) +RUN uv sync --frozen --no-default-groups --no-editable \ + --extra proxy \ + --extra proxy-runtime \ + --extra extra_proxy \ + --extra semantic-router \ + --python python3 -# There should be only one wheel file now, assume the build only creates one -RUN ls -1 dist/*.whl | head -1 +RUN prisma generate --schema=./schema.prisma -# Install the package -RUN pip install dist/*.whl - -# install dependencies as wheels -RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt - -# ensure pyjwt is used, not jwt -RUN pip uninstall jwt -y -RUN pip uninstall PyJWT -y -RUN pip install PyJWT==2.12.0 --no-cache-dir +RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ + sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh # Runtime stage FROM $LITELLM_RUNTIME_IMAGE AS runtime -# Ensure runtime stage runs as root USER root -# Install runtime dependencies (libsndfile needed for audio processing on ARM64) -RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \ +RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \ npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ - # SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested - # levels inside its dependency tree. `npm install -g ` only creates a - # SEPARATE global package, it does NOT replace npm's internal copies. - # We must find and replace EVERY copy inside npm's directory. GLOBAL="$(npm root -g)" && \ find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ @@ -70,73 +88,24 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ done && \ - # SECURITY FIX: patch npm's own package.json metadata so scanners see the - # actual installed versions instead of the stale declared dependencies. find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \ sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \ npm cache clean --force && \ - # Remove the apk-tracked npm so its stale SBOM metadata (tar 7.5.9) is - # no longer visible to image scanners. The globally installed npm@latest - # at /usr/local/lib/node_modules/npm/ remains fully functional. { apk del --no-cache npm 2>/dev/null || true; } WORKDIR /app -# Copy the current directory contents into the container at /app -COPY . . -RUN ls -la /app +ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ + XDG_CACHE_HOME=/app/.cache \ + PATH="/app/.venv/bin:${PATH}" -# Copy the built wheel from the builder stage to the runtime stage; assumes only one wheel file is present -COPY --from=builder /app/dist/*.whl . -COPY --from=builder /wheels/ /wheels/ +COPY --from=builder /app /app -# Install the built wheel using pip; again using a wildcard if it's the only file -RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && rm -f *.whl && rm -rf /wheels - -# Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130) -RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \ - if [ -n "$NODEJS_WHEEL_NODE" ]; then cp /usr/bin/node "$NODEJS_WHEEL_NODE"; fi - -# Remove test files and keys from dependencies -RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \ - find /usr/lib -type d -path "*/tornado/test" -delete - -# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete -# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/. -# Patch every copy of tar, glob, and brace-expansion inside that tree. -RUN GLOBAL="$(npm root -g)" && \ - [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \ - find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ - done && \ - find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ - done && \ - find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ - done && \ - find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ - done && \ - find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ - done - -# Install semantic_router and aurelio-sdk using script -# Convert Windows line endings to Unix and make executable -RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh - -# Generate prisma client using the correct schema -RUN prisma generate --schema=./litellm/proxy/schema.prisma -# Convert Windows line endings to Unix for entrypoint scripts -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh -RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh +RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ + find /app/.venv -type d -path "*/tornado/test" -delete EXPOSE 4000/tcp -RUN apk add --no-cache supervisor COPY docker/supervisord.conf /etc/supervisord.conf ENTRYPOINT ["docker/prod_entrypoint.sh"] - -# Append "--detailed_debug" to the end of CMD to view detailed debug logs CMD ["--port", "4000"] diff --git a/GEMINI.md b/GEMINI.md index a9d40c910b2..9e950d89b33 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -22,11 +22,11 @@ This file provides guidance to Gemini when working with code in this repository. - `make lint-mypy` - Run MyPy type checking only ### Single Test Files -- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file -- `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test +- `uv run pytest tests/path/to/test_file.py -v` - Run specific test file +- `uv run pytest tests/path/to/test_file.py::test_function -v` - Run specific test ### Running Scripts -- `poetry run python script.py` - Run Python scripts (use for non-test files) +- `uv run python script.py` - Run Python scripts (use for non-test files) ### GitHub Issue & PR Templates When contributing to the project, use the appropriate templates: @@ -105,4 +105,4 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components: ### Enterprise Features - Enterprise-specific code in `enterprise/` directory - Optional features enabled via environment variables -- Separate licensing and authentication for enterprise features \ No newline at end of file +- Separate licensing and authentication for enterprise features diff --git a/Makefile b/Makefile index 74031f418d6..b6b674ff3b1 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ help: @echo " make install-proxy-dev - Install proxy development dependencies" @echo " make install-dev-ci - Install dev dependencies (CI-compatible, pins OpenAI)" @echo " make install-proxy-dev-ci - Install proxy dev dependencies (CI-compatible)" - @echo " make install-test-deps - Install test dependencies" + @echo " make install-test-deps - Install the full local test environment" @echo " make install-helm-unittest - Install helm unittest plugin" @echo " make format - Apply Black code formatting" @echo " make format-check - Check Black code formatting (matches CI)" @@ -40,49 +40,44 @@ help: @echo " make test-integration - Run integration tests" @echo " make test-unit-helm - Run helm unit tests" -# Keep PIP simple for edge cases: -PIP := $(shell command -v pip > /dev/null 2>&1 && echo "pip" || echo "python3 -m pip") +UV := uv +UV_RUN := $(UV) run --no-sync # Show info info: - @echo "PIP: $(PIP)" + @echo "UV: $(UV)" # Installation targets install-dev: - poetry install --with dev + $(UV) sync --frozen install-proxy-dev: - poetry install --with dev,proxy-dev --extras proxy + $(UV) sync --frozen --group proxy-dev --extra proxy # CI-compatible installations (matches GitHub workflows exactly) install-dev-ci: - $(PIP) install openai==2.8.0 - poetry install --with dev - $(PIP) install openai==2.8.0 + $(UV) sync --frozen install-proxy-dev-ci: - poetry install --with dev,proxy-dev --extras proxy - $(PIP) install openai==2.8.0 + $(UV) sync --frozen --group proxy-dev --extra proxy install-test-deps: install-proxy-dev - poetry run $(PIP) install "pytest-retry==1.6.3" - poetry run $(PIP) install pytest-xdist - poetry run $(PIP) install openapi-core - cd enterprise && poetry run $(PIP) install -e . && cd .. + $(UV) sync --frozen --all-groups --all-extras + $(UV_RUN) prisma generate --schema litellm/proxy/schema.prisma install-helm-unittest: helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists" # Formatting format: install-dev - cd litellm && poetry run black . && cd .. + cd litellm && $(UV_RUN) black . && cd .. format-check: install-dev - cd litellm && poetry run black --check . && cd .. + cd litellm && $(UV_RUN) black --check . && cd .. # Linting targets lint-ruff: install-dev - cd litellm && poetry run ruff check . && cd .. + cd litellm && $(UV_RUN) ruff check . && cd .. # faster linter for developing ... # inspiration from: @@ -96,37 +91,36 @@ lint-format-changed: install-dev $$start = $$1; $$count = $$2 || 1; $$end = $$start + $$count - 1; \ print "$$file:$$start:1-$$end:999\n"; \ }' | \ - while read range; do \ - file="$${range%%:*}"; \ - lines="$${range#*:}"; \ - echo "Formatting $$file (lines $$lines)"; \ - poetry run ruff format --range "$$lines" "$$file"; \ - done + while read range; do \ + file="$${range%%:*}"; \ + lines="$${range#*:}"; \ + echo "Formatting $$file (lines $$lines)"; \ + $(UV_RUN) ruff format --range "$$lines" "$$file"; \ + done lint-ruff-dev: install-dev @tmpfile=$$(mktemp /tmp/ruff-dev.XXXXXX) && \ cd litellm && \ - (poetry run ruff check . --output-format=pylint || true) > "$$tmpfile" && \ - poetry run diff-quality --violations=pylint "$$tmpfile" --compare-branch=origin/main && \ + ($(UV_RUN) ruff check . --output-format=pylint || true) > "$$tmpfile" && \ + $(UV_RUN) diff-quality --violations=pylint "$$tmpfile" --compare-branch=origin/main && \ cd .. ; \ rm -f "$$tmpfile" lint-ruff-FULL-dev: install-dev @files=$$(git diff --name-only origin/main -- '*.py'); \ - if [ -n "$$files" ]; then echo "$$files" | xargs poetry run ruff check; \ + if [ -n "$$files" ]; then echo "$$files" | xargs $(UV_RUN) ruff check; \ else echo "No changed .py files to check."; fi lint-mypy: install-dev - poetry run $(PIP) install types-requests types-setuptools types-redis types-PyYAML - cd litellm && poetry run mypy . --ignore-missing-imports && cd .. + cd litellm && $(UV_RUN) mypy . --ignore-missing-imports && cd .. lint-black: format-check check-circular-imports: install-dev - cd litellm && poetry run python ../tests/documentation_tests/test_circular_imports.py && cd .. + cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd .. check-import-safety: install-dev - @poetry run python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) + @$(UV_RUN) python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) # Combined linting (matches test-linting.yml workflow) lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety @@ -135,46 +129,46 @@ lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safet lint-dev: lint-format-changed lint-mypy check-circular-imports check-import-safety # Testing targets -test: - poetry run pytest tests/ +test: install-test-deps + $(UV_RUN) pytest tests/ test-unit: install-test-deps - poetry run pytest tests/test_litellm -x -vv -n 4 + $(UV_RUN) pytest tests/test_litellm -x -vv -n 4 # Matrix test targets (matching CI workflow groups) test-unit-llms: install-test-deps - poetry run pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20 test-unit-proxy-guardrails: install-test-deps - poetry run pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20 test-unit-proxy-core: install-test-deps - poetry run pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20 test-unit-proxy-misc: install-test-deps - poetry run pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20 test-unit-integrations: install-test-deps - poetry run pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20 test-unit-core-utils: install-test-deps - poetry run pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20 + $(UV_RUN) pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20 test-unit-other: install-test-deps - poetry run pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20 test-unit-root: install-test-deps - poetry run pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20 # Proxy unit tests (tests/proxy_unit_tests split alphabetically) test-proxy-unit-a: install-test-deps - poetry run pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20 + $(UV_RUN) pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20 test-proxy-unit-b: install-test-deps - poetry run pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20 + $(UV_RUN) pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20 -test-integration: - poetry run pytest tests/ -k "not test_litellm" +test-integration: install-test-deps + $(UV_RUN) pytest tests/ -k "not test_litellm" test-unit-helm: install-helm-unittest helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm @@ -188,6 +182,6 @@ test-llm-translation-single: install-test-deps @echo "Running single LLM translation test file..." @if [ -z "$(FILE)" ]; then echo "Usage: make test-llm-translation-single FILE=test_filename.py"; exit 1; fi @mkdir -p test-results - poetry run pytest tests/llm_translation/$(FILE) \ + $(UV_RUN) pytest tests/llm_translation/$(FILE) \ --junitxml=test-results/junit.xml \ -v --tb=short --maxfail=100 --timeout=300 diff --git a/README.md b/README.md index 894b4bcb054..2c109dabf8c 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,45 @@ Group 7154 (1) +--- -## Use LiteLLM for +## What is LiteLLM + +LiteLLM is an open source AI Gateway that gives you a single, unified interface to call 100+ LLM providers — OpenAI, Anthropic, Gemini, Bedrock, Azure, and more — using the OpenAI format. + +Use it as a **Python SDK** for direct library integration, or deploy the **AI Gateway (Proxy Server)** as a centralized service for your team or organization. + +[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://docs.litellm.ai/docs/simple_proxy)
+[**Jump to Supported LLM Providers**](https://docs.litellm.ai/docs/providers) + +--- + +## Why LiteLLM + +Managing LLM calls across providers gets complicated fast — different SDKs, auth patterns, request formats, and error types for every model. LiteLLM removes that friction: + +- **Unified API** — one interface for 100+ LLMs, no provider-specific SDK juggling +- **Drop-in OpenAI compatibility** — swap providers without rewriting your code +- **Production-ready gateway** — virtual keys, spend tracking, guardrails, load balancing, and an admin dashboard out of the box +- **8ms P95 latency** at 1k RPS ([benchmarks](https://docs.litellm.ai/docs/benchmarks)) + +### OSS Adopters + + + + + + + + + + + +
StripeimageGoogle ADKGreptileOpenHands

Netflix

OpenAI Agents SDK
+ +--- + +## Features
LLMs - Call 100+ LLMs (Python SDK + AI Gateway) @@ -50,7 +87,7 @@ ### Python SDK ```shell -pip install litellm +uv add litellm ``` ```python @@ -72,7 +109,7 @@ response = completion(model="anthropic/claude-sonnet-4-20250514", messages=[{"ro [**Getting Started - E2E Tutorial**](https://docs.litellm.ai/docs/proxy/docker_quick_start) - Setup virtual keys, make your first request ```shell -pip install 'litellm[proxy]' +uv tool install 'litellm[proxy]' litellm --model gpt-4o ``` @@ -223,63 +260,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
---- - -## How to use LiteLLM - -You can use LiteLLM through either the Proxy Server or Python SDK. Both gives you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs: - - - - - - - - - - - - - - - - - - - - - - - - - - -
LiteLLM AI GatewayLiteLLM Python SDK
Use CaseCentral service (LLM Gateway) to access multiple LLMsUse LiteLLM directly in your Python code
Who Uses It?Gen AI Enablement / ML Platform TeamsDevelopers building LLM projects
Key FeaturesCentralized API gateway with authentication and authorization, multi-tenant cost tracking and spend management per project/user, per-project customization (logging, guardrails, caching), virtual keys for secure access control, admin dashboard UI for monitoring and managementDirect Python library integration in your codebase, Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - Router, application-level load balancing and cost tracking, exception handling with OpenAI-compatible errors, observability callbacks (Lunary, MLflow, Langfuse, etc.)
- -LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https://docs.litellm.ai/docs/benchmarks)) - -[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://docs.litellm.ai/docs/simple_proxy)
-[**Jump to Supported LLM Providers**](https://docs.litellm.ai/docs/providers) - -**Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle) - -Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+). - -## OSS Adopters - - - - - - - - - - - -
StripeimageGoogle ADKGreptileOpenHands

Netflix

OpenAI Agents SDK
- -## Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers)) +### Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers)) | Provider | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` | |-------------------------------------------------------------------------------------|---------------------|-------------|--------------|---------------|----------------------|-------------------------|-----------------|----------------|-----------|-----------| @@ -386,25 +367,62 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature [**Read the Docs**](https://docs.litellm.ai/docs/) -## Run in Developer mode -### Services +--- + +## Get Started + +You can use LiteLLM through either the Proxy Server or Python SDK. Both give you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs: + + + + + + + + + + + + + + + + + + + + + + + + + + +
LiteLLM AI GatewayLiteLLM Python SDK
Use CaseCentral service (LLM Gateway) to access multiple LLMsUse LiteLLM directly in your Python code
Who Uses It?Gen AI Enablement / ML Platform TeamsDevelopers building LLM projects
Key FeaturesCentralized API gateway with authentication and authorization, multi-tenant cost tracking and spend management per project/user, per-project customization (logging, guardrails, caching), virtual keys for secure access control, admin dashboard UI for monitoring and managementDirect Python library integration in your codebase, Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - Router, application-level load balancing and cost tracking, exception handling with OpenAI-compatible errors, observability callbacks (Lunary, MLflow, Langfuse, etc.)
+ +**Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle) + +Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+). + +### Run in Developer Mode +#### Services 1. Setup .env file in root 2. Run dependant services `docker-compose up db prometheus` -### Backend +#### Backend 1. (In root) create virtual environment `python -m venv .venv` 2. Activate virtual environment `source .venv/bin/activate` -3. Install dependencies `pip install -e ".[all]"` -4. `pip install prisma` +3. Install dependencies `uv sync --all-extras --group proxy-dev` +4. `uv run prisma generate` 5. `prisma generate` 6. Start proxy backend `python litellm/proxy/proxy_cli.py` -### Frontend +#### Frontend 1. Navigate to `ui/litellm-dashboard` 2. Install dependencies `npm install` 3. Run `npm run dev` to start the dashboard -# Verify Docker Image Signatures +### Verify Docker Image Signatures All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0). @@ -430,6 +448,8 @@ cosign verify \ Replace `` with the version you are deploying (e.g. `v1.83.0-stable`). +--- + # Enterprise For companies that need better security, user management and professional support @@ -450,7 +470,7 @@ We welcome contributions to LiteLLM! Whether you're fixing bugs, adding features ## Quick Start for Contributors -This requires poetry to be installed. +This requires uv to be installed. ```bash git clone https://github.com/BerriAI/litellm.git @@ -486,10 +506,6 @@ All these checks must pass before your PR can be merged. - [Community Slack 💭](https://www.litellm.ai/support) - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai -# Why did we build this - -- **Need for simplicity**: Our code started to get extremely complicated managing & translating calls between Azure, OpenAI and Cohere. - # Contributors @@ -504,4 +520,3 @@ All these checks must pass before your PR can be merged. - diff --git a/cookbook/litellm-ollama-docker-image/requirements.txt b/cookbook/litellm-ollama-docker-image/requirements.txt index 7990d251cc9..815a42a679e 100644 --- a/cookbook/litellm-ollama-docker-image/requirements.txt +++ b/cookbook/litellm-ollama-docker-image/requirements.txt @@ -1 +1 @@ -litellm==1.61.15 \ No newline at end of file +litellm==1.83.5 \ No newline at end of file diff --git a/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md b/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md index ab2cf334459..4a6fa9367fc 100644 --- a/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md +++ b/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md @@ -9,6 +9,32 @@ This document provides comprehensive instructions for AI agents to generate rele 3. **Previous Version Commit Hash** - To compare model pricing changes 4. **Reference Release Notes** - Use recent stable releases (v1.76.3-stable, v1.77.2-stable) as templates for consistent formatting +### Resolving Staging PRs + +The GitHub release page (e.g. `https://github.com/BerriAI/litellm/releases/tag/v1.83.3-stable`) does **not** list the real changelog directly. The "What's Changed" section contains **staging PRs** that each bundle many individual commits/PRs. For example: + +- `Litellm oss staging 03 14 2026 by @RheagalFire in #23686` +- `Litellm ryan march 16 by @ryan-crabbe in #23822` + +To get the real changelog, you MUST click into each staging PR (e.g. `#23686`, `#23822`), open its **Commits** tab, and extract every underlying commit/PR (look for the `(#NNNNN)` suffix on commit titles). Those underlying PRs — not the staging PRs — are what get categorized in the release notes. Never treat a staging PR title as a single changelog entry. + +**IMPORTANT — staging PRs are not the complete source.** Some PRs land on the release branch *before* the staging PRs and are therefore not reachable via `gh api /pulls//commits`. GitHub's auto-generated "What's Changed" on the release page also misses these. To catch every PR in the release, you MUST additionally walk the full git log range between the previous release's commit and this release's commit: + +```bash +git fetch origin --tags +git log .. --oneline | grep -oE '#[0-9]+' | sort -u +``` + +Union the PR set from the staging-PR walk with the PR set from `git log`. Any PR in `git log` but missing from your staging-expanded set is almost certainly a content PR that merged directly to the release branch — fetch its title/body with `gh pr view ` and categorize it. Do not trust the GH release body or the staging PRs alone as the authoritative list. + +**Sanity check for new contributors.** The GH release body's "New Contributors" list is a *floor*, not authoritative. For every PR author who appears in the release (including underlying PRs from staging and PRs found only via `git log`), verify whether they are a first-time contributor by running: + +```bash +gh api "search/issues?q=is:pr+author:+repo:BerriAI/litellm+is:merged&sort=created&order=asc" --jq '.items[0] | {n:.number, merged:.closed_at}' +``` + +If the author's earliest merged PR number matches a PR in this release window, they are a new contributor. If their earliest merged PR predates the previous release tag, they are not. Do not copy the GH release body's list blindly — it can both miss contributors (PRs that merged via an older dev branch) and falsely include contributors whose "first" PR in this window was not actually their first ever. + ## Step-by-Step Process ### 1. Initial Setup and Analysis diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index bbc1ef4562b..1a85ee5c02b 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -3,55 +3,66 @@ ARG LITELLM_BUILD_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b # Runtime image ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856 +ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9@sha256:10902f58a1606787602f303954cea099626a4adb02acbac4c69920fe9d278f82 + +FROM $UV_IMAGE AS uvbin -# Builder stage FROM $LITELLM_BUILD_IMAGE AS builder -# Set the working directory to /app WORKDIR /app -# Install build dependencies -RUN apk add --no-cache gcc python3-dev musl-dev +COPY --from=uvbin /uv /usr/local/bin/uv +COPY --from=uvbin /uvx /usr/local/bin/uvx -RUN pip install --upgrade pip==26.0.1 && \ - pip install build==1.4.2 +RUN apk add --no-cache gcc python3-dev musl-dev nodejs npm libsndfile -# Copy the current directory contents into the container at /app +ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ + UV_PROJECT_ENVIRONMENT=/app/.venv \ + UV_LINK_MODE=copy \ + XDG_CACHE_HOME=/app/.cache \ + PATH="/app/.venv/bin:${PATH}" + +# Copy dependency metadata first for layer caching +COPY pyproject.toml uv.lock ./ +COPY enterprise/pyproject.toml enterprise/ +COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ + +# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) +RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ + --extra proxy \ + --extra proxy-runtime \ + --extra extra_proxy \ + --extra semantic-router \ + --python python3 + +# Copy full source tree COPY . . -# Build the package -RUN rm -rf dist/* && python -m build +# Install project and workspace packages (fast - deps already cached) +RUN uv sync --frozen --no-default-groups --no-editable \ + --extra proxy \ + --extra proxy-runtime \ + --extra extra_proxy \ + --extra semantic-router \ + --python python3 -# There should be only one wheel file now, assume the build only creates one -RUN ls -1 dist/*.whl | head -1 +RUN prisma generate --schema=./schema.prisma -# Install the package -RUN pip install dist/*.whl +RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ + sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh -# install dependencies as wheels -RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt - -# Runtime stage FROM $LITELLM_RUNTIME_IMAGE AS runtime -# Update dependencies and clean up, install libsndfile for audio processing -RUN apk upgrade --no-cache && apk add --no-cache libsndfile +RUN apk upgrade --no-cache && apk add --no-cache libsndfile nodejs npm WORKDIR /app +ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ + XDG_CACHE_HOME=/app/.cache \ + PATH="/app/.venv/bin:${PATH}" -# Copy the built wheel from the builder stage to the runtime stage; assumes only one wheel file is present -COPY --from=builder /app/dist/*.whl . -COPY --from=builder /wheels/ /wheels/ - -# Install the built wheel using pip; again using a wildcard if it's the only file -RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && rm -f *.whl && rm -rf /wheels - -# Convert Windows line endings to Unix for entrypoint scripts -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh -RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh +COPY --from=builder /app /app EXPOSE 4000/tcp -# Set your entrypoint and command ENTRYPOINT ["docker/prod_entrypoint.sh"] CMD ["--port", "4000"] diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 36dd5a78741..57ecef81eb8 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -3,53 +3,72 @@ ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92 # Runtime image ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502 -# Builder stage +ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9@sha256:10902f58a1606787602f303954cea099626a4adb02acbac4c69920fe9d278f82 + +FROM $UV_IMAGE AS uvbin + FROM $LITELLM_BUILD_IMAGE AS builder -# Set the working directory to /app WORKDIR /app - USER root -# Install build dependencies +COPY --from=uvbin /uv /usr/local/bin/uv +COPY --from=uvbin /uvx /usr/local/bin/uvx + RUN apk add --no-cache \ bash \ gcc \ - py3-pip \ python3 \ python3-dev \ openssl \ - openssl-dev + openssl-dev \ + nodejs \ + npm \ + libsndfile -RUN python -m pip install build==1.4.2 +ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ + UV_PROJECT_ENVIRONMENT=/app/.venv \ + UV_LINK_MODE=copy \ + XDG_CACHE_HOME=/app/.cache \ + PATH="/app/.venv/bin:${PATH}" -# Copy the current directory contents into the container at /app +# Copy dependency metadata first for layer caching +COPY pyproject.toml uv.lock ./ +COPY enterprise/pyproject.toml enterprise/ +COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ + +# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) +RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ + --extra proxy \ + --extra proxy-runtime \ + --extra extra_proxy \ + --extra semantic-router \ + --python python3 + +# Copy full source tree COPY . . -# Build Admin UI -# Convert Windows line endings to Unix and make executable +# Build Admin UI before final sync RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh -# Build the package -RUN rm -rf dist/* && python -m build +# Install project and workspace packages (fast - deps already cached) +RUN uv sync --frozen --no-default-groups --no-editable \ + --extra proxy \ + --extra proxy-runtime \ + --extra extra_proxy \ + --extra semantic-router \ + --python python3 -# There should be only one wheel file now, assume the build only creates one -RUN ls -1 dist/*.whl | head -1 +RUN prisma generate --schema=./schema.prisma -# Install the package -RUN pip install dist/*.whl +RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ + sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh -# install dependencies as wheels -RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt - -# Runtime stage FROM $LITELLM_RUNTIME_IMAGE AS runtime -# Ensure runtime stage runs as root USER root -# Install runtime dependencies -RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \ +RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \ npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ GLOBAL="$(npm root -g)" && \ find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ @@ -73,66 +92,18 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile { apk del --no-cache npm 2>/dev/null || true; } WORKDIR /app -# Copy the current directory contents into the container at /app -COPY . . -RUN ls -la /app +ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ + XDG_CACHE_HOME=/app/.cache \ + PATH="/app/.venv/bin:${PATH}" -# Copy the built wheel from the builder stage to the runtime stage; assumes only one wheel file is present -COPY --from=builder /app/dist/*.whl . -COPY --from=builder /wheels/ /wheels/ +COPY --from=builder /app /app -# Install the built wheel using pip; again using a wildcard if it's the only file -RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && rm -f *.whl && rm -rf /wheels +RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ + find /app/.venv -type d -path "*/tornado/test" -delete -# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete -# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/. -# Patch every copy of tar, glob, and brace-expansion inside that tree. -RUN GLOBAL="$(npm root -g)" && \ - [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \ - find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ - done && \ - find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ - done && \ - find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ - done && \ - find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ - done && \ - find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ - done - -# Install semantic_router and aurelio-sdk using script -# Convert Windows line endings to Unix and make executable -RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh - -# ensure pyjwt is used, not jwt -RUN pip uninstall jwt -y -RUN pip uninstall PyJWT -y -RUN pip install PyJWT==2.12.0 --no-cache-dir - -# Build Admin UI (runtime stage) -# Convert Windows line endings to Unix and make executable -RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh - -# Generate prisma client -RUN prisma generate -# Convert Windows line endings to Unix for entrypoint scripts -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh -RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh EXPOSE 4000/tcp -RUN apk add --no-cache supervisor COPY docker/supervisord.conf /etc/supervisord.conf -# # Set your entrypoint and command - - ENTRYPOINT ["docker/prod_entrypoint.sh"] - -# Append "--detailed_debug" to the end of CMD to view detailed debug logs -# CMD ["--port", "4000", "--detailed_debug"] CMD ["--port", "4000"] diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index fb84230adcb..88be7a6980c 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -3,60 +3,70 @@ ARG LITELLM_BUILD_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973a # Runtime image ARG LITELLM_RUNTIME_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d +ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9@sha256:10902f58a1606787602f303954cea099626a4adb02acbac4c69920fe9d278f82 + +FROM $UV_IMAGE AS uvbin -# Builder stage FROM $LITELLM_BUILD_IMAGE AS builder -# Set the working directory to /app WORKDIR /app - USER root -# Install build dependencies in one layer +COPY --from=uvbin /uv /usr/local/bin/uv +COPY --from=uvbin /uvx /usr/local/bin/uvx + RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ g++ \ python3-dev \ libssl-dev \ pkg-config \ - && rm -rf /var/lib/apt/lists/* \ - && pip install --upgrade pip==26.0.1 build==1.4.2 + nodejs \ + npm \ + && rm -rf /var/lib/apt/lists/* -# Copy requirements first for better layer caching -COPY requirements.txt . +ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ + UV_PROJECT_ENVIRONMENT=/app/.venv \ + UV_LINK_MODE=copy \ + XDG_CACHE_HOME=/app/.cache \ + PATH="/app/.venv/bin:${PATH}" -# Install Python dependencies with cache mount for faster rebuilds -RUN --mount=type=cache,target=/root/.cache/pip \ - pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt +# Copy dependency metadata first for layer caching +COPY pyproject.toml uv.lock ./ +COPY enterprise/pyproject.toml enterprise/ +COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ -# Fix JWT dependency conflicts early -RUN pip uninstall jwt -y || true && \ - pip uninstall PyJWT -y || true && \ - pip install PyJWT==2.12.0 --no-cache-dir +# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) +RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ + --extra proxy \ + --extra proxy-runtime \ + --extra extra_proxy \ + --extra semantic-router \ + --python python -# Copy only necessary files for build -COPY pyproject.toml README.md schema.prisma poetry.lock ./ -COPY litellm/ ./litellm/ -COPY enterprise/ ./enterprise/ -COPY docker/ ./docker/ +# Copy full source tree +COPY . . -# Build Admin UI once -# Convert Windows line endings to Unix and make executable +# Build Admin UI before final sync RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh -# Build the package -RUN rm -rf dist/* && python -m build +# Install project and workspace packages (fast - deps already cached) +RUN uv sync --frozen --no-default-groups --no-editable \ + --extra proxy \ + --extra proxy-runtime \ + --extra extra_proxy \ + --extra semantic-router \ + --python python -# Install the built package -RUN pip install dist/*.whl +RUN prisma generate --schema=./schema.prisma + +RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ + sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh -# Runtime stage FROM $LITELLM_RUNTIME_IMAGE AS runtime -# Ensure runtime stage runs as root USER root -# Install only runtime dependencies RUN apt-get update && apt-get upgrade -y \ libxml2 \ libexpat1 \ @@ -72,9 +82,9 @@ RUN apt-get update && apt-get upgrade -y \ libc6 \ && apt-get install -y --no-install-recommends \ libssl3 \ - libatomic1 \ - nodejs \ - npm \ + libatomic1 \ + nodejs \ + npm \ && rm -rf /var/lib/apt/lists/* \ && npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \ && GLOBAL="$(npm root -g)" \ @@ -99,53 +109,13 @@ RUN apt-get update && apt-get upgrade -y \ && apt-get purge -y npm WORKDIR /app +ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ + XDG_CACHE_HOME=/app/.cache \ + PATH="/app/.venv/bin:${PATH}" -# Copy only necessary runtime files -COPY docker/entrypoint.sh docker/prod_entrypoint.sh ./docker/ -COPY litellm/ ./litellm/ -COPY pyproject.toml README.md schema.prisma poetry.lock ./ - -# Copy pre-built wheels and install everything at once -COPY --from=builder /wheels/ /wheels/ -COPY --from=builder /app/dist/*.whl . - -# Install all dependencies in one step with no-cache for smaller image -RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && \ - rm -f *.whl && \ - rm -rf /wheels - -# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete -# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/. -# Patch every copy of tar, glob, and brace-expansion inside that tree. -RUN GLOBAL="$(npm root -g)" && \ - [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \ - find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ - done && \ - find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ - done && \ - find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ - done && \ - find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ - done && \ - find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ - done - -# Generate prisma client and set permissions -# Convert Windows line endings to Unix for entrypoint scripts -RUN prisma generate && \ - sed -i 's/\r$//' docker/entrypoint.sh && \ - sed -i 's/\r$//' docker/prod_entrypoint.sh && \ - chmod +x docker/entrypoint.sh && \ - chmod +x docker/prod_entrypoint.sh +COPY --from=builder /app /app EXPOSE 4000/tcp ENTRYPOINT ["docker/prod_entrypoint.sh"] - -# Append "--detailed_debug" to the end of CMD to view detailed debug logs -CMD ["--port", "4000"] \ No newline at end of file +CMD ["--port", "4000"] diff --git a/docker/Dockerfile.health_check b/docker/Dockerfile.health_check index e968bec340e..b2cbb467f46 100644 --- a/docker/Dockerfile.health_check +++ b/docker/Dockerfile.health_check @@ -1,16 +1,22 @@ +ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9@sha256:10902f58a1606787602f303954cea099626a4adb02acbac4c69920fe9d278f82 +FROM $UV_IMAGE AS uvbin + FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d WORKDIR /app -# Copy health check script and requirements +# Copy the uv binary and the health check script. +COPY --from=uvbin /uv /usr/local/bin/uv +COPY pyproject.toml uv.lock /app/ COPY scripts/health_check/health_check_client.py /app/health_check_client.py -COPY scripts/health_check/health_check_requirements.txt /app/requirements.txt -# Install dependencies -RUN pip install --no-cache-dir -r requirements.txt - -# Make script executable -RUN chmod +x /app/health_check_client.py +# Resolve and install the health-check dependencies from the project lockfile +# so the runtime image stays self-contained and reproducible. +RUN uv export --frozen --no-default-groups --only-group healthcheck --no-emit-project --no-hashes --output-file /tmp/health-check-requirements.txt \ + && uv pip install --system -r /tmp/health-check-requirements.txt \ + && rm /tmp/health-check-requirements.txt \ + && rm /app/pyproject.toml /app/uv.lock \ + && chmod +x /app/health_check_client.py # Run as non-root user RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index f3c7728146d..5451bff808d 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -2,51 +2,84 @@ ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502 ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502 ARG PROXY_EXTRAS_SOURCE=published +ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9@sha256:10902f58a1606787602f303954cea099626a4adb02acbac4c69920fe9d278f82 + +FROM $UV_IMAGE AS uvbin -# ----------------- -# Builder Stage -# ----------------- FROM $LITELLM_BUILD_IMAGE AS builder ARG PROXY_EXTRAS_SOURCE WORKDIR /app USER root -# Install build dependencies with retry logic (includes node for UI build) +COPY --from=uvbin /uv /usr/local/bin/uv +COPY --from=uvbin /uvx /usr/local/bin/uvx + RUN for i in 1 2 3; do \ apk add --no-cache \ - python3 \ - python3-dev \ - py3-pip \ - clang \ - llvm \ - lld \ - gcc \ - linux-headers \ - build-base \ - bash \ - nodejs \ - npm && break || sleep 5; \ - done \ - && pip install --no-cache-dir --upgrade pip==26.0.1 build==1.4.2 + python3 \ + python3-dev \ + clang \ + llvm \ + lld \ + gcc \ + linux-headers \ + build-base \ + bash \ + coreutils \ + curl \ + openssl \ + openssl-dev \ + nodejs \ + npm \ + libsndfile && break || sleep 5; \ + done -# Cache Python dependencies -COPY requirements.txt . -RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt \ - && pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.12.0" +ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ + UV_LINK_MODE=copy \ + NVM_DIR=/root/.nvm \ + PATH="/root/.nvm/versions/node/v20.20.2/bin:/app/.venv/bin:${PATH}" \ + LITELLM_NON_ROOT=true \ + PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ + PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \ + XDG_CACHE_HOME=/app/.cache -# Copy source after dependency layers +# Copy dependency metadata first for layer caching +COPY pyproject.toml uv.lock ./ +COPY enterprise/pyproject.toml enterprise/ +COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ + +# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) +RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ + --extra proxy \ + --extra proxy-runtime \ + --extra extra_proxy \ + --extra semantic-router \ + --python python3 + +# Copy full source tree COPY . . # Set non-root flag for build time consistency ENV LITELLM_NON_ROOT=true -# Build Admin UI using the upstream command order while keeping a single RUN layer +# Build Admin UI once and stage the static output for the runtime image. # NOTE: .npmrc files (which may set ignore-scripts=true and min-release-age=3d) # are temporarily renamed during npm install/ci so they don't block lifecycle # scripts needed by the build. This is safe because npm ci installs from # package-lock.json with pinned versions + integrity hashes. -RUN mkdir -p /var/lib/litellm/ui && \ +RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \ ([ -f /app/.npmrc ] && mv /app/.npmrc /app/.npmrc.bak || true) && \ + NVM_VERSION="v0.40.4" && \ + NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f" && \ + NODE_VERSION="v20.20.2" && \ + NVM_SCRIPT="/tmp/install-nvm.sh" && \ + curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT" && \ + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | sha256sum -c - && \ + bash "$NVM_SCRIPT" && \ + export NVM_DIR="$HOME/.nvm" && \ + . "$NVM_DIR/nvm.sh" && \ + nvm install "${NODE_VERSION}" && \ + nvm use "${NODE_VERSION}" && \ npm install -g npm@11.12.1 && \ npm install -g node-gyp@12.2.0 && \ ln -sf "$(npm root -g)/node-gyp" "$(npm root -g)/npm/node_modules/node-gyp" && \ @@ -56,12 +89,11 @@ RUN mkdir -p /var/lib/litellm/ui && \ cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ fi && \ ([ -f .npmrc ] && mv .npmrc .npmrc.bak || true) && \ - npm ci && \ + npm ci --no-audit --no-fund && \ ([ -f .npmrc.bak ] && mv .npmrc.bak .npmrc || true) && \ ([ -f /app/.npmrc.bak ] && mv /app/.npmrc.bak /app/.npmrc || true) && \ npm run build && \ cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \ - mkdir -p /var/lib/litellm/assets && \ cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \ ( cd /var/lib/litellm/ui && \ for html_file in *.html; do \ @@ -74,175 +106,106 @@ RUN mkdir -p /var/lib/litellm/ui && \ touch .litellm_ui_ready ) && \ cd /app/ui/litellm-dashboard && rm -rf ./out -# Build litellm wheel and place it in wheels dir (replace any PyPI wheels) -RUN rm -rf dist/* && python -m build && \ - rm -f /wheels/litellm-*.whl && \ - cp dist/*.whl /wheels/ - -# Optionally build local litellm-proxy-extras wheel -RUN if [ "$PROXY_EXTRAS_SOURCE" = "local" ]; then \ - cd /app/litellm-proxy-extras && rm -rf dist && python -m build && \ - cp dist/*.whl /wheels/; \ +RUN if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \ + uv sync --frozen --no-default-groups --no-editable \ + --extra proxy \ + --extra proxy-runtime \ + --extra extra_proxy \ + --extra semantic-router \ + --python python3 \ + --no-sources-package litellm-proxy-extras; \ + else \ + uv sync --frozen --no-default-groups --no-editable \ + --extra proxy \ + --extra proxy-runtime \ + --extra extra_proxy \ + --extra semantic-router \ + --python python3; \ fi -# Pre-cache Prisma binaries in the builder stage -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}" - -RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.13.1 \ - && mkdir -p /app/.cache/npm - -RUN NPM_CONFIG_CACHE=/app/.cache/npm \ - python -c "import prisma.cli.prisma as p; p.ensure_cached()" - -RUN prisma generate && \ +RUN mkdir -p /app/.cache/npm && \ + prisma generate --schema=./schema.prisma && \ prisma --version && \ prisma migrate diff --from-empty --to-schema-datamodel ./schema.prisma --script > /dev/null 2>&1 || true -# ----------------- -# Runtime Stage -# ----------------- +RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ + sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh + FROM $LITELLM_RUNTIME_IMAGE AS runtime ARG PROXY_EXTRAS_SOURCE WORKDIR /app USER root -# Install runtime dependencies with retry RUN for i in 1 2 3; do \ apk upgrade --no-cache && break || sleep 5; \ - done \ - && for i in 1 2 3; do \ - apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \ - done \ - && apk upgrade --no-cache nodejs \ - && npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \ - && GLOBAL="$(npm root -g)" \ - && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ - done \ - && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \ - sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \ - && npm cache clean --force \ - && { apk del --no-cache npm 2>/dev/null || true; } + done && \ + for i in 1 2 3; do \ + apk add --no-cache python3 bash openssl tzdata nodejs npm supervisor libsndfile && break || sleep 5; \ + done && \ + apk upgrade --no-cache nodejs && \ + npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ + GLOBAL="$(npm root -g)" && \ + find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ + done && \ + find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ + done && \ + find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ + done && \ + find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ + done && \ + find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ + rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ + done && \ + find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \ + sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \ + npm cache clean --force && \ + { apk del --no-cache npm 2>/dev/null || true; } -# Copy artifacts from builder -COPY --from=builder /app/requirements.txt /app/requirements.txt -COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/ -COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf -COPY --from=builder /app/schema.prisma /app/ -# Keep enterprise bridge module in runtime so `enterprise.enterprise_hooks` -# can load and register managed enterprise hooks (e.g. managed_files). -COPY --from=builder /app/enterprise /app/enterprise -# Copy prisma_migration.py for Helm migrations job compatibility -COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py -COPY --from=builder /wheels/ /wheels/ +COPY --from=builder /app /app COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets -COPY --from=builder /app/.cache /app/.cache -COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras -COPY --from=builder \ - /usr/lib/python3.13/site-packages/nodejs* \ - /usr/lib/python3.13/site-packages/prisma* \ - /usr/lib/python3.13/site-packages/tomlkit* \ - /usr/lib/python3.13/site-packages/nodeenv* \ - /usr/lib/python3.13/site-packages/ -COPY --from=builder /usr/bin/prisma /usr/bin/prisma +COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf -# Final runtime environment configuration -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ +ENV PATH="/app/.venv/bin:${PATH}" \ + PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \ HOME=/app \ LITELLM_NON_ROOT=true \ - XDG_CACHE_HOME=/app/.cache - -# Install packages from wheels and optional extras without network -RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \ - pip install --no-index --find-links=/wheels/ /wheels/litellm-*-py3-none-any.whl && \ - pip install --no-index --find-links=/wheels/ --no-deps semantic_router==0.1.11 && \ - pip install --no-index --find-links=/wheels/ aurelio-sdk==0.0.19 && \ - if [ "$PROXY_EXTRAS_SOURCE" = "local" ]; then \ - if ls /wheels/litellm_proxy_extras-*.whl >/dev/null 2>&1; then \ - pip install --no-index --find-links=/wheels/ /wheels/litellm_proxy_extras-*.whl; \ - else \ - echo "litellm_proxy_extras wheel not found; skipping local install"; \ - fi; \ - fi - -# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete -# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/. -# Patch every copy of tar, glob, and brace-expansion inside that tree. -RUN GLOBAL="$(npm root -g)" && \ - [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \ - find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ - done && \ - find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ - done && \ - find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ - done && \ - find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ - done && \ - find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ - done - -# Permissions, cleanup, and Prisma prep -# Convert Windows line endings to Unix for entrypoint scripts -RUN sed -i 's/\r$//' docker/entrypoint.sh && \ - sed -i 's/\r$//' docker/prod_entrypoint.sh && \ - chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \ - mkdir -p /nonexistent /.npm /var/lib/litellm/assets /var/lib/litellm/ui && \ - chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm && \ - pip uninstall jwt -y || true && \ - pip uninstall PyJWT -y || true && \ - pip install --no-index --find-links=/wheels/ PyJWT==2.12.0 --no-cache-dir && \ - rm -rf /wheels && \ - PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \ - chown -R nobody:nogroup $PRISMA_PATH && \ - LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \ - [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH && \ - LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \ - chgrp -R 0 $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \ - [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \ - chmod -R g=u $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \ - [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \ - chmod -R g+w $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \ - [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \ - chmod -R g+rX $PRISMA_PATH && \ - chmod -R g+rX /app/.cache && \ - mkdir -p /tmp/.npm /nonexistent /.npm - -# Switch to non-root user for runtime -USER nobody - -# Generate Prisma client as nobody user to ensure correct file ownership -RUN prisma generate - -# Prisma runtime knobs for offline containers -ENV PRISMA_SKIP_POSTINSTALL_GENERATE=1 \ + XDG_CACHE_HOME=/app/.cache \ + PRISMA_SKIP_POSTINSTALL_GENERATE=1 \ PRISMA_HIDE_UPDATE_MESSAGE=1 \ PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \ NPM_CONFIG_CACHE=/app/.cache/npm \ NPM_CONFIG_PREFER_OFFLINE=true \ PRISMA_OFFLINE_MODE=true +RUN sed -i 's/\r$//' docker/entrypoint.sh && \ + sed -i 's/\r$//' docker/prod_entrypoint.sh && \ + chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \ + mkdir -p /nonexistent /.npm /var/lib/litellm/assets /var/lib/litellm/ui /tmp/.npm && \ + chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm /tmp/.npm && \ + PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \ + chown -R nobody:nogroup "$PRISMA_PATH" && \ + LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \ + [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup "$LITELLM_PKG_MIGRATIONS_PATH" || true && \ + LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \ + chgrp -R 0 "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \ + [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 "$LITELLM_PROXY_EXTRAS_PATH" || true && \ + chmod -R g=u "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \ + [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u "$LITELLM_PROXY_EXTRAS_PATH" || true && \ + chmod -R g+w "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \ + [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w "$LITELLM_PROXY_EXTRAS_PATH" || true && \ + chmod -R g+rX "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets /app/.cache + +USER nobody + +RUN prisma generate --schema=./schema.prisma + EXPOSE 4000/tcp + ENTRYPOINT ["/app/docker/prod_entrypoint.sh"] CMD ["--port", "4000"] diff --git a/docker/build_from_pip/Dockerfile.build_from_pip b/docker/build_from_pip/Dockerfile.build_from_pip index f26b993cce5..bda742c71a9 100644 --- a/docker/build_from_pip/Dockerfile.build_from_pip +++ b/docker/build_from_pip/Dockerfile.build_from_pip @@ -1,27 +1,53 @@ +ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9@sha256:10902f58a1606787602f303954cea099626a4adb02acbac4c69920fe9d278f82 +FROM $UV_IMAGE AS uvbin + FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d +ARG LITELLM_VERSION=1.83.0 + WORKDIR /app -ENV HOME=/home/litellm -ENV PATH="${HOME}/venv/bin:$PATH" +COPY --from=uvbin /uv /usr/local/bin/uv +COPY --from=uvbin /uvx /usr/local/bin/uvx -# Install runtime dependencies needed for building native extensions RUN apt-get update && \ - apt-get install -y --no-install-recommends gcc libffi-dev && \ + apt-get install -y --no-install-recommends gcc libffi-dev nodejs npm && \ rm -rf /var/lib/apt/lists/* -RUN python -m venv ${HOME}/venv -RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip==26.0.1 +ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ + UV_LINK_MODE=copy \ + PATH="/app/.venv/bin:${PATH}" -COPY docker/build_from_pip/requirements.txt . -RUN --mount=type=cache,target=${HOME}/.cache/pip \ - ${HOME}/venv/bin/pip install -r requirements.txt - -# Copy Prisma schema file COPY schema.prisma . -# Generate prisma client -RUN prisma generate +# This image is specifically for validating/installing the published PyPI +# artifact, not the checked-out source tree. +# Keep the moved proxy-runtime packages explicit until the published PyPI +# artifact includes that extra; newer releases will simply dedupe these. +RUN uv venv --python python && \ + uv pip install --python /app/.venv/bin/python \ + "litellm[proxy,proxy-runtime]==${LITELLM_VERSION}" \ + "google-cloud-aiplatform==1.133.0" \ + "google-genai==1.37.0" \ + "anthropic[vertex]==0.84.0" \ + "grpcio==1.78.0" \ + "prometheus-client==0.20.0" \ + "langfuse==2.59.7" \ + "opentelemetry-api==1.28.0" \ + "opentelemetry-sdk==1.28.0" \ + "opentelemetry-exporter-otlp==1.28.0" \ + "ddtrace==2.19.0" \ + "sentry-sdk==2.21.0" \ + "mangum==0.17.0" \ + "azure-ai-contentsafety==1.0.0" \ + "azure-storage-file-datalake==12.20.0" \ + "pypdf==6.7.5" \ + "llm-sandbox==0.3.31" \ + "detect-secrets==1.5.0" \ + "prisma==0.11.0" \ + "openai==2.24.0" + +RUN prisma generate --schema=./schema.prisma EXPOSE 4000/tcp diff --git a/docker/build_from_pip/requirements.txt b/docker/build_from_pip/requirements.txt deleted file mode 100644 index ec6cf2438db..00000000000 --- a/docker/build_from_pip/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -litellm[proxy]==1.83.0 -prometheus_client==0.20.0 -langfuse==2.59.7 -prisma==0.11.0 -openai==2.24.0 -ddtrace==2.19.0 # for advanced DD tracing / profiling diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index a028e542629..003d9b21db8 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,13 +1,16 @@ #!/bin/bash -echo $(pwd) +set -euo pipefail -# Run the Python migration script -python3 litellm/proxy/prisma_migration.py +REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +VENV_PYTHON="$REPO_ROOT/.venv/bin/python" +MIGRATION_SCRIPT="$REPO_ROOT/litellm/proxy/prisma_migration.py" -# Check if the Python script executed successfully -if [ $? -eq 0 ]; then - echo "Migration script ran successfully!" +if [ -x "$VENV_PYTHON" ]; then + "$VENV_PYTHON" "$MIGRATION_SCRIPT" +elif command -v uv >/dev/null 2>&1; then + (cd "$REPO_ROOT" && uv run --no-sync python "$MIGRATION_SCRIPT") else - echo "Migration script failed!" - exit 1 + python3 "$MIGRATION_SCRIPT" fi + +echo "Migration script ran successfully!" diff --git a/docker/install_auto_router.sh b/docker/install_auto_router.sh index 057baa19f59..4fedf201b41 100755 --- a/docker/install_auto_router.sh +++ b/docker/install_auto_router.sh @@ -1,3 +1,4 @@ #!/bin/bash -pip install semantic_router==0.1.11 --no-deps -pip install aurelio-sdk==0.0.19 --no-deps \ No newline at end of file +set -euo pipefail + +# semantic-router dependencies are installed via `uv sync`. diff --git a/docs/my-website/Dockerfile b/docs/my-website/Dockerfile index 87d1537237d..4693d3a6574 100644 --- a/docs/my-website/Dockerfile +++ b/docs/my-website/Dockerfile @@ -1,9 +1,32 @@ +ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9 + +FROM $UV_IMAGE AS uvbin + FROM python:3.14.0a3-slim +COPY --from=uvbin /uv /usr/local/bin/uv +COPY --from=uvbin /uvx /usr/local/bin/uvx COPY . /app WORKDIR /app -RUN pip install -r requirements.txt + +ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ + UV_LINK_MODE=copy \ + PATH="/app/.venv/bin:${PATH}" + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + python3-dev \ + libssl-dev \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +RUN uv sync --frozen --no-default-groups --no-editable \ + --extra proxy \ + --extra proxy-runtime \ + --extra extra_proxy \ + --extra semantic-router \ + --python python EXPOSE $PORT -CMD litellm --host 0.0.0.0 --port $PORT --workers 10 --config config.yaml \ No newline at end of file +CMD ["sh", "-c", "litellm --host 0.0.0.0 --port $PORT --workers 10 --config config.yaml"] diff --git a/docs/my-website/docs/adding_provider/generic_prompt_management_api.md b/docs/my-website/docs/adding_provider/generic_prompt_management_api.md index d1b119d94c5..21055de3a7f 100644 --- a/docs/my-website/docs/adding_provider/generic_prompt_management_api.md +++ b/docs/my-website/docs/adding_provider/generic_prompt_management_api.md @@ -378,7 +378,7 @@ if __name__ == "__main__": 1. Install dependencies: ```bash -pip install fastapi uvicorn +uv add fastapi uvicorn ``` 2. Save the code above to `prompt_server.py` diff --git a/docs/my-website/docs/caching/all_caches.md b/docs/my-website/docs/caching/all_caches.md index 6f81da9105a..7cc329c93e3 100644 --- a/docs/my-website/docs/caching/all_caches.md +++ b/docs/my-website/docs/caching/all_caches.md @@ -23,7 +23,7 @@ import TabItem from '@theme/TabItem'; Install redis ```shell -pip install redis +uv add redis ``` For the hosted version you can setup your own Redis DB here: https://redis.io/try-free/ @@ -55,7 +55,7 @@ response2 = completion( For GCP Memorystore Redis with IAM authentication: ```shell -pip install google-cloud-iam +uv add google-cloud-iam ``` ```python @@ -150,7 +150,7 @@ response2 = completion( Install boto3 ```shell -pip install boto3 +uv add boto3 ``` Set AWS environment variables @@ -187,7 +187,7 @@ response2 = completion( Install azure-storage-blob and azure-identity ```shell -pip install azure-storage-blob azure-identity +uv add azure-storage-blob azure-identity ``` ```python @@ -219,7 +219,7 @@ response2 = completion( Install redisvl client ```shell -pip install redisvl==0.4.1 +uv add redisvl==0.4.1 ``` For the hosted version you can setup your own Redis DB here: https://redis.io/try-free/ @@ -366,7 +366,7 @@ response2 = completion( Install the disk caching extra: ```shell -pip install "litellm[caching]" +uv add "litellm[caching]" ``` Then you can use the disk cache as follows. diff --git a/docs/my-website/docs/completion/anthropic_advisor_tool.md b/docs/my-website/docs/completion/anthropic_advisor_tool.md index 3cb87ffdf3d..23be7c776ee 100644 --- a/docs/my-website/docs/completion/anthropic_advisor_tool.md +++ b/docs/my-website/docs/completion/anthropic_advisor_tool.md @@ -14,12 +14,48 @@ The advisor tool is in beta. Include `anthropic-beta: advisor-tool-2026-03-01` i ## Supported Providers -| Provider | Chat Completions API | Messages API | -|----------|---------------------|--------------| -| **Anthropic API** | ✅ | ✅ | -| **Azure Anthropic** | ❌ (coming soon) | ❌ (coming soon) | -| **Google Cloud Vertex AI** | ❌ (coming soon) | ❌ (coming soon) | -| **Amazon Bedrock** | ❌ (coming soon) | ❌ (coming soon) | +| Provider | Chat Completions API | Messages API | Notes | +|----------|---------------------|--------------|-------| +| **Anthropic API** | ✅ | ✅ | Native — runs server-side | +| **OpenAI / Azure OpenAI** | ✅ | ✅ | LiteLLM orchestration loop | +| **Amazon Bedrock** | ✅ | ✅ | LiteLLM orchestration loop | +| **Google Vertex AI** | ✅ | ✅ | LiteLLM orchestration loop | +| **Groq / Mistral / others** | ✅ | ✅ | LiteLLM orchestration loop | + +## How it works (LiteLLM native orchestration) + +For non-Anthropic providers, LiteLLM implements the advisor loop itself. The API you call is identical — LiteLLM handles everything transparently. + +When a request arrives with an `advisor_20260301` tool and a non-Anthropic provider, `AdvisorOrchestrationHandler` intercepts it. It translates the advisor tool into a regular function tool the provider understands, then runs an orchestration loop: + +```mermaid +flowchart TD + A["Your request\ntools: advisor_20260301\nmodel: e.g. openai/gpt-4.1-mini"] --> B["AdvisorOrchestrationHandler\ntranslates advisor → regular fn tool"] + + B --> C["EXECUTOR CALL\nopenai / bedrock / vertex / etc."] + + C --> D{"executor calls\nadvisor tool?"} + + D -->|"yes — tool_use\nname=advisor"| E{"max_uses\nexceeded?"} + + E -->|no| F["ADVISOR SUB-CALL\nclaude-opus-4-6\nfull transcript forwarded\nno tools"] + + F --> G["Inject advice as\ntool_result into history"] + + G --> C + + E -->|yes| H["AdvisorMaxIterationsError"] + + D -->|"no — end_turn\nor other stop reason"| I["Clean final response\nno advisor blocks in output"] +``` + +**What LiteLLM does for you:** + +- Strips `advisor_20260301` from the outgoing request — the provider only sees a standard function tool named `advisor` +- When the executor calls it, intercepts before the result reaches you, runs the advisor sub-call, and injects the advice +- Strips any `advisor_tool_result` / `server_tool_use` blocks from message history on re-send so non-Anthropic providers never see Anthropic-specific types +- Wraps the final response in an SSE stream if you requested `stream=True` +- Enforces `max_uses` as a hard cap — `AdvisorMaxIterationsError` is raised if exceeded; `max_uses=0` disables the advisor entirely ## Model Compatibility @@ -305,6 +341,37 @@ response = client.beta.messages.create( print(response) ``` +#### Non-Anthropic Provider (LiteLLM orchestration loop) + +```python showLineNumbers title="Advisor Tool with OpenAI executor" +import asyncio +import litellm + +async def main(): + # executor: openai/gpt-4.1-mini | advisor: claude-opus-4-6 + # LiteLLM runs the orchestration loop automatically + response = await litellm.anthropic.messages.acreate( + model="openai/gpt-4.1-mini", + messages=[ + {"role": "user", "content": "Implement a Python LRU cache with O(1) get and put."} + ], + tools=[ + { + "type": "advisor_20260301", + "name": "advisor", + "model": "claude-opus-4-6", + "max_uses": 3, + } + ], + max_tokens=1024, + custom_llm_provider="openai", + ) + # Final response is clean — no advisor tool_use blocks + print(response["content"][0]["text"]) + +asyncio.run(main()) +``` + --- ## Response Structure diff --git a/docs/my-website/docs/completion/message_sanitization.md b/docs/my-website/docs/completion/message_sanitization.md index 17482c59339..6114b640f0f 100644 --- a/docs/my-website/docs/completion/message_sanitization.md +++ b/docs/my-website/docs/completion/message_sanitization.md @@ -401,7 +401,7 @@ response = litellm.completion( 3. Ensure you're using a recent version of LiteLLM: ```bash - pip install --upgrade litellm + uv add --upgrade-package litellm litellm ``` ### Unexpected Dummy Tool Results diff --git a/docs/my-website/docs/completion/prompt_compression.md b/docs/my-website/docs/completion/prompt_compression.md new file mode 100644 index 00000000000..2d999291af6 --- /dev/null +++ b/docs/my-website/docs/completion/prompt_compression.md @@ -0,0 +1,123 @@ +# Prompt Compression (`compress()`) + +Use `litellm.compress()` to shrink long conversation history before calling `completion()`. + +The function keeps high-relevance and recent context, replaces low-relevance content with lightweight stubs, and returns a retrieval tool so the model can request full content only when needed. + +## Quickstart + +```python +import litellm + +messages = [ + {"role": "system", "content": "You are a coding assistant."}, + {"role": "user", "content": "# auth.py\n" + "def authenticate():\n pass\n" * 2000}, + {"role": "user", "content": "# utils.py\n" + "def helper():\n pass\n" * 2000}, + {"role": "user", "content": "Fix the bug in auth.py"}, +] + +compressed = litellm.compress( + messages=messages, + model="gpt-4o", + compression_trigger=1000, + compression_target=500, +) + +response = litellm.completion( + model="gpt-4o", + messages=compressed["messages"], + tools=compressed["tools"], +) +``` + +## What It Returns + +`compress()` returns a dictionary with: + +- `messages`: compressed conversation messages +- `original_tokens`: token count before compression +- `compressed_tokens`: token count after compression +- `compression_ratio`: fraction of tokens removed +- `cache`: key-value mapping of stub key -> original full content +- `tools`: retrieval tool definition (`litellm_content_retrieve`) for on-demand restoration + +## Parameters + +- `messages` (`List[dict]`, required): input conversation messages +- `model` (`str`, required): model name used for token counting +- `compression_trigger` (`int`, default `200000`): compress only if input token count exceeds this +- `compression_target` (`Optional[int]`, default `70% of compression_trigger`): desired post-compression token budget +- `embedding_model` (`Optional[str]`): if set, combines BM25 + embedding relevance scoring +- `embedding_model_params` (`Optional[dict]`): additional kwargs passed to `litellm.embedding()` +- `compression_cache` (`Optional[DualCache]`): optional cache used by embedding scoring + +## Behavior Notes + +- Messages below `compression_trigger` are passed through unchanged. +- System messages, the last user message, and the last assistant message are always preserved. +- If a relevant message does not fully fit the remaining budget, `compress()` may keep a truncated version of it. +- Compressed-out content is never lost; it is stored in `cache` and addressable by `litellm_content_retrieve`. + +## Handling Retrieval Tool Calls + +If the model calls `litellm_content_retrieve`, look up the requested key in `compressed["cache"]` and return that value as tool output. + +```python +import json + +tool_call = response.choices[0].message.tool_calls[0] +args = json.loads(tool_call.function.arguments) +full_content = compressed["cache"][args["key"]] +``` + +## Performance + +Benchmarked on [SWE-bench Lite](https://huggingface.co/datasets/princeton-nlp/SWE-bench_Lite_bm25_27K) (real GitHub issues with ~27k tokens of BM25-retrieved repo context per problem). + +### Claude Opus — 5 problems, trigger=10k + +| Metric | Baseline | Compressed | Delta | +|---|---|---|---| +| File overlap | 1.000 | 1.000 | +0.000 | +| Exact file match | 100% | 100% | +0.0% | +| Hunk overlap | 0.582 | 0.361 | -0.221 | +| Content similarity | 0.367 | 0.373 | +0.006 | +| Avg prompt tokens | 30,828 | 6,890 | -77.7% | +| Avg cost/problem | $0.488 | $0.136 | **-72.0%** | + +**Key takeaways:** + +- **File-level targeting is fully preserved** — the model edits the same files with or without compression. +- **Content similarity matches baseline** — the actual lines changed are comparable. +- **Hunk overlap drops modestly** (-0.221) — the model targets the right files but may edit slightly different line ranges with less surrounding context. +- **72% cost savings** with 78% token reduction. + +### Metrics explained + +| Metric | What it measures | +|---|---| +| **File overlap** | Fraction of gold-patch files present in the generated patch | +| **Exact file match** | Whether the generated patch touches exactly the same set of files | +| **Hunk overlap** | Fraction of gold hunk line ranges covered by generated hunks | +| **Content similarity** | Jaccard similarity of changed lines (added/removed) between gold and generated patches | + +### Running the SWE-bench eval + +```bash +# 5-problem quick check +python tests/eval_swe_bench.py --model claude-opus-4-20250514 --problems 5 + +# Custom trigger/target +python tests/eval_swe_bench.py --model gpt-4o --problems 20 \ + --compression-trigger 15000 --compression-target 10000 + +# With embedding scoring +python tests/eval_swe_bench.py --model gpt-4o --problems 10 \ + --embedding-model text-embedding-3-small +``` + +### Running the HumanEval-style eval + +```bash +python scripts/eval_compression.py --model gpt-4o --problems 5 +``` diff --git a/docs/my-website/docs/contributing.md b/docs/my-website/docs/contributing.md index 168d092ddc7..9e2799ddd6c 100644 --- a/docs/my-website/docs/contributing.md +++ b/docs/my-website/docs/contributing.md @@ -29,7 +29,7 @@ general_settings: Start the proxy on port 4000: ```bash -poetry run litellm --config config.yaml --port 4000 +uv run litellm --config config.yaml --port 4000 ``` The UI comes pre-built in the repo. Access it at `http://localhost:4000/ui` diff --git a/docs/my-website/docs/default_code_snippet.md b/docs/my-website/docs/default_code_snippet.md index 0921c316685..34c842de7f7 100644 --- a/docs/my-website/docs/default_code_snippet.md +++ b/docs/my-website/docs/default_code_snippet.md @@ -16,7 +16,7 @@ If you want to use the non-hosted version, [go here](https://docs.litellm.ai/doc ``` -pip install litellm +uv add litellm ``` \ No newline at end of file diff --git a/docs/my-website/docs/extras/contributing_code.md b/docs/my-website/docs/extras/contributing_code.md index 673a83aca05..95d82f2c9ce 100644 --- a/docs/my-website/docs/extras/contributing_code.md +++ b/docs/my-website/docs/extras/contributing_code.md @@ -41,7 +41,7 @@ git clone https://github.com/BerriAI/litellm.git Step 2: Install dev dependencies ```shell -poetry install --with dev --extras proxy +uv sync --group dev --extra proxy ``` ### 2. Adding tests diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index 6410e052b05..2f9ed281b49 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -26,13 +26,13 @@ import Image from '@theme/IdealImage'; ## Installation ```shell -pip install litellm +uv add litellm ``` To run the full Proxy Server (LLM Gateway): ```shell -pip install 'litellm[proxy]' +uv tool install 'litellm[proxy]' ``` --- @@ -336,7 +336,7 @@ The proxy is a self-hosted OpenAI-compatible gateway. Any client that works with #### Step 1 — Start the proxy - + ```shell litellm --model huggingface/bigcode/starcoder diff --git a/docs/my-website/docs/integrations/letta.md b/docs/my-website/docs/integrations/letta.md index 9711999df5e..1be902065b5 100644 --- a/docs/my-website/docs/integrations/letta.md +++ b/docs/my-website/docs/integrations/letta.md @@ -16,7 +16,7 @@ Letta allows you to build LLM agents that can: ## Prerequisites ```bash -pip install letta litellm +uv add letta litellm ``` ## Quick Start @@ -910,7 +910,7 @@ for model in models: ``` ### Common SDK Issues -- **Import errors**: Ensure `pip install litellm letta` is run +- **Import errors**: Ensure `uv add litellm letta` is run - **Model format**: Use `provider/model` format (e.g., `openai/gpt-4`) - **API key format**: Different providers have different key formats - **Rate limits**: Implement exponential backoff for retries diff --git a/docs/my-website/docs/langchain/langchain.md b/docs/my-website/docs/langchain/langchain.md index c67375ce1be..b692f1bfd7a 100644 --- a/docs/my-website/docs/langchain/langchain.md +++ b/docs/my-website/docs/langchain/langchain.md @@ -5,7 +5,7 @@ import TabItem from '@theme/TabItem'; ## Pre-Requisites ```shell -!pip install litellm langchain +!uv add litellm langchain ``` ## Quick Start diff --git a/docs/my-website/docs/learn/gateway_quickstart.md b/docs/my-website/docs/learn/gateway_quickstart.md index acec259758c..eb7a15cfd41 100644 --- a/docs/my-website/docs/learn/gateway_quickstart.md +++ b/docs/my-website/docs/learn/gateway_quickstart.md @@ -13,7 +13,7 @@ If you need a Docker or database-first setup, use the [Docker + Database tutoria ## 1. Install The Gateway ```bash -pip install 'litellm[proxy]' +uv tool install 'litellm[proxy]' ``` ## 2. Set One Provider Key diff --git a/docs/my-website/docs/learn/sdk_quickstart.md b/docs/my-website/docs/learn/sdk_quickstart.md index 0fb8c3f02a5..522a7251e31 100644 --- a/docs/my-website/docs/learn/sdk_quickstart.md +++ b/docs/my-website/docs/learn/sdk_quickstart.md @@ -11,7 +11,7 @@ Use this path if you are integrating LiteLLM directly into application code. ## 1. Install LiteLLM ```bash -pip install litellm==1.82.6 +uv add 'litellm==1.82.6' ``` ## 2. Set Provider Credentials diff --git a/docs/my-website/docs/load_test.md b/docs/my-website/docs/load_test.md index 071b097904b..52274024eb8 100644 --- a/docs/my-website/docs/load_test.md +++ b/docs/my-website/docs/load_test.md @@ -17,7 +17,7 @@ model_list: api_base: https://exampleopenaiendpoint-production.up.railway.app/ ``` -2. `pip install locust` +2. `uv add locust` 3. Create a file called `locustfile.py` on your local machine. Copy the contents from the litellm load test located [here](https://github.com/BerriAI/litellm/blob/main/.github/workflows/locustfile.py) diff --git a/docs/my-website/docs/load_test_advanced.md b/docs/my-website/docs/load_test_advanced.md index d7bc35e74e0..b23f0da35c3 100644 --- a/docs/my-website/docs/load_test_advanced.md +++ b/docs/my-website/docs/load_test_advanced.md @@ -70,7 +70,7 @@ litellm_settings: callbacks: ["prometheus"] # Enterprise LiteLLM Only - use prometheus to get metrics on your load test ``` -2. `pip install locust` +2. `uv add locust` 3. Create a file called `locustfile.py` on your local machine. Copy the contents from the litellm load test located [here](https://github.com/BerriAI/litellm/blob/main/.github/workflows/locustfile.py) @@ -138,7 +138,7 @@ litellm_settings: callbacks: ["prometheus"] # Enterprise LiteLLM Only - use prometheus to get metrics on your load test ``` -2. `pip install locust` +2. `uv add locust` 3. Create a file called `locustfile.py` on your local machine. Copy the contents from the litellm load test located [here](https://github.com/BerriAI/litellm/blob/main/.github/workflows/locustfile.py) diff --git a/docs/my-website/docs/mcp_aws_sigv4.md b/docs/my-website/docs/mcp_aws_sigv4.md index e556ad244f8..337bc83869a 100644 --- a/docs/my-website/docs/mcp_aws_sigv4.md +++ b/docs/my-website/docs/mcp_aws_sigv4.md @@ -224,7 +224,7 @@ SigV4-authenticated MCP servers skip the standard health check on proxy startup. Install the `botocore` package: ```bash -pip install botocore +uv add botocore ``` `botocore` is used for SigV4 credential handling and is required when using `aws_sigv4` auth. diff --git a/docs/my-website/docs/mcp_oauth.md b/docs/my-website/docs/mcp_oauth.md index 5c4b70cc5b3..3340533286b 100644 --- a/docs/my-website/docs/mcp_oauth.md +++ b/docs/my-website/docs/mcp_oauth.md @@ -205,7 +205,7 @@ sequenceDiagram Use [BerriAI/mock-oauth2-mcp-server](https://github.com/BerriAI/mock-oauth2-mcp-server) to test locally: ```bash title="Terminal 1 - Start mock server" showLineNumbers -pip install fastapi uvicorn +uv add fastapi uvicorn python mock_oauth2_mcp_server.py # starts on :8765 ``` diff --git a/docs/my-website/docs/observability/braintrust.md b/docs/my-website/docs/observability/braintrust.md index 645ce074ca5..84f54dc0fdc 100644 --- a/docs/my-website/docs/observability/braintrust.md +++ b/docs/my-website/docs/observability/braintrust.md @@ -9,7 +9,7 @@ import TabItem from '@theme/TabItem'; ## Quick Start ```python -# pip install braintrust +# uv add braintrust import litellm import os diff --git a/docs/my-website/docs/observability/lago.md b/docs/my-website/docs/observability/lago.md index 337a2b553ee..a7663cb98c7 100644 --- a/docs/my-website/docs/observability/lago.md +++ b/docs/my-website/docs/observability/lago.md @@ -22,7 +22,7 @@ litellm.callbacks = ["lago"] # logs cost + usage of successful calls to lago ```python -# pip install lago +# uv add lago import litellm import os diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md index 32849ebdb92..f696f9be41c 100644 --- a/docs/my-website/docs/observability/langfuse_integration.md +++ b/docs/my-website/docs/observability/langfuse_integration.md @@ -26,9 +26,9 @@ For Langfuse v3, we recommend using the [Langfuse OTEL](./langfuse_otel_integrat ## Usage with LiteLLM Python SDK ### Pre-Requisites -Ensure you have run `pip install langfuse` for this integration +Ensure you have run `uv add langfuse` for this integration ```shell -pip install langfuse==2.59.7 litellm +uv add langfuse==2.59.7 litellm ``` ### Quick Start @@ -44,7 +44,7 @@ litellm.success_callback = ["langfuse"] litellm.failure_callback = ["langfuse"] # logs errors to langfuse ``` ```python -# pip install langfuse +# uv add langfuse import litellm import os @@ -335,7 +335,7 @@ Be aware that if you are continuing an existing trace, and you set `update_trace ## Troubleshooting & Errors ### Data not getting logged to Langfuse ? -- Ensure you're on the latest version of langfuse `pip install langfuse -U`. The latest version allows litellm to log JSON input/outputs to langfuse +- Ensure you're on the latest version of langfuse `uv add langfuse -U`. The latest version allows litellm to log JSON input/outputs to langfuse - Follow [this checklist](https://langfuse.com/faq/all/missing-traces) if you don't see any traces in langfuse. ## Support & Talk to Founders diff --git a/docs/my-website/docs/observability/langfuse_otel_integration.md b/docs/my-website/docs/observability/langfuse_otel_integration.md index 79ad2f6f75d..90f7f7becca 100644 --- a/docs/my-website/docs/observability/langfuse_otel_integration.md +++ b/docs/my-website/docs/observability/langfuse_otel_integration.md @@ -24,7 +24,7 @@ The Langfuse OpenTelemetry integration allows you to send LiteLLM traces and obs 2. **API Keys**: Get your public and secret keys from your Langfuse project settings 3. **Dependencies**: Install required packages: ```bash - pip install litellm opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp + uv add litellm opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp ``` ## Configuration diff --git a/docs/my-website/docs/observability/langsmith_integration.md b/docs/my-website/docs/observability/langsmith_integration.md index bf867319de8..5eb36cd8149 100644 --- a/docs/my-website/docs/observability/langsmith_integration.md +++ b/docs/my-website/docs/observability/langsmith_integration.md @@ -18,7 +18,7 @@ join our [discord](https://discord.gg/wuPM9dRgDw) ## Pre-Requisites ```shell -pip install litellm +uv add litellm ``` ## Quick Start diff --git a/docs/my-website/docs/observability/levo_integration.md b/docs/my-website/docs/observability/levo_integration.md index 3e46cf6b921..c11e720aebe 100644 --- a/docs/my-website/docs/observability/levo_integration.md +++ b/docs/my-website/docs/observability/levo_integration.md @@ -36,7 +36,7 @@ Send all your LLM requests and responses to Levo for monitoring and analysis usi **1. Install OpenTelemetry dependencies:** ```bash -pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-grpc +uv add opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-grpc ``` **2. Enable Levo callback in your LiteLLM config:** @@ -133,7 +133,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ``` 4. **Check for initialization errors**: Look for errors in LiteLLM startup logs. Common issues: - - Missing OpenTelemetry packages: Install with `pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-grpc` + - Missing OpenTelemetry packages: Install with `uv add opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-grpc` - Missing required environment variables: All four required variables must be set - Invalid collector URL: Ensure the URL is correct and reachable @@ -150,7 +150,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ - Solution: Set the `LEVOAI_COLLECTOR_URL` environment variable with your collector endpoint URL from Levo support. **Error: "No module named 'opentelemetry'"** -- Solution: Install OpenTelemetry packages: `pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-grpc` +- Solution: Install OpenTelemetry packages: `uv add opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-grpc` ## Additional Resources diff --git a/docs/my-website/docs/observability/literalai_integration.md b/docs/my-website/docs/observability/literalai_integration.md index 128c86b2cc3..88ae7309215 100644 --- a/docs/my-website/docs/observability/literalai_integration.md +++ b/docs/my-website/docs/observability/literalai_integration.md @@ -11,7 +11,7 @@ import Image from '@theme/IdealImage'; Ensure you have the `literalai` package installed: ```shell -pip install literalai litellm +uv add literalai litellm ``` ## Quick Start diff --git a/docs/my-website/docs/observability/logfire_integration.md b/docs/my-website/docs/observability/logfire_integration.md index 00652c0f1a1..bf6b03e205f 100644 --- a/docs/my-website/docs/observability/logfire_integration.md +++ b/docs/my-website/docs/observability/logfire_integration.md @@ -17,11 +17,11 @@ join our [discord](https://discord.gg/wuPM9dRgDw) Ensure you have installed the following packages to use this integration ```shell -pip install litellm +uv add litellm -pip install opentelemetry-api==1.25.0 -pip install opentelemetry-sdk==1.25.0 -pip install opentelemetry-exporter-otlp==1.25.0 +uv add opentelemetry-api==1.25.0 +uv add opentelemetry-sdk==1.25.0 +uv add opentelemetry-exporter-otlp==1.25.0 ``` ## Quick Start @@ -33,7 +33,7 @@ litellm.callbacks = ["logfire"] ``` ```python -# pip install logfire +# uv add logfire import litellm import os diff --git a/docs/my-website/docs/observability/lunary_integration.md b/docs/my-website/docs/observability/lunary_integration.md index 4f1dca9d4af..fee07091cbd 100644 --- a/docs/my-website/docs/observability/lunary_integration.md +++ b/docs/my-website/docs/observability/lunary_integration.md @@ -15,7 +15,7 @@ You can reach out to us anytime by [email](mailto:hello@lunary.ai) or directly [ ### Pre-Requisites ```shell -pip install litellm lunary +uv add litellm lunary ``` ### Quick Start @@ -124,7 +124,7 @@ my_chain("Chain input") ### Step1: Install dependencies and set your environment variables Install the dependencies ```shell -pip install litellm lunary +uv add litellm lunary ``` Get you Lunary public key from from https://app.lunary.ai/settings diff --git a/docs/my-website/docs/observability/mlflow.md b/docs/my-website/docs/observability/mlflow.md index 5fa46bdfdac..4018c970482 100644 --- a/docs/my-website/docs/observability/mlflow.md +++ b/docs/my-website/docs/observability/mlflow.md @@ -17,7 +17,7 @@ MLflow’s integration with LiteLLM supports advanced observability compatible w Install MLflow: ```shell -pip install "litellm[mlflow]" +uv add "litellm[mlflow]" ``` To enable MLflow auto tracing for LiteLLM: @@ -167,7 +167,7 @@ This approach generates a unified trace, combining your custom Python code with For using `mlflow` on LiteLLM Proxy Server, you need to install the `mlflow` package on your docker container. ```shell -pip install "mlflow>=3.1.4" +uv add "mlflow>=3.1.4" ``` ### Configuration diff --git a/docs/my-website/docs/observability/openmeter.md b/docs/my-website/docs/observability/openmeter.md index 2f53568757f..b3e07ef8ff9 100644 --- a/docs/my-website/docs/observability/openmeter.md +++ b/docs/my-website/docs/observability/openmeter.md @@ -28,7 +28,7 @@ litellm.callbacks = ["openmeter"] # logs cost + usage of successful calls to ope ```python -# pip install openmeter +# uv add openmeter import litellm import os diff --git a/docs/my-website/docs/observability/opentelemetry_integration.md b/docs/my-website/docs/observability/opentelemetry_integration.md index 80ef1bcc989..f8fcebf7ab6 100644 --- a/docs/my-website/docs/observability/opentelemetry_integration.md +++ b/docs/my-website/docs/observability/opentelemetry_integration.md @@ -27,7 +27,7 @@ USE_OTEL_LITELLM_REQUEST_SPAN=true Install the OpenTelemetry SDK: ``` -pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp +uv add opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp ``` Set the environment variables (different providers may require different variables): @@ -63,7 +63,7 @@ OTEL_EXPORTER_OTLP_PROTOCOL=grpc OTEL_EXPORTER_OTLP_HEADERS="api-key=key,other-config-value=value" ``` -> Note: OTLP gRPC requires `grpcio`. Install via `pip install "litellm[grpc]"` (or `grpcio`). +> Note: OTLP gRPC requires `grpcio`. Install via `uv add "litellm[grpc]"` (or `grpcio`). @@ -75,7 +75,7 @@ OTEL_ENDPOINT="https://api.lmnr.ai:8443" OTEL_HEADERS="authorization=Bearer " ``` -> Note: OTLP gRPC requires `grpcio`. Install via `pip install "litellm[grpc]"` (or `grpcio`). +> Note: OTLP gRPC requires `grpcio`. Install via `uv add "litellm[grpc]"` (or `grpcio`). diff --git a/docs/my-website/docs/observability/phoenix_integration.md b/docs/my-website/docs/observability/phoenix_integration.md index 67ba815557e..998e0fca6c2 100644 --- a/docs/my-website/docs/observability/phoenix_integration.md +++ b/docs/my-website/docs/observability/phoenix_integration.md @@ -22,7 +22,7 @@ Use just 2 lines of code, to instantly log your responses **across all providers You can also use the instrumentor option instead of the callback, which you can find [here](https://docs.arize.com/phoenix/tracing/integrations-tracing/litellm). ```bash -pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp litellm[proxy] +uv add opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp litellm[proxy] ``` ```python litellm.callbacks = ["arize_phoenix"] @@ -73,7 +73,7 @@ environment_variables: PHOENIX_COLLECTOR_HTTP_ENDPOINT: "https://app.phoenix.arize.com/s//v1/traces" # OPTIONAL - For setting the HTTP endpoint ``` -> Note: If you set the gRPC endpoint, install `grpcio` via `pip install "litellm[grpc]"` (or `grpcio`). +> Note: If you set the gRPC endpoint, install `grpcio` via `uv add "litellm[grpc]"` (or `grpcio`). 2. Start the proxy diff --git a/docs/my-website/docs/observability/qualifire_integration.md b/docs/my-website/docs/observability/qualifire_integration.md index cf866f467bf..cf376136e17 100644 --- a/docs/my-website/docs/observability/qualifire_integration.md +++ b/docs/my-website/docs/observability/qualifire_integration.md @@ -23,7 +23,7 @@ Looking for Qualifire Guardrails? Check out the [Qualifire Guardrails Integratio 2. Get your API key and webhook URL from the Qualifire dashboard ```bash -pip install litellm +uv add litellm ``` ## Quick Start diff --git a/docs/my-website/docs/observability/raw_request_response.md b/docs/my-website/docs/observability/raw_request_response.md index 71305dae692..011a3a74af7 100644 --- a/docs/my-website/docs/observability/raw_request_response.md +++ b/docs/my-website/docs/observability/raw_request_response.md @@ -12,7 +12,7 @@ See the raw request/response sent by LiteLLM in your logging provider (OTEL/Lang ```python -# pip install langfuse +# uv add langfuse import litellm import os diff --git a/docs/my-website/docs/observability/scrub_data.md b/docs/my-website/docs/observability/scrub_data.md index f8bb4d556c7..4e13d1b5a1e 100644 --- a/docs/my-website/docs/observability/scrub_data.md +++ b/docs/my-website/docs/observability/scrub_data.md @@ -60,7 +60,7 @@ litellm.callbacks = [customHandler] 3. Test it! ```python -# pip install langfuse +# uv add langfuse import os import litellm diff --git a/docs/my-website/docs/observability/signoz.md b/docs/my-website/docs/observability/signoz.md index f306b143ef0..7af0c294063 100644 --- a/docs/my-website/docs/observability/signoz.md +++ b/docs/my-website/docs/observability/signoz.md @@ -17,7 +17,7 @@ Instrumenting LiteLLM in your AI applications with telemetry ensures full observ - A [SigNoz Cloud account](https://signoz.io/teams/) with an active ingestion key - Internet access to send telemetry data to SigNoz Cloud - [LiteLLM](https://www.litellm.ai/) SDK or Proxy integration -- For Python: `pip` installed for managing Python packages and _(optional but recommended)_ a Python virtual environment to isolate dependencies +- For Python: `uv` installed for managing Python packages and _(optional but recommended)_ a Python virtual environment to isolate dependencies ## Monitoring LiteLLM @@ -37,7 +37,7 @@ No-code auto-instrumentation is recommended for quick setup with minimal code ch **Step 1:** Install the necessary packages in your Python environment. ```bash -pip install \ +uv add \ opentelemetry-api \ opentelemetry-distro \ opentelemetry-exporter-otlp \ @@ -99,7 +99,7 @@ OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=openai \ opentelemetry-instrument ``` -> Note: OTLP gRPC requires `grpcio`. Install via `pip install "litellm[grpc]"` (or `grpcio`). +> Note: OTLP gRPC requires `grpcio`. Install via `uv add "litellm[grpc]"` (or `grpcio`). > 📌 Note: We're using `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=openai` in the run command to disable the OpenAI instrumentor for tracing. This avoids conflicts with LiteLLM's native telemetry/instrumentation, ensuring that telemetry is captured exclusively through LiteLLM's built-in instrumentation. @@ -120,7 +120,7 @@ Code-based instrumentation gives you fine-grained control over your telemetry co **Step 1:** Install the necessary packages in your Python environment. ```bash -pip install \ +uv add \ opentelemetry-api \ opentelemetry-sdk \ opentelemetry-exporter-otlp \ @@ -338,7 +338,7 @@ You can also check out our custom LiteLLM SDK dashboard [here](https://signoz.i **Step 1:** Install the necessary packages in your Python environment. ```bash -pip install opentelemetry-api \ +uv add opentelemetry-api \ opentelemetry-sdk \ opentelemetry-exporter-otlp \ 'litellm[proxy]' @@ -364,7 +364,7 @@ export OTEL_METRICS_EXPORTER="otlp" export OTEL_LOGS_EXPORTER="otlp" ``` -> Note: OTLP gRPC requires `grpcio`. Install via `pip install "litellm[grpc]"` (or `grpcio`). +> Note: OTLP gRPC requires `grpcio`. Install via `uv add "litellm[grpc]"` (or `grpcio`). - Set the `` to match your SigNoz Cloud [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint) - Replace `` with your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/) diff --git a/docs/my-website/docs/observability/slack_integration.md b/docs/my-website/docs/observability/slack_integration.md index 468d8b5945c..2b7737a0cfe 100644 --- a/docs/my-website/docs/observability/slack_integration.md +++ b/docs/my-website/docs/observability/slack_integration.md @@ -13,7 +13,7 @@ join our [discord](https://discord.gg/wuPM9dRgDw) ### Step 1 ```shell -pip install litellm +uv add litellm ``` ### Step 2 diff --git a/docs/my-website/docs/observability/sumologic_integration.md b/docs/my-website/docs/observability/sumologic_integration.md index 87e20ca57ed..d7f057df52a 100644 --- a/docs/my-website/docs/observability/sumologic_integration.md +++ b/docs/my-website/docs/observability/sumologic_integration.md @@ -25,7 +25,7 @@ join our [discord](https://discord.gg/wuPM9dRgDw) For more details, see the [HTTP Logs & Metrics Source](https://www.sumologic.com/help/docs/send-data/hosted-collectors/http-source/logs-metrics/) documentation. ```shell -pip install litellm +uv add litellm ``` ## Quick Start diff --git a/docs/my-website/docs/observability/wandb_integration.md b/docs/my-website/docs/observability/wandb_integration.md index 3c1a3363957..1126998c99e 100644 --- a/docs/my-website/docs/observability/wandb_integration.md +++ b/docs/my-website/docs/observability/wandb_integration.md @@ -21,9 +21,9 @@ join our [discord](https://discord.gg/wuPM9dRgDw) ::: ## Pre-Requisites -Ensure you have run `pip install wandb` for this integration +Ensure you have run `uv add wandb` for this integration ```shell -pip install wandb litellm +uv add wandb litellm ``` ## Quick Start @@ -33,7 +33,7 @@ Use just 2 lines of code, to instantly log your responses **across all providers litellm.success_callback = ["wandb"] ``` ```python -# pip install wandb +# uv add wandb import litellm import os diff --git a/docs/my-website/docs/oidc.md b/docs/my-website/docs/oidc.md index 23eb431b7e0..c4b82a08d17 100644 --- a/docs/my-website/docs/oidc.md +++ b/docs/my-website/docs/oidc.md @@ -57,12 +57,31 @@ oidc/config_name_here/ #### Unofficial Providers (not recommended) -For the unofficial `file` provider, you can use the following format: +For the unofficial `file` provider, you can use the following format +(note the double slash — the path after `oidc/file/` must be absolute): ``` -oidc/file/home/user/dave/this_is_a_file_with_a_token.txt +oidc/file//var/run/secrets/my-token ``` +For safety, the resolved path must live inside an allowed credential +directory. By default the following directories are allowed: + +- `/var/run/secrets` +- `/run/secrets` + +If your deployment mounts credentials elsewhere, set the +`LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS` environment variable to a +comma-separated list of absolute directories. The value replaces the +default list, so include the defaults if you still need them: + +```bash +export LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS="/var/run/secrets,/etc/litellm/creds" +``` + +Paths that resolve (after following symlinks and `..`) outside the +allowlist are rejected. + For the unofficial `env`, use the following format, where `SECRET_TOKEN` is the name of the environment variable that contains the token: ``` diff --git a/docs/my-website/docs/pass_through/bedrock.md b/docs/my-website/docs/pass_through/bedrock.md index 65c5d8caadc..19345c031fe 100644 --- a/docs/my-website/docs/pass_through/bedrock.md +++ b/docs/my-website/docs/pass_through/bedrock.md @@ -566,7 +566,7 @@ You can use the [LangChain AWS SDK](https://python.langchain.com/docs/integratio **1. Install LangChain AWS**: ```bash showLineNumbers -pip install langchain-aws +uv add langchain-aws ``` **2. Setup LiteLLM Proxy**: diff --git a/docs/my-website/docs/projects/Harbor.md b/docs/my-website/docs/projects/Harbor.md index 684dfa93720..ee9d355dcbf 100644 --- a/docs/my-website/docs/projects/Harbor.md +++ b/docs/my-website/docs/projects/Harbor.md @@ -5,7 +5,7 @@ ```bash # Install -pip install harbor +uv add harbor # Run a benchmark with any LiteLLM-supported model harbor run --dataset terminal-bench@2.0 \ diff --git a/docs/my-website/docs/projects/openai-agents.md b/docs/my-website/docs/projects/openai-agents.md index 86983e7e510..7d7ff0c0b01 100644 --- a/docs/my-website/docs/projects/openai-agents.md +++ b/docs/my-website/docs/projects/openai-agents.md @@ -12,7 +12,7 @@ The [OpenAI Agents SDK](https://github.com/openai/openai-agents-python) is a lig ### 1. Install Dependencies ```bash -pip install "openai-agents[litellm]" +uv add "openai-agents[litellm]" ``` ### 2. Add Model to Config diff --git a/docs/my-website/docs/providers/azure/azure.md b/docs/my-website/docs/providers/azure/azure.md index 682f263c108..de6ab6a07eb 100644 --- a/docs/my-website/docs/providers/azure/azure.md +++ b/docs/my-website/docs/providers/azure/azure.md @@ -1143,7 +1143,7 @@ In production, [Router connects to a Redis Cache](#redis-queue) to track usage a #### Quick Start ```python -pip install litellm +uv add litellm ``` ```python diff --git a/docs/my-website/docs/providers/azure_ai.md b/docs/my-website/docs/providers/azure_ai.md index 68e2df676e6..c39967dba37 100644 --- a/docs/my-website/docs/providers/azure_ai.md +++ b/docs/my-website/docs/providers/azure_ai.md @@ -121,7 +121,7 @@ response = completion( See all litellm.completion supported params [here](../completion/input.md#translated-openai-params) ```python -# !pip install litellm +# !uv add litellm from litellm import completion import os ## set ENV variables diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index e5942cc1119..750b91f8cad 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -16,7 +16,7 @@ ALL Bedrock models (Anthropic, Meta, Deepseek, Mistral, Amazon, etc.) are Suppor LiteLLM requires `boto3` to be installed on your system for Bedrock requests ```shell -pip install boto3>=1.28.57 +uv add boto3>=1.28.57 ``` :::info diff --git a/docs/my-website/docs/providers/bedrock_realtime_with_audio.md b/docs/my-website/docs/providers/bedrock_realtime_with_audio.md index a2d9813ffd9..d725f6ecd12 100644 --- a/docs/my-website/docs/providers/bedrock_realtime_with_audio.md +++ b/docs/my-website/docs/providers/bedrock_realtime_with_audio.md @@ -319,7 +319,7 @@ Complete working examples are available in the LiteLLM repository: ## Requirements ```bash -pip install litellm websockets pyaudio +uv add litellm websockets pyaudio ``` ## AWS Configuration diff --git a/docs/my-website/docs/providers/bytez.md b/docs/my-website/docs/providers/bytez.md index fc7a684ee8d..3e2222fe684 100644 --- a/docs/my-website/docs/providers/bytez.md +++ b/docs/my-website/docs/providers/bytez.md @@ -126,7 +126,7 @@ If you wish to use custom formatting, please let us know via either [help@bytez. See all litellm.completion supported params [here](https://docs.litellm.ai/docs/completion/input) ```py -# !pip install litellm +# !uv add litellm from litellm import completion import os ## set ENV variables @@ -160,7 +160,7 @@ Any kwarg supported by huggingface we also support! (Provided the model supports Example `repetition_penalty` ```py -# !pip install litellm +# !uv add litellm from litellm import completion import os ## set ENV variables diff --git a/docs/my-website/docs/providers/clarifai.md b/docs/my-website/docs/providers/clarifai.md index eb46901db22..d1f592fe394 100644 --- a/docs/my-website/docs/providers/clarifai.md +++ b/docs/my-website/docs/providers/clarifai.md @@ -14,7 +14,7 @@ Anthropic, OpenAI, Qwen, xAI, Gemini and most of Open soured LLMs are Supported ## Pre-Requisites ```bash -pip install litellm +uv add litellm ``` ## Required Environment Variables diff --git a/docs/my-website/docs/providers/databricks.md b/docs/my-website/docs/providers/databricks.md index 2791d55dff1..aaccb930738 100644 --- a/docs/my-website/docs/providers/databricks.md +++ b/docs/my-website/docs/providers/databricks.md @@ -59,7 +59,7 @@ If no credentials are provided, LiteLLM will use the Databricks SDK for automati from litellm import completion # No environment variables needed - uses Databricks SDK unified auth -# Requires: pip install databricks-sdk +# Requires: uv add databricks-sdk response = completion( model="databricks/databricks-dbrx-instruct", messages=[{"role": "user", "content": "Hello!"}], @@ -220,7 +220,7 @@ response = completion( See all litellm.completion supported params [here](../completion/input.md#translated-openai-params) ```python -# !pip install litellm +# !uv add litellm from litellm import completion import os ## set ENV variables @@ -457,7 +457,7 @@ For embedding models, databricks lets you pass in an additional param 'instructi ```python -# !pip install litellm +# !uv add litellm from litellm import embedding import os ## set ENV variables diff --git a/docs/my-website/docs/providers/gemini/videos.md b/docs/my-website/docs/providers/gemini/videos.md index 5b5d5a8a636..3af43656929 100644 --- a/docs/my-website/docs/providers/gemini/videos.md +++ b/docs/my-website/docs/providers/gemini/videos.md @@ -9,8 +9,8 @@ LiteLLM supports Google's Veo video generation models through a unified API inte |-------|-------| | Description | Google's Veo AI video generation models | | Provider Route on LiteLLM | `gemini/` | -| Supported Models | `veo-3.0-generate-preview`, `veo-3.1-generate-preview` | -| Cost Tracking | ✅ Duration-based pricing | +| Supported Models | Veo 3.0 / 3.1 preview and production IDs (see table below), including **Veo 3.1 Lite** | +| Cost Tracking | ✅ Duration-based pricing; optional **per-resolution** tiers where the catalog lists them (e.g. 720p vs 1080p) | | Logging Support | ✅ Full request/response logging | | Proxy Server Support | ✅ Full proxy integration with virtual keys | | Spend Management | ✅ Budget tracking and rate limiting | @@ -79,6 +79,11 @@ print("Video downloaded successfully!") |------------|-------------|--------------|--------| | veo-3.0-generate-preview | Veo 3.0 video generation | 8 seconds | Preview | | veo-3.1-generate-preview | Veo 3.1 video generation | 8 seconds | Preview | +| veo-3.1-lite-generate-preview | Veo 3.1 **Lite** (cost-efficient; [Gemini pricing](https://ai.google.dev/gemini-api/docs/video)) | Per Google docs | Preview | +| veo-3.1-fast-generate-preview / `…-001` | Faster / prod variants | Per Google docs | Preview / GA | +| veo-3.1-generate-001 | Veo 3.1 production | Per Google docs | GA | + +Use the full LiteLLM model id with the `gemini/` prefix (for example `gemini/veo-3.1-lite-generate-preview`). ## Video Generation Parameters @@ -87,14 +92,29 @@ LiteLLM automatically maps OpenAI-style parameters to Veo's format: | OpenAI Parameter | Veo Parameter | Description | Example | |------------------|---------------|-------------|---------| | `prompt` | `prompt` | Text description of the video | "A cat playing" | -| `size` | `aspectRatio` | Video dimensions → aspect ratio | "1280x720" → "16:9" | +| `size` | `aspectRatio` and, when applicable, **`resolution`** | Standard widths/heights map to landscape/portrait **and** to `720p` or `1080p` for the API | See below | | `seconds` | `durationSeconds` | Duration in seconds | "8" → 8 | | `input_reference` | `image` | Reference image to animate | File object or path | | `model` | `model` | Model to use | "gemini/veo-3.0-generate-preview" | -### Size to Aspect Ratio Mapping +### `size` and output resolution + +When you pass a **standard `size`** string, LiteLLM sets both: + +- **Aspect ratio** (`16:9` or `9:16`) — same as before. +- **Output resolution** (`720p` or `1080p`) when the height is clear from the preset, so the correct Veo tier is requested without extra fields. + +| `size` | Aspect ratio | Resolution sent to Veo | +|--------|----------------|-------------------------| +| `1280x720`, `720x1280` | `16:9` / `9:16` | `720p` | +| `1920x1080`, `1080x1920` | `16:9` / `9:16` | `1080p` | + +Other `size` values still map to an aspect ratio (defaulting to `16:9` when unknown); resolution is left to **Google’s default** unless you set it yourself. + +You can also pass Veo’s **`resolution`** (for example via `extra_body`) if you need an explicit value that does not match the presets above. If you set `resolution` yourself, it takes precedence over the value inferred from `size`. + +### Size to aspect ratio (reference) -LiteLLM automatically converts size dimensions to Veo's aspect ratio format: - `"1280x720"`, `"1920x1080"` → `"16:9"` (landscape) - `"720x1280"`, `"1080x1920"` → `"9:16"` (portrait) @@ -293,7 +313,14 @@ with open("video.mp4", "wb") as f: -## Cost Tracking +## Cost tracking and spend + +LiteLLM estimates **video spend** from: + +1. **How long** the generated clip is billed for (seconds), and +2. **The per-second price** for that model in LiteLLM’s model catalog (aligned with [Google’s Gemini API video pricing](https://ai.google.dev/gemini-api/docs/video) where applicable). + +Some models charge **different per-second rates** for **720p** vs **1080p**. When you use the standard `size` presets above (or set `resolution` explicitly), LiteLLM uses the matching tier so **proxy spend, logs, and budgets** line up with the resolution you requested. LiteLLM automatically tracks costs for Veo video generation: @@ -314,8 +341,8 @@ response = litellm.video_generation( | Feature | OpenAI (Sora) | Gemini (Veo) | |---------|---------------|--------------| | Reference Images | ✅ Supported | ❌ Not supported | -| Size Control | ✅ Supported | ❌ Not supported | -| Duration Control | ✅ Supported | ❌ Not supported | +| Size / dimensions | ✅ Supported | ✅ Supported via `size` → aspect ratio + `720p`/`1080p` where preset | +| Duration (`seconds`) | ✅ Supported | ✅ Supported (maps to `durationSeconds`; limits per Google docs) | | Video Remix/Edit | ✅ Supported | ❌ Not supported | | Video List | ✅ Supported | ❌ Not supported | | Prompt-based Generation | ✅ Supported | ✅ Supported | diff --git a/docs/my-website/docs/providers/huggingface.md b/docs/my-website/docs/providers/huggingface.md index 985351e9f69..46ea93bbe0b 100644 --- a/docs/my-website/docs/providers/huggingface.md +++ b/docs/my-website/docs/providers/huggingface.md @@ -341,7 +341,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ```python -# pip install openai +# uv add openai from openai import OpenAI client = OpenAI( diff --git a/docs/my-website/docs/providers/langgraph.md b/docs/my-website/docs/providers/langgraph.md index 9b4b24cf8f5..eea8459c723 100644 --- a/docs/my-website/docs/providers/langgraph.md +++ b/docs/my-website/docs/providers/langgraph.md @@ -187,7 +187,7 @@ Before using LiteLLM with LangGraph, you need a running LangGraph server. ### 1. Install the LangGraph CLI ```bash -pip install "langgraph-cli[inmem]" +uv add "langgraph-cli[inmem]" ``` ### 2. Create a new LangGraph project @@ -200,7 +200,7 @@ cd my-agent ### 3. Install dependencies ```bash -pip install -e . +uv add -e . ``` ### 4. Set your API key diff --git a/docs/my-website/docs/providers/oci.md b/docs/my-website/docs/providers/oci.md index 1d7a0a3d502..182bb4407a7 100644 --- a/docs/my-website/docs/providers/oci.md +++ b/docs/my-website/docs/providers/oci.md @@ -80,7 +80,7 @@ Use an OCI SDK `Signer` object for authentication. This method: To use this method, install the OCI SDK: ```bash -pip install oci +uv add oci ``` This method is an alternative when using the LiteLLM SDK on Oracle Cloud Infrastructure (instances or Oracle Kubernetes Engine). diff --git a/docs/my-website/docs/providers/ollama.md b/docs/my-website/docs/providers/ollama.md index d59d9dd0cee..bf32993c1dd 100644 --- a/docs/my-website/docs/providers/ollama.md +++ b/docs/my-website/docs/providers/ollama.md @@ -49,7 +49,7 @@ for chunk in response: ## Example usage - Streaming + Acompletion Ensure you have async_generator installed for using ollama acompletion with streaming ```shell -pip install async_generator +uv add async_generator ``` ```python diff --git a/docs/my-website/docs/providers/petals.md b/docs/my-website/docs/providers/petals.md index b5dd1705b43..c64b097c7e4 100644 --- a/docs/my-website/docs/providers/petals.md +++ b/docs/my-website/docs/providers/petals.md @@ -8,7 +8,7 @@ Petals: https://github.com/bigscience-workshop/petals ## Pre-Requisites Ensure you have `petals` installed ```shell -pip install git+https://github.com/bigscience-workshop/petals +uv add git+https://github.com/bigscience-workshop/petals ``` ## Usage diff --git a/docs/my-website/docs/providers/predibase.md b/docs/my-website/docs/providers/predibase.md index 9f25309c193..978db3d14d1 100644 --- a/docs/my-website/docs/providers/predibase.md +++ b/docs/my-website/docs/providers/predibase.md @@ -186,7 +186,7 @@ model_list: See all litellm.completion supported params [here](https://docs.litellm.ai/docs/completion/input) ```python -# !pip install litellm +# !uv add litellm from litellm import completion import os ## set ENV variables @@ -219,7 +219,7 @@ Send params [not supported by `litellm.completion()`](https://docs.litellm.ai/do Example `adapter_id`, `adapter_source` are Predibase specific param - [See List](https://github.com/BerriAI/litellm/blob/8a35354dd6dbf4c2fcefcd6e877b980fcbd68c58/litellm/llms/predibase.py#L54) ```python -# !pip install litellm +# !uv add litellm from litellm import completion import os ## set ENV variables diff --git a/docs/my-website/docs/providers/pydantic_ai_agent.md b/docs/my-website/docs/providers/pydantic_ai_agent.md index e96295faaf3..4e24e6d4e41 100644 --- a/docs/my-website/docs/providers/pydantic_ai_agent.md +++ b/docs/my-website/docs/providers/pydantic_ai_agent.md @@ -23,7 +23,7 @@ LiteLLM requires Pydantic AI agents to follow the [A2A (Agent-to-Agent) protocol #### Install Dependencies ```bash -pip install pydantic-ai fasta2a uvicorn +uv add pydantic-ai fasta2a uvicorn ``` #### Create Agent diff --git a/docs/my-website/docs/providers/replicate.md b/docs/my-website/docs/providers/replicate.md index 8e71d3ac999..db24d218275 100644 --- a/docs/my-website/docs/providers/replicate.md +++ b/docs/my-website/docs/providers/replicate.md @@ -231,7 +231,7 @@ Model Name | Function Call See all litellm.completion supported params [here](https://docs.litellm.ai/docs/completion/input) ```python -# !pip install litellm +# !uv add litellm from litellm import completion import os ## set ENV variables @@ -264,7 +264,7 @@ Send params [not supported by `litellm.completion()`](https://docs.litellm.ai/do Example `seed`, `min_tokens` are Replicate specific param ```python -# !pip install litellm +# !uv add litellm from litellm import completion import os ## set ENV variables diff --git a/docs/my-website/docs/providers/sap.md b/docs/my-website/docs/providers/sap.md index 3877cb6ef19..5d11dba5c07 100644 --- a/docs/my-website/docs/providers/sap.md +++ b/docs/my-website/docs/providers/sap.md @@ -51,7 +51,7 @@ The resource group is typically configured separately in your AI Core deployment ### Step 1: Install LiteLLM ```bash -pip install litellm +uv add litellm ``` ### Step 2: Set Your Credentials diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index a3eb673f039..0079bd2f57e 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -1216,7 +1216,7 @@ curl http://0.0.0.0:4000/chat/completions \ ## Pre-requisites -* `pip install google-cloud-aiplatform` (pre-installed on proxy docker image) +* `uv add google-cloud-aiplatform` (pre-installed on proxy docker image) * Authentication: * run `gcloud auth application-default login` See [Google Cloud Docs](https://cloud.google.com/docs/authentication/external/set-up-adc) * Alternatively you can set `GOOGLE_APPLICATION_CREDENTIALS` diff --git a/docs/my-website/docs/providers/vllm.md b/docs/my-website/docs/providers/vllm.md index 1a37f2f10e7..6fc3a9f3287 100644 --- a/docs/my-website/docs/providers/vllm.md +++ b/docs/my-website/docs/providers/vllm.md @@ -517,11 +517,11 @@ curl -X POST http://0.0.0.0:4000/chat/completions \ -## (Deprecated) for `vllm pip package` +## (Deprecated) for packaged `vllm` installs ### Using - `litellm.completion` ``` -pip install litellm vllm +uv add litellm vllm ``` ```python import litellm @@ -616,4 +616,3 @@ test_vllm_custom_model() ``` [Implementation Code](https://github.com/BerriAI/litellm/blob/6b3cb1898382f2e4e80fd372308ea232868c78d1/litellm/utils.py#L1414) - diff --git a/docs/my-website/docs/proxy/caching.md b/docs/my-website/docs/proxy/caching.md index 3357dcb28b2..39a9cfefc73 100644 --- a/docs/my-website/docs/proxy/caching.md +++ b/docs/my-website/docs/proxy/caching.md @@ -214,7 +214,7 @@ For GCP Memorystore Redis with IAM authentication, install the required dependen ::: ```shell -pip install google-cloud-iam +uv add google-cloud-iam ``` diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index c276a3bf45d..a40eaa8960b 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -831,6 +831,7 @@ router_settings: | LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS | TTL in seconds for the distributed lock used by the key rotation job. Default is 600 (10 minutes). | LITELLM_LICENSE | License key for LiteLLM usage | LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS | Set to `True` to use the local bundled Anthropic beta headers config only, disabling remote fetching. Default is `False` +| LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS | Comma-separated list of absolute directories from which the `oidc/file/` provider is permitted to read token files. Defaults to `/var/run/secrets,/run/secrets`. | LITELLM_LOCAL_BLOG_POSTS | When set to `True`, uses the local bundled blog posts only, disabling remote fetching from GitHub. Default is `False` | LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM | LITELLM_LOCAL_POLICY_TEMPLATES | When set to "true", uses local backup policy templates instead of fetching from GitHub. Policy templates are fetched from https://raw.githubusercontent.com/BerriAI/litellm/main/policy_templates.json by default, with automatic fallback to local backup on failure @@ -914,6 +915,7 @@ router_settings: | MODEL_COST_MAP_MAX_SHRINK_RATIO | Maximum allowed shrinkage ratio when validating a fetched model cost map against the local backup. Rejects the fetched map if it is smaller than this fraction of the backup. Default is 0.5 | MODEL_COST_MAP_MIN_MODEL_COUNT | Minimum number of models a fetched cost map must contain to be considered valid. Default is 50 | NO_DOCS | Flag to disable Swagger UI documentation +| NO_OPENAPI | Flag to disable the /openapi.json endpoint | NO_REDOC | Flag to disable Redoc documentation | NO_PROXY | List of addresses to bypass proxy | NON_LLM_CONNECTION_TIMEOUT | Timeout in seconds for non-LLM service connections. Default is 15 diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index f7833fc02c7..c04c3e2cc1c 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -32,10 +32,10 @@ docker pull docker.litellm.ai/berriai/litellm:main-latest - + ```shell -$ pip install 'litellm[proxy]' +$ uv tool install 'litellm[proxy]' ``` @@ -191,33 +191,32 @@ EXPOSE 4000/tcp CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"] ``` -### Build from litellm `pip` package +### Build from published LiteLLM packages -Follow these instructions to build a docker container from the litellm pip package. If your company has a strict requirement around security / building images you can follow these steps. +Follow these instructions to build a Docker container from published LiteLLM packages. If your company has a strict requirement around security or image provenance, you can follow these steps. -**Note:** You'll need to copy the `schema.prisma` file from the [litellm repository](https://github.com/BerriAI/litellm/blob/main/schema.prisma) to your build directory alongside the Dockerfile and requirements.txt. +**Note:** Copy the `schema.prisma` file from the [LiteLLM repository](https://github.com/BerriAI/litellm/blob/main/schema.prisma) into your build directory alongside this Dockerfile. Dockerfile ```shell FROM cgr.dev/chainguard/python:latest-dev +ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9 USER root WORKDIR /app -ENV HOME=/home/litellm -ENV PATH="${HOME}/venv/bin:$PATH" +ENV UV_TOOL_BIN_DIR=/usr/local/bin # Install runtime dependencies RUN apk update && \ apk add --no-cache gcc python3-dev openssl openssl-dev -RUN python -m venv ${HOME}/venv -RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip +COPY --from=$UV_IMAGE /uv /usr/local/bin/uv +COPY --from=$UV_IMAGE /uvx /usr/local/bin/uvx -COPY requirements.txt . -RUN --mount=type=cache,target=${HOME}/.cache/pip \ - ${HOME}/venv/bin/pip install -r requirements.txt +RUN uv tool install 'litellm[proxy,proxy-runtime,extra_proxy]==1.57.3' \ + --python python # Copy Prisma schema file COPY schema.prisma . @@ -232,22 +231,12 @@ CMD ["--port", "4000"] ``` -Example `requirements.txt` - -```shell -litellm[proxy]==1.57.3 # Specify the litellm version you want to use -litellm-enterprise -prometheus_client -langfuse -prisma -``` - Build the docker image ```shell docker build \ - -f Dockerfile.build_from_pip \ - -t litellm-proxy-with-pip-5 . + -f Dockerfile \ + -t litellm-proxy-from-package-5 . ``` Run the docker image @@ -258,7 +247,7 @@ docker run \ -e OPENAI_API_KEY="sk-1222" \ -e DATABASE_URL="postgresql://xxxxxxxxx \ -p 4000:4000 \ - litellm-proxy-with-pip-5 \ + litellm-proxy-from-package-5 \ --config /app/config.yaml --detailed_debug ``` @@ -760,7 +749,7 @@ RUN chmod +x ./docker/entrypoint.sh EXPOSE 4000/tcp # 👉 Key Change: Install hypercorn -RUN pip install hypercorn +RUN uv add hypercorn # Override the CMD instruction with your desired command and arguments # WARNING: FOR PROD DO NOT USE `--detailed_debug` it slows down response times, instead use the following CMD diff --git a/docs/my-website/docs/proxy/docker_quick_start.md b/docs/my-website/docs/proxy/docker_quick_start.md index 58a56604751..391793773f1 100644 --- a/docs/my-website/docs/proxy/docker_quick_start.md +++ b/docs/my-website/docs/proxy/docker_quick_start.md @@ -70,15 +70,15 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ }' ``` -:::tip Already have pip installed? -You can skip the curl install and run `litellm --setup` directly after `pip install 'litellm[proxy]'`. +:::tip Already have uv installed? +You can skip the curl install and run `litellm --setup` directly after `uv tool install 'litellm[proxy]'`. ::: --- ## Pre-Requisites -Choose your install method. **Docker Compose** users complete their full setup inside the tab and are done. **Docker** and **pip** users continue with the steps below the tabs. +Choose your install method. **Docker Compose** users complete their full setup inside the tab and are done. **Docker** and **LiteLLM CLI** users continue with the steps below the tabs. @@ -92,10 +92,10 @@ docker pull docker.litellm.ai/berriai/litellm:main-latest - + ```shell -$ pip install 'litellm[proxy]' +$ uv tool install 'litellm[proxy]' ``` @@ -269,7 +269,7 @@ Virtual keys let you track spend, set rate limits, and control model access per :::note Docker Compose users -Your setup is complete — the steps below are for **Docker** and **pip** users only. +Your setup is complete — the steps below are for **Docker** and **LiteLLM CLI** users only. ::: --- @@ -336,7 +336,7 @@ docker run \ - + ```shell $ litellm --config /app/config.yaml --detailed_debug @@ -463,7 +463,7 @@ Track spend and control model access via virtual keys for the proxy. Your Postgres container is already running — skip ahead to [Create Key w/ RPM Limit](#create-key-w-rpm-limit) below. ::: -**Docker / pip users** — you need a Postgres database (e.g. [Supabase](https://supabase.com/), [Neon](https://neon.tech/), or self-hosted). Add `general_settings` to your `config.yaml`: +**Docker / LiteLLM CLI users** — you need a Postgres database (e.g. [Supabase](https://supabase.com/), [Neon](https://neon.tech/), or self-hosted). Add `general_settings` to your `config.yaml`: ```yaml model_list: diff --git a/docs/my-website/docs/proxy/guardrails/hiddenlayer.md b/docs/my-website/docs/proxy/guardrails/hiddenlayer.md index 1ec892972d0..2aab139cd24 100644 --- a/docs/my-website/docs/proxy/guardrails/hiddenlayer.md +++ b/docs/my-website/docs/proxy/guardrails/hiddenlayer.md @@ -174,6 +174,7 @@ guardrails: - **`default_on`**: Automatically attach the guardrail to every request unless the client opts out. - **`hl-project-id` header**: Routes scans to a specific HiddenLayer project. - **`hl-requester-id` header**: Sets `metadata.requester_id` for auditing. +- **`hl-session-id` header**: Groups related requests into a session for contextual analysis and tracing in the HiddenLayer console. ## Environment variables diff --git a/docs/my-website/docs/proxy/guardrails/lasso_security.md b/docs/my-website/docs/proxy/guardrails/lasso_security.md index 363be894e4d..c1d7ea4895c 100644 --- a/docs/my-website/docs/proxy/guardrails/lasso_security.md +++ b/docs/my-website/docs/proxy/guardrails/lasso_security.md @@ -11,7 +11,7 @@ Use [Lasso Security](https://www.lasso.security/) to protect your LLM applicatio The Lasso guardrail requires the `ulid-py` package (version 1.1.0 or higher) for generating unique conversation identifiers: ```shell -pip install ulid-py>=1.1.0 +uv add ulid-py>=1.1.0 ``` This package is used to create lexicographically sortable identifiers for tracking conversations and sessions in the Lasso Security platform. diff --git a/docs/my-website/docs/proxy/guardrails/policy_flow_builder.md b/docs/my-website/docs/proxy/guardrails/policy_flow_builder.md index 630930aa893..200a7ed9b18 100644 --- a/docs/my-website/docs/proxy/guardrails/policy_flow_builder.md +++ b/docs/my-website/docs/proxy/guardrails/policy_flow_builder.md @@ -71,11 +71,105 @@ For each step you choose an action for **pass**, **fail**, and optionally **erro 3. Select **Flow Builder** (instead of the simple form) 4. Design your flow: - **Trigger** — Incoming LLM request (runs when the policy matches) - - **Steps** — Add guardrails, set **ON PASS**, **ON FAIL**, and **ON ERROR** actions per step (ON ERROR is optional; when unset, errors follow ON FAIL) - - **End** — Request proceeds to the LLM -5. Use the **+** between steps to insert new steps -6. Use the **Test** panel to run sample messages through the pipeline before saving -7. Click **Save** to create or update the policy + - **Steps** — Add guardrails; set **ON PASS**, **ON FAIL**, and **ON API FAILURE** / **ON ERROR** per step (when **ON API FAILURE** is unset, technical errors follow **ON FAIL**) + - **End** — Request proceeds to the LLM when the pipeline allows it +5. Use **+** between steps to insert another guardrail step (for fallbacks, retries, or stricter second checks) +6. Use **Test Pipeline** to run sample messages before saving +7. Click **Save Policy** (or **Save**) to create or update the policy + +### Configure guardrail fallbacks in the UI (walkthrough) + +1. Click **Policies** + +![Policies tab in the Admin UI](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/1333f4ae-d7df-4645-bd33-fee11c80cb96/ascreenshot_ce21e8bd79324c4685ad6c191e39d89e_text_export.jpeg) + +2. Click **+ Add New Policy** + +![Add new policy](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/353c08ab-cdb5-490f-b54f-734f77c87c45/ascreenshot_223033a61071485187e87cbb8c41081e_text_export.jpeg) + +3. Click **Flow Builder** + +![Choose Flow Builder](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/70e99d1b-fd76-4143-93f4-296b8b4c3904/ascreenshot_ef49b2e2c5dc40e39cf8da7a37f346ac_text_export.jpeg) + +4. Click **Continue to Builder** + +![Continue to Builder](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/3de1beaf-9c52-4f03-9100-ce4d47e41967/ascreenshot_a1d64e7e58c54b6cb8a311173ffe435a_text_export.jpeg) + +5. Click the **guardrail search** field on the first step + +![Select first guardrail — search field](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/640f699b-bdde-4e6d-a226-1fede9477b22/ascreenshot_27f14445b78b4e61872f3f95c1c9bacd_text_export.jpeg) + +6. Choose **Test Moderation** (or your primary guardrail) + +![Pick Test Moderation](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/d46f7ab6-4231-44fb-b377-59f817cdfbe5/ascreenshot_e3a9f8e25ffe46ad82a73641b81d157c_text_export.jpeg) + +7. For one branch (e.g. **ON API FAILURE**), set the action to **Next Step** so the pipeline can fall through to the next guardrail when the API errors + +![Set action to Next Step](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/3a7ddc2a-4317-417b-9341-ff6b0913e64b/ascreenshot_8878486dc12b4dddafe0c8ba4382a0fb_text_export.jpeg) + +8. For **ON PASS**, set **Allow** (or **Next Step** if you need more steps before allowing) + +![Set ON PASS to Allow](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/0e31cde8-3075-4e17-b771-b2b1696db98f/ascreenshot_b4b1d232459e4941904c9fbcf90c70ca_text_export.jpeg) + +9. Open the next outcome’s search/dropdown (e.g. **ON FAIL**) + +![Configure another branch — search field](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/715fc3ad-f245-4ee8-bb36-cc13400d635d/ascreenshot_395fece82c124d4d826fb5d84c9c0529_text_export.jpeg) + +10. Set that branch to **Next Step** if failed checks should continue to your backup guardrail + +![ON FAIL or branch — Next Step](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/83156e9b-fc3f-4cc2-a6cb-2a13a5e77b06/ascreenshot_c61429bf7b354063afc57c40a6b45c7a_text_export.jpeg) + +11. Click **+** between steps to add a second guardrail + +![Add step — plus control](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/e76cff13-af73-4775-90f6-4d29cb97d401/ascreenshot_52c478e7afd5410f9f63b616c753c851_text_export.jpeg) + +12. Open the guardrail search field on the new step + +![Second step — guardrail search](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/5c1c4eea-d7da-41e5-bebd-945e97562aa5/ascreenshot_cef70e9146b148b1936e721638de0783_text_export.jpeg) + +13. Select **Insults & Personal Attacks** (or your fallback / stricter guardrail) + +![Pick Insults and Personal Attacks](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/e796c733-351f-494f-9261-795c27f2b519/ascreenshot_f0f778d50c2146e48829ffb203c7de92_text_export.jpeg) + +14. Set **Next Step** or **Block** on the branches as needed for this step + +![Second step branch — Next Step](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/c5fad953-4f4b-47ec-ab6d-81d21b2fb7b8/ascreenshot_b515fadec0534c6a9b9d66091398d82d_text_export.jpeg) + +15. Set **ON PASS** to **Allow** when this guardrail should complete the pipeline successfully + +![Second step — Allow on pass](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/8210f32a-8704-41b1-97cc-7d183682a2a4/ascreenshot_23361af2b7da482a8d89025ab285a72e_text_export.jpeg) + +16. Open the branch where you want a **Custom Response** (e.g. **ON FAIL** on the last step) + +![Custom response — open branch selector](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/98ab3a2c-f22f-4478-a146-d5d26cae9b10/ascreenshot_6a3b673654e64ce29c8c93fbf30c52ed_text_export.jpeg) + +17. Choose **Custom Response** + +![Select Custom Response](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/a9e69e82-d517-4426-95da-034643a2388b/ascreenshot_f8ef581fbfb440cdbf145a2e9368c8e8_text_export.jpeg) + +18. Click **Enter custom response...** and type your message + +![Custom response text field](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/ef0f90ba-d0bc-4220-874f-4998b2dcc5f6/ascreenshot_f3e825b57fa0478a92f56840af266e03_text_export.jpeg) + +19. Confirm or edit the message in **Enter custom response...** as needed + +![Custom response — confirm message](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/f9a4711d-655c-4f15-b0ea-6b7d33fe6e60/ascreenshot_5df4b465bc484d8f86a4af5a45e9ab42_text_export.jpeg) + +20. Open **Test Pipeline** + +![Test Pipeline panel](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/3f9ac555-66fe-43e0-a8d8-2288a5966c73/ascreenshot_b2319dae363346ebb4da5d09180b56e8_text_export.jpeg) + +21. Click **Run Test** + +![Run Test](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/8e21e973-8193-404b-9d97-fd85be5f90b6/ascreenshot_619ca71e3be244449ca2ab01dde3cc45_text_export.jpeg) + +22. Expand **Step 1** (or the first guardrail row) in the results to see **ERROR** / **Next Step** vs **PASS** / **Allow** + +![Expand first step in test results](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/b8010e20-dd9a-4e59-b0ca-1f2ba4c7b6ac/ascreenshot_da99f5761bbf44a08af4f1e1175a95fc_text_export.jpeg) + +23. Expand **Step 2** (e.g. **Insults & Personal Attacks**) to confirm **PASS** and **Allow** after the fallback + +![Expand Step 2 — second guardrail outcome](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/cac5273c-dd4f-48a0-af58-12c428d0f0d0/ascreenshot_f74da58e280a47319a7d2fa41519f4fb_text_export.jpeg) ## Config (YAML) diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 2f81498799a..166269af47c 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -351,7 +351,7 @@ We will use the `--config` to set `litellm.success_callback = ["langfuse"]` this **Step 1** Install langfuse ```shell -pip install langfuse>=2.0.0 +uv add langfuse>=2.0.0 ``` **Step 2**: Create a `config.yaml` file and set `litellm_settings`: `success_callback` @@ -982,7 +982,7 @@ OTEL_ENDPOINT="http:/0.0.0.0:4317" OTEL_HEADERS="x-honeycomb-team=" # Optional ``` -> Note: OTLP gRPC requires `grpcio`. Install via `pip install "litellm[grpc]"` (or `grpcio`). +> Note: OTLP gRPC requires `grpcio`. Install via `uv add "litellm[grpc]"` (or `grpcio`). Add `otel` as a callback on your `litellm_config.yaml` @@ -1587,7 +1587,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ #### Step1: Install dependencies and set your environment variables Install the dependencies ```shell -pip install litellm lunary +uv add litellm lunary ``` Get you Lunary public key from from https://app.lunary.ai/settings @@ -2516,7 +2516,7 @@ If api calls fail (llm/database) you can log those to Sentry: **Step 1** Install Sentry ```shell -pip install --upgrade sentry-sdk +uv add --upgrade sentry-sdk ``` **Step 2**: Save your Sentry_DSN and add `litellm_settings`: `failure_callback` diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md index d8f0d83b59d..33459572471 100644 --- a/docs/my-website/docs/proxy/prometheus.md +++ b/docs/my-website/docs/proxy/prometheus.md @@ -9,7 +9,7 @@ LiteLLM Exposes a `/metrics` endpoint for Prometheus to Poll ## Quick Start -If you're using the LiteLLM CLI with `litellm --config proxy_config.yaml` then you need to `pip install prometheus_client==0.20.0`. **This is already pre-installed on the litellm Docker image** +If you're using the LiteLLM CLI with `litellm --config proxy_config.yaml` then you need to `uv add prometheus_client==0.20.0`. **This is already pre-installed on the litellm Docker image** Add this to your proxy config.yaml ```yaml diff --git a/docs/my-website/docs/proxy/pyroscope_profiling.md b/docs/my-website/docs/proxy/pyroscope_profiling.md index fa3db3a8782..19d12ba24ea 100644 --- a/docs/my-website/docs/proxy/pyroscope_profiling.md +++ b/docs/my-website/docs/proxy/pyroscope_profiling.md @@ -7,13 +7,13 @@ LiteLLM proxy can send continuous CPU profiles to [Grafana Pyroscope](https://gr 1. **Install the optional dependency** (required only when enabling Pyroscope): ```bash - pip install pyroscope-io + uv add pyroscope-io ``` Or install the proxy extra: ```bash - pip install "litellm[proxy]" + uv add "litellm[proxy]" ``` 2. **Set environment variables** before starting the proxy: diff --git a/docs/my-website/docs/proxy/quick_start.md b/docs/my-website/docs/proxy/quick_start.md index cf1ab78b352..dbc018e129d 100644 --- a/docs/my-website/docs/proxy/quick_start.md +++ b/docs/my-website/docs/proxy/quick_start.md @@ -13,7 +13,7 @@ LiteLLM Server (LLM Gateway) manages: * **Load Balancing**: between [Multiple Models](#multiple-models---quick-start) + [Deployments of the same model](#multiple-instances-of-1-model) - LiteLLM proxy can handle 1.5k+ requests/second during load tests. ```shell -$ pip install 'litellm[proxy]' +$ uv tool install 'litellm[proxy]' ``` ## Quick Start - LiteLLM Proxy CLI diff --git a/docs/my-website/docs/proxy/user_keys.md b/docs/my-website/docs/proxy/user_keys.md index 72ec8ccd759..7bce1523217 100644 --- a/docs/my-website/docs/proxy/user_keys.md +++ b/docs/my-website/docs/proxy/user_keys.md @@ -881,7 +881,7 @@ Credits [@vividfog](https://github.com/ollama/ollama/issues/305#issuecomment-175 ```shell -$ pip install aider +$ uv add aider $ aider --openai-api-base http://0.0.0.0:4000 --openai-api-key fake-key ``` @@ -889,7 +889,7 @@ $ aider --openai-api-base http://0.0.0.0:4000 --openai-api-key fake-key ```python -pip install pyautogen +uv add pyautogen ``` ```python diff --git a/docs/my-website/docs/proxy_api.md b/docs/my-website/docs/proxy_api.md index 7612645fb54..73c5a565874 100644 --- a/docs/my-website/docs/proxy_api.md +++ b/docs/my-website/docs/proxy_api.md @@ -66,16 +66,16 @@ git clone https://github.com/krrishdholakia/open-interpreter-litellm-fork ``` To run it do: ``` -poetry build +uv build # call gpt-4 - always add 'litellm_proxy/' in front of the model name -poetry run interpreter --model litellm_proxy/gpt-4 +uv run interpreter --model litellm_proxy/gpt-4 # call llama-70b - always add 'litellm_proxy/' in front of the model name -poetry run interpreter --model litellm_proxy/togethercomputer/llama-2-70b-chat +uv run interpreter --model litellm_proxy/togethercomputer/llama-2-70b-chat # call claude-2 - always add 'litellm_proxy/' in front of the model name -poetry run interpreter --model litellm_proxy/claude-2 +uv run interpreter --model litellm_proxy/claude-2 ``` And that's it! @@ -83,4 +83,4 @@ And that's it! Now you can call any model you like! -Want us to add more models? [Let us know!](https://github.com/BerriAI/litellm/issues/new/choose) \ No newline at end of file +Want us to add more models? [Let us know!](https://github.com/BerriAI/litellm/issues/new/choose) diff --git a/docs/my-website/docs/proxy_auth.md b/docs/my-website/docs/proxy_auth.md index 91084b34a37..bb5601cb85f 100644 --- a/docs/my-website/docs/proxy_auth.md +++ b/docs/my-website/docs/proxy_auth.md @@ -72,7 +72,7 @@ response = litellm.completion( -**Required package:** `pip install azure-identity` +**Required package:** `uv add azure-identity` ### Generic OAuth2 (Okta, Auth0, Keycloak, etc.) diff --git a/docs/my-website/docs/proxy_server.md b/docs/my-website/docs/proxy_server.md index 7b6f15a604b..1c056207534 100644 --- a/docs/my-website/docs/proxy_server.md +++ b/docs/my-website/docs/proxy_server.md @@ -13,7 +13,7 @@ Docs outdated. New docs 👉 [here](./simple_proxy) ## Usage ```shell -pip install 'litellm[proxy]' +uv tool install 'litellm[proxy]' ``` ```shell $ litellm --model ollama/codellama @@ -213,7 +213,7 @@ docker compose up -d ```python -pip install pyautogen +uv add pyautogen ``` ```python @@ -329,7 +329,7 @@ git clone https://github.com/OpenBMB/ChatDev.git cd ChatDev conda create -n ChatDev_conda_env python=3.9 -y conda activate ChatDev_conda_env -pip install -r requirements.txt +uv add -r requirements.txt ``` ### Run ChatDev w/ Proxy ```shell @@ -346,7 +346,7 @@ python3 run.py --task "a script that says hello world" --name "hello world" ```python -pip install langroid +uv add langroid ``` ```python @@ -383,7 +383,7 @@ Credits [@pchalasani](https://github.com/pchalasani) and [Langroid](https://gith Here's how to use the local proxy to test codellama/mistral/etc. models for different github repos ```shell -pip install litellm +uv add litellm ``` ```shell @@ -440,7 +440,7 @@ Credits [@vividfog](https://github.com/ollama/ollama/issues/305#issuecomment-175 ```shell -$ pip install aider +$ uv add aider $ aider --openai-api-base http://0.0.0.0:8000 --openai-api-key fake-key ``` @@ -448,7 +448,7 @@ $ aider --openai-api-base http://0.0.0.0:8000 --openai-api-key fake-key ```python -pip install pyautogen +uv add pyautogen ``` ```python @@ -564,7 +564,7 @@ git clone https://github.com/OpenBMB/ChatDev.git cd ChatDev conda create -n ChatDev_conda_env python=3.9 -y conda activate ChatDev_conda_env -pip install -r requirements.txt +uv add -r requirements.txt ``` ### Run ChatDev w/ Proxy ```shell @@ -581,7 +581,7 @@ python3 run.py --task "a script that says hello world" --name "hello world" ```python -pip install langroid +uv add langroid ``` ```python diff --git a/docs/my-website/docs/rag_ingest.md b/docs/my-website/docs/rag_ingest.md index 7adc2d70b5b..35b2cf4c327 100644 --- a/docs/my-website/docs/rag_ingest.md +++ b/docs/my-website/docs/rag_ingest.md @@ -287,7 +287,7 @@ When `vector_store_id` is omitted, LiteLLM automatically creates: 1. Create a RAG corpus in Vertex AI console or via API 2. Create a GCS bucket for file uploads 3. Authenticate via `gcloud auth application-default login` -4. Install: `pip install 'google-cloud-aiplatform>=1.60.0'` +4. Install: `uv add 'google-cloud-aiplatform>=1.60.0'` ::: ### vector_store (AWS S3 Vectors) diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md index 0c428000c72..3ab61a97a4e 100644 --- a/docs/my-website/docs/response_api.md +++ b/docs/my-website/docs/response_api.md @@ -831,7 +831,7 @@ The system automatically selects the appropriate mode based on provider capabili ```python showLineNumbers title="WebSocket with Python" import json -from websocket import create_connection # pip install websocket-client +from websocket import create_connection # uv add websocket-client # Connect to LiteLLM proxy WebSocket endpoint ws = create_connection( diff --git a/docs/my-website/docs/sdk_custom_pricing.md b/docs/my-website/docs/sdk_custom_pricing.md index c8577115109..011229abe58 100644 --- a/docs/my-website/docs/sdk_custom_pricing.md +++ b/docs/my-website/docs/sdk_custom_pricing.md @@ -5,7 +5,7 @@ Register custom pricing for sagemaker completion model. For cost per second pricing, you **just** need to register `input_cost_per_second`. ```python -# !pip install boto3 +# !uv add boto3 from litellm import completion, completion_cost os.environ["AWS_ACCESS_KEY_ID"] = "" @@ -35,7 +35,7 @@ def test_completion_sagemaker(): ```python -# !pip install boto3 +# !uv add boto3 from litellm import completion, completion_cost ## set ENV variables diff --git a/docs/my-website/docs/secret_managers/azure_key_vault.md b/docs/my-website/docs/secret_managers/azure_key_vault.md index 4ea53d2ea9e..3e697ebdedc 100644 --- a/docs/my-website/docs/secret_managers/azure_key_vault.md +++ b/docs/my-website/docs/secret_managers/azure_key_vault.md @@ -14,7 +14,7 @@ 1. Install Proxy dependencies ```bash -pip install 'litellm[proxy]' 'litellm[extra_proxy]' +uv tool install 'litellm[proxy]' 'litellm[extra_proxy]' ``` 2. Save Azure details in your environment diff --git a/docs/my-website/docs/troubleshoot/pip_venv_upgrade.md b/docs/my-website/docs/troubleshoot/pip_venv_upgrade.md index 6f5699e3fb0..3bdaa6a05a6 100644 --- a/docs/my-website/docs/troubleshoot/pip_venv_upgrade.md +++ b/docs/my-website/docs/troubleshoot/pip_venv_upgrade.md @@ -1,21 +1,21 @@ -# Upgrading LiteLLM Proxy (pip/venv) +# Upgrading LiteLLM Proxy (uv/venv) -Guide for upgrading LiteLLM Proxy when installed via pip in a virtual environment. +Guide for upgrading LiteLLM Proxy when installed via uv in a virtual environment. :::info Important Always activate your virtual environment before running any `litellm` or `prisma` commands. All commands in this guide assume you're working inside an activated venv. ::: -## How pip/venv Upgrades Work +## How uv/venv Upgrades Work There are two pieces that need to stay in sync: 1. **Prisma client** - Generated Python code that talks to the DB 2. **DB schema** - Tables/columns in PostgreSQL -When you upgrade via pip, the `litellm-proxy-extras` package ships with a new `schema.prisma` and a `migrations/` directory. But unlike the Docker image, pip install does NOT automatically regenerate the Prisma client or run migrations. You have to do both manually. +When you upgrade via uv, the `litellm-proxy-extras` package ships with a new `schema.prisma` and a `migrations/` directory. But unlike the Docker image, `uv add` does not automatically regenerate the Prisma client or run migrations. You have to do both manually. -## Upgrade Workflow (pip/venv) +## Upgrade Workflow (uv/venv) ### 1. Stop the proxy @@ -30,7 +30,7 @@ pg_dump -h -U -d -F c -f backup_$(date +%Y%m%d).dump ### 3. Upgrade the package ```bash -pip install 'litellm[proxy]==' +uv add 'litellm[proxy]==' ``` ### 4. Regenerate the Prisma client @@ -91,7 +91,7 @@ litellm --config your_config.yaml --port 4000 ### Before applying migrations: Preview what will change -Run `pip install 'litellm[proxy]=='` first (Step 3) so the new `schema.prisma` is available. +Run `uv add 'litellm[proxy]=='` first (Step 3) so the new `schema.prisma` is available. ```bash prisma migrate diff \ diff --git a/docs/my-website/docs/tutorials/TogetherAI_liteLLM.md b/docs/my-website/docs/tutorials/TogetherAI_liteLLM.md index dd9dd288672..97159dbba4c 100644 --- a/docs/my-website/docs/tutorials/TogetherAI_liteLLM.md +++ b/docs/my-website/docs/tutorials/TogetherAI_liteLLM.md @@ -4,7 +4,7 @@ https://together.ai/ ```python -!pip install litellm +!uv add litellm ``` diff --git a/docs/my-website/docs/tutorials/claude_agent_sdk.md b/docs/my-website/docs/tutorials/claude_agent_sdk.md index c56784ba2df..f01fc778c43 100644 --- a/docs/my-website/docs/tutorials/claude_agent_sdk.md +++ b/docs/my-website/docs/tutorials/claude_agent_sdk.md @@ -12,7 +12,7 @@ The Claude Agent SDK provides a high-level interface for building AI agents. By ### 1. Install Dependencies ```bash -pip install claude-agent-sdk +uv add claude-agent-sdk ``` ### 2. Start LiteLLM Proxy @@ -104,7 +104,7 @@ See our [cookbook example](https://github.com/BerriAI/litellm/tree/main/cookbook # Clone and run the example git clone https://github.com/BerriAI/litellm.git cd litellm/cookbook/anthropic_agent_sdk -pip install -r requirements.txt +uv add -r requirements.txt python main.py ``` diff --git a/docs/my-website/docs/tutorials/claude_non_anthropic_models.md b/docs/my-website/docs/tutorials/claude_non_anthropic_models.md index 75ac08e3094..0bba0f8ad06 100644 --- a/docs/my-website/docs/tutorials/claude_non_anthropic_models.md +++ b/docs/my-website/docs/tutorials/claude_non_anthropic_models.md @@ -22,7 +22,7 @@ LiteLLM automatically translates between different provider formats, allowing yo First, install LiteLLM with proxy support: ```bash -pip install 'litellm[proxy]' +uv tool install 'litellm[proxy]' ``` ## Configuration diff --git a/docs/my-website/docs/tutorials/claude_responses_api.md b/docs/my-website/docs/tutorials/claude_responses_api.md index 2a6a1236ab1..bf46036f228 100644 --- a/docs/my-website/docs/tutorials/claude_responses_api.md +++ b/docs/my-website/docs/tutorials/claude_responses_api.md @@ -28,7 +28,7 @@ This tutorial is based on [Anthropic's official LiteLLM configuration documentat First, install LiteLLM with proxy support: ```bash -pip install 'litellm[proxy]' +uv tool install 'litellm[proxy]' ``` ### 1. Setup config.yaml diff --git a/docs/my-website/docs/tutorials/compare_llms.md b/docs/my-website/docs/tutorials/compare_llms.md index 0252263a16e..72c27aa2f1e 100644 --- a/docs/my-website/docs/tutorials/compare_llms.md +++ b/docs/my-website/docs/tutorials/compare_llms.md @@ -23,7 +23,7 @@ cd litellm/cookbook/benchmark ### Install Dependencies ``` -pip install litellm click tqdm tabulate termcolor +uv add litellm click tqdm tabulate termcolor ``` ### Configuration - Set LLM API Keys + LLMs in benchmark.py @@ -88,7 +88,7 @@ Benchmark Results for 'When will BerriAI IPO?':