diff --git a/.circleci/AGENTS.md b/.circleci/AGENTS.md deleted file mode 100644 index b3591145a3e..00000000000 --- a/.circleci/AGENTS.md +++ /dev/null @@ -1,7 +0,0 @@ -# .circleci - -The `integration` workflow on `main` runs from a scheduled pipeline in the CircleCI project settings, cron `9 0-8,14-23 * * *` (UTC) on `main` with no pipeline parameters. It has no `triggers:` block in this file because a workflow with one runs only on its schedule, so the API pipelines that set `run_migration_tests` or `routing_parity_base` would stop running it. If main stops getting integration runs, check that the schedule still exists and recreate it with these values - -Jobs clone with the `checkout_over_https` command, never the built-in `checkout` step, because CircleCI falls back to the project's SSH checkout key without its GitHub App access, GitHub rejects that key, and the public repo needs no key. It fetches the job's commit with full history and no file contents (`--filter=blob:none`), like the built-in step did, so `scripts/path_filter.sh` can find a merge base - -A step that checks out files from another commit fetches that commit with `--no-filter`, as `routing_parity` does, because the clone has no file contents and git would otherwise download each missing file in its own request, which took 210 to 896 seconds for the routing parity base diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 15129f1e5e3..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,1785 +0,0 @@ -version: 2.1 -parameters: - run_migration_tests: - type: boolean - default: false - migration_candidate_image: - type: string - default: "" - migration_baseline_image: - type: string - default: "ghcr.io/berriai/litellm-database:v1.102.0" - migration_source_sha: - type: string - default: "" - routing_parity_base: - type: string - default: "" -orbs: - codecov: codecov/codecov@4.0.1 - node: circleci/node@5.1.0 # Add this line to declare the node orb - win: circleci/windows@5.0 # Add Windows orb - -commands: - checkout_over_https: - parameters: - revision: - type: string - default: "" - shell: - type: string - default: /bin/bash -eo pipefail - steps: - - run: - name: Check out the public repository over HTTPS - shell: << parameters.shell >> - environment: - REQUESTED_REVISION: << parameters.revision >> - command: | - revision="${REQUESTED_REVISION:-$CIRCLE_SHA1}" - [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1 - git init - git remote add origin https://github.com/BerriAI/litellm.git - for attempt in 1 2 3; do - git fetch --filter=blob:none origin "$revision" && git checkout --detach FETCH_HEAD && break - [[ $attempt -lt 3 ]] || exit 1 - sleep 5 - done - skip_if_unrelated_changes: - parameters: - category: - type: enum - enum: ["backend", "client"] - default: "backend" - steps: - - run: - name: "Skip job when no << parameters.category >>-relevant files changed" - command: bash .circleci/scripts/path_filter.sh << parameters.category >> - setup_google_dns: - steps: - - run: - name: "Configure Google DNS" - command: | - # Backup original resolv.conf - sudo cp /etc/resolv.conf /etc/resolv.conf.backup - # Add both local and Google DNS servers - echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf - echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf - echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf - wait_for_service: - description: "Poll a TCP or HTTP endpoint until it responds (replaces dockerize -wait)" - parameters: - url: - type: string - timeout: - type: string - default: "60" - steps: - - run: - name: "Wait for << parameters.url >>" - command: | - TIMEOUT=<< parameters.timeout >> - URL="<< parameters.url >>" - ELAPSED=0 - echo "Waiting up to ${TIMEOUT}s for ${URL} ..." - if echo "$URL" | grep -q '^tcp://'; then - HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1) - PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2) - while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do - sleep 2; ELAPSED=$((ELAPSED+2)) - if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi - done - else - while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do - sleep 2; ELAPSED=$((ELAPSED+2)) - if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi - done - fi - echo "Service ready after ${ELAPSED}s" - install_helm: - steps: - - run: - name: Install Helm v3.17.3 - command: | - curl -sSLf -o /tmp/helm.tar.gz \ - https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz - echo "ee88b3c851ae6466a3de507f7be73fe94d54cbf2987cbaa3d1a3832ea331f2cd /tmp/helm.tar.gz" | sha256sum -c - - sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/helm.tar.gz linux-amd64/helm - rm -f /tmp/helm.tar.gz - install_kind: - steps: - - run: - name: Install Kind v0.20.0 - command: | - curl -sSLf -o /tmp/kind \ - https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 - echo "513a7213d6d3332dd9ef27c24dab35e5ef10a04fa27274fe1c14d8a246493ded /tmp/kind" | sha256sum -c - - chmod +x /tmp/kind - sudo mv /tmp/kind /usr/local/bin/kind - install_uv: - description: "Install pinned uv (0.10.9) with checksum verification. Adds ~/.local/bin to PATH." - steps: - - run: - name: Install uv (pinned 0.10.9) - command: | - curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh - echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh - rm -f /tmp/uv-install.sh - echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" - export PATH="$HOME/.local/bin:$PATH" - install_node: - description: "Install the Node.js version pinned in ui/litellm-dashboard/.nvmrc (24.19.0, which bundles npm 11.17.0) with checksum verification, and prepend it to PATH. Run this on any executor whose image does not already ship that version, or `npm ci` in ui/litellm-dashboard fails EBADENGINE against the engines floor. Installs into /opt/node rather than over /usr/local on purpose: cimg/python:*-browsers ships its own node there, and unpacking the tarball on top of it leaves npm 11.17 files merged with the image's npm 11.9 tree, which reports the new version and then exits 1 on `npm ci` with no error text at all. Requires checkout, which the .nvmrc drift check reads." - steps: - - run: - name: Install Node.js 24.19.0 - command: | - NODE_VERSION="24.19.0" - NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz" - NODE_EXPECTED_SHA="14b342e71204f811bde6153be8e04b62aef63c236fef92b55f9c83154b409647" - NVMRC_VERSION="$(tr -d '[:space:]' < ui/litellm-dashboard/.nvmrc)" - if [ "$NVMRC_VERSION" != "$NODE_VERSION" ]; then - echo "install_node: ui/litellm-dashboard/.nvmrc pins ${NVMRC_VERSION} but this command pins ${NODE_VERSION}; update NODE_VERSION and NODE_EXPECTED_SHA together" >&2 - exit 1 - fi - curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" - echo "${NODE_EXPECTED_SHA} /tmp/${NODE_TARBALL}" | sha256sum -c - - sudo mkdir -p /opt/node - sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /opt/node --strip-components=1 - rm -f "/tmp/${NODE_TARBALL}" - echo 'export PATH="/opt/node/bin:$PATH"' >> "$BASH_ENV" - export PATH="/opt/node/bin:$PATH" - node --version - npm --version - install_rust: - description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself. Also restores the dev-profile cargo cache that save_cargo_target writes on main, minus the workspace crates' fingerprints so those always rebuild from the checked-out source." - steps: - - run: - name: Install Rust (rustup 1.28.2, toolchain 1.98.0) - command: | - case "$(uname -m)" in - x86_64) - RUSTUP_TRIPLE=x86_64-unknown-linux-gnu - RUSTUP_SHA256=20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c - ;; - aarch64) - RUSTUP_TRIPLE=aarch64-unknown-linux-gnu - RUSTUP_SHA256=e3853c5a252fca15252d07cb23a1bdd9377a8c6f3efa01531109281ae47f841c - ;; - *) - echo "install_rust: unsupported architecture $(uname -m)" >&2 - exit 1 - ;; - esac - curl -sSLf --retry 3 --retry-all-errors --retry-max-time 120 --connect-timeout 15 --max-time 60 -o /tmp/rustup-init \ - "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init" - echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c - - chmod +x /tmp/rustup-init - /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0 - rm -f /tmp/rustup-init - echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV" - echo 'export CARGO_INCREMENTAL=0' >> "$BASH_ENV" - export PATH="$HOME/.cargo/bin:$PATH" - rustc --version - cargo --version - { rustc -vV; cc --version; cat /etc/os-release; } > /tmp/cargo-build-env - - restore_cache: - keys: - - v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }} - - v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}- - - run: - name: Force a rebuild of the workspace crates restored from the cargo cache - command: rm -rf litellm-rust/target/debug/.fingerprint/litellm-* - save_cargo_target: - steps: - - when: - condition: - equal: [main, << pipeline.git.branch >>] - steps: - - save_cache: - key: v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }} - paths: - - ~/.cargo/registry - - ~/project/litellm-rust/target/debug - start_postgres: - description: "Start a postgres-db container on port 5432 and wait until it accepts connections." - parameters: - db_name: - type: string - default: circle_test - image: - type: string - default: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26 - server_args: - type: string - default: "" - steps: - - run: - name: Start PostgreSQL - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=<< parameters.db_name >> \ - -p 5432:5432 \ - << parameters.image >> << parameters.server_args >> - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - start_redis: - description: "Start a redis container on port 6379 and wait until it accepts connections. Use this to isolate a job from the shared remote Redis so concurrent CI pipelines don't contend for pod locks or buffer keys." - steps: - - run: - name: Start Redis - command: | - docker run -d \ - --name redis-cache \ - -p 6379:6379 \ - redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6 - - wait_for_service: - url: tcp://localhost:6379 - timeout: "60" - install_codecov_cli: - steps: - - run: - name: Install Codecov CLI (pinned v11.3.1) - when: always - command: | - curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov - curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM - [ "$(cat /tmp/codecov.SHA256SUM)" = "ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d codecov" ] - (cd /tmp && sha256sum -c codecov.SHA256SUM) - chmod +x /tmp/codecov - mkdir -p "$HOME/.local/bin" - mv /tmp/codecov "$HOME/.local/bin/codecov" - start_openai_record_replay_proxy: - description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay Redis cache hits. Cache misses call the live provider and store its response, so these legacy E2E jobs still require provider availability and may incur provider charges. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream// so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced." - steps: - - run: - name: Start record/replay proxy - background: true - command: | - CASSETTE_REDIS_URL="$CASSETTE_REDIS_URL" \ - RECORDER_UPSTREAM_BASE_URL="https://api.openai.com" \ - uv run --no-sync python tests/_openai_record_replay_proxy.py --host 0.0.0.0 --port 8090 - - run: - name: Wait for record/replay proxy - command: | - for i in $(seq 1 30); do - if curl -sf http://localhost:8090/__recorder_health >/dev/null 2>&1; then - echo "record/replay proxy is up" - exit 0 - fi - sleep 1 - done - echo "record/replay proxy did not become ready" >&2 - exit 1 - start_fake_openai_endpoint: - description: "Start the canned OpenAI mock (tests/_fake_openai_endpoint_server.py) on host port 8190 and wait until healthy. Models whose api_base points here (via FAKE_OPENAI_API_BASE) get well-formed chat/text/embedding responses with realistic usage, so the E2E run neither pays for nor depends on the live provider. A request whose model is '429' returns HTTP 429 for rate-limit/cooldown tests. Run after uv deps are synced." - steps: - - run: - name: Start fake OpenAI endpoint - background: true - command: | - uv run --no-sync python tests/_fake_openai_endpoint_server.py --host 0.0.0.0 --port 8190 - - run: - name: Wait for fake OpenAI endpoint - command: | - for i in $(seq 1 30); do - if curl -sf http://localhost:8190/health >/dev/null 2>&1; then - echo "fake OpenAI endpoint is up" - exit 0 - fi - sleep 1 - done - echo "fake OpenAI endpoint did not become ready" >&2 - exit 1 - setup_litellm_enterprise_pip: - steps: - - run: - name: "Install local version of litellm-enterprise" - command: | - # litellm-enterprise is a uv workspace member and is already installed - # by the main `uv sync --all-groups --all-extras`. Do NOT run - # `uv sync --package litellm-enterprise` here — that overwrites the - # shared .venv and strips out dev/test deps (pytest, prisma, etc.). - uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)" - install_windows_toolchain: - steps: - - run: - name: Install Rust and uv - no_output_timeout: 30m - command: | - $rustupInit = Join-Path $env:TEMP "rustup-init.exe" - $rustupVersion = "1.28.2" - $rustupUrl = "https://static.rust-lang.org/rustup/archive/$rustupVersion/x86_64-pc-windows-msvc/rustup-init.exe" - Invoke-WebRequest -Uri $rustupUrl -OutFile $rustupInit - $rustupExpected = "88d8258dcf6ae4f7a80c7d1088e1f36fa7025a1cfd1343731b4ee6f385121fc0" - $rustupActual = (Get-FileHash -Path $rustupInit -Algorithm SHA256).Hash.ToLower() - if ($rustupActual -ne $rustupExpected) { - throw "rustup installer hash mismatch: expected $rustupExpected got $rustupActual" - } - & $rustupInit -y --profile minimal --default-toolchain 1.98.0 - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - Remove-Item $rustupInit - $cargoBin = Join-Path $HOME ".cargo\bin" - $env:Path = "$cargoBin;$env:Path" - rustc --version - cargo --version - $installer = Join-Path $env:TEMP "uv-install.ps1" - Invoke-WebRequest -Uri https://astral.sh/uv/0.10.9/install.ps1 -OutFile $installer - $expected = "d43ffff8d28e7d1e7d1831a212465f12b24a43c7f87f386e95e2a5915aee5d7d" - $actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower() - if ($actual -ne $expected) { - throw "uv installer hash mismatch: expected $expected got $actual" - } - & $installer - Remove-Item $installer - $uvBin = Join-Path $HOME ".local\bin" - $env:Path = "$uvBin;$env:Path" - if (!(Test-Path $PROFILE)) { - New-Item -ItemType File -Force -Path $PROFILE | Out-Null - } - if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) { - Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`"" - } - if (-not (Select-String -Path $PROFILE -SimpleMatch $cargoBin -Quiet)) { - Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`"" - } - setup_litellm_test_deps: - parameters: - checkout: - type: boolean - default: true - steps: - - when: - condition: << parameters.checkout >> - steps: - - checkout_over_https - - setup_google_dns - - install_uv - - install_rust - - restore_cache: - keys: - - v3-integration-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ~/.cache/uv - key: v3-integration-uv-cache-{{ checksum "uv.lock" }} - - save_cargo_target - -jobs: - windows_release_wheel: - executor: - name: win/default - shell: powershell.exe - size: xlarge - working_directory: ~/project - environment: - UV_PYTHON: "3.11" - CARGO_HTTP_MULTIPLEXING: "false" - CARGO_NET_RETRY: "5" - steps: - - checkout_over_https: - shell: bash.exe - - run: - name: Skip job when no windows-release-relevant files changed - shell: bash.exe - command: bash .circleci/scripts/path_filter.sh windows-release - - run: - name: Install Python 3.11.9 from python.org - no_output_timeout: 6m - command: | - $ProgressPreference = "SilentlyContinue" - $installer = Join-Path $env:TEMP "python-3.11.9-amd64.exe" - Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe -OutFile $installer -TimeoutSec 120 - $expected = "5ee42c4eee1e6b4464bb23722f90b45303f79442df63083f05322f1785f5fdde" - $actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower() - if ($actual -ne $expected) { - throw "Python installer hash mismatch: expected $expected got $actual" - } - $arguments = "/quiet InstallAllUsers=1 PrependPath=1 Include_test=0 Include_doc=0 Include_launcher=0 TargetDir=C:\Python311" - $process = Start-Process -FilePath $installer -ArgumentList $arguments -PassThru - if (-not $process.WaitForExit(300000)) { - $process.Kill() - throw "Python installer did not finish within 5 minutes" - } - if ($process.ExitCode -ne 0) { - throw "Python installer exited with $($process.ExitCode)" - } - Remove-Item $installer - & C:\Python311\python.exe --version - - install_windows_toolchain - - run: - name: Record the Rust build environment for the release cargo cache key - command: | - & "$HOME\.cargo\bin\rustc.exe" -vV | Out-File -Encoding ascii .cargo-build-env - - restore_cache: - keys: - - v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }} - - v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}- - - run: - name: Force a rebuild of the workspace crates restored from the cargo cache - command: | - $fingerprints = "litellm-rust/target/release/.fingerprint" - if (Test-Path $fingerprints) { - Get-ChildItem -Path $fingerprints -Filter "litellm-*" | Remove-Item -Recurse -Force - } - - run: - name: Build the release wheel and install it under a worst-case MAX_PATH prefix - no_output_timeout: 30m - environment: - UV_HTTP_TIMEOUT: "300" - command: | - $env:Path = "C:\Python311;C:\Python311\Scripts;$HOME\.cargo\bin;$HOME\.local\bin;$env:Path" - uv build --wheel --out-dir dist - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - python tests/windows_tests/check_windows_wheel_install.py - - when: - condition: - equal: [main, << pipeline.git.branch >>] - steps: - - save_cache: - key: v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }} - paths: - - ~/.cargo/registry - - ~/project/litellm-rust/target/release - - base_sdk_install: - docker: - - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - steps: - - checkout_over_https - - setup_google_dns - - install_uv - - install_rust - - run: - name: Build the wheel - no_output_timeout: 30m - environment: - UV_HTTP_TIMEOUT: "300" - command: | - uv build --wheel --out-dir dist - - run: - name: Install the wheel with no extras and smoke-check it - environment: - UV_HTTP_TIMEOUT: "300" - command: | - uv venv /tmp/base-sdk --python 3.12 - VIRTUAL_ENV=/tmp/base-sdk uv pip install dist/*.whl - /tmp/base-sdk/bin/python tests/base_sdk_tests/check_base_sdk_install.py - - run: - name: Guard against MAX_PATH-busting packaged wheel paths - command: | - python3 tests/windows_tests/check_windows_wheel_install.py --lengths-only - - helm_chart_testing: - machine: - image: ubuntu-2204:2024.04.1 # Use machine executor instead of docker - resource_class: medium - working_directory: ~/project - - steps: - - checkout_over_https - - skip_if_unrelated_changes - - attach_workspace: - at: ~/project - - setup_google_dns - - install_helm - - install_kind - - # Install kubectl (pinned version with hardcoded checksum) - - run: - name: Install kubectl v1.31.4 - command: | - curl -sSLf -o /tmp/kubectl \ - https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl - echo "298e19e9c6c17199011404278f0ff8168a7eca4217edad9097af577023a5620f /tmp/kubectl" | sha256sum -c - - chmod +x /tmp/kubectl - sudo mv /tmp/kubectl /usr/local/bin/ - - # Create kind cluster - - run: - name: Create Kind Cluster - command: | - kind create cluster --name litellm-test - - - run: - name: Load Docker Database Image for helm tests - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - IMAGE_TAG=${CIRCLE_SHA1:-ci} - docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG} - - - run: - name: Load Docker image into Kind - command: | - IMAGE_TAG=${CIRCLE_SHA1:-ci} - kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test - - - run: - name: Run helm lint - command: | - helm lint ./helm/litellm -f ./helm/litellm/ci/test-values.yaml - - - run: - name: Start PostgreSQL for the chart - command: | - kubectl apply -f ./helm/litellm/ci/postgres.yaml - kubectl rollout status deployment/litellm-ci-postgres --timeout=3m - - - run: - name: Run helm tests - command: | - IMAGE_TAG=${CIRCLE_SHA1:-ci} - helm install litellm ./helm/litellm -f ./helm/litellm/ci/test-values.yaml \ - --set image.repository=litellm-ci \ - --set image.tag=${IMAGE_TAG} \ - --set image.pullPolicy=Never \ - --wait --wait-for-jobs --timeout 10m || { - kubectl get pods -o wide - kubectl describe pods -l app.kubernetes.io/instance=litellm - kubectl logs -l app.kubernetes.io/component=proxy --tail=200 || true - kubectl logs -l app.kubernetes.io/component=migrations --tail=200 || true - exit 1 - } - - echo "Printing proxy pod logs..." - kubectl logs -l app.kubernetes.io/component=proxy --tail=200 - - helm test litellm --logs - - # Cleanup - - run: - name: Cleanup - command: | - kind delete cluster --name litellm-test - when: always # This ensures cleanup runs even if previous steps fail - - db_migration_disable_update_check: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: medium - working_directory: ~/project - steps: - - checkout_over_https - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - skip_if_unrelated_changes - - setup_google_dns - - install_uv - - install_rust - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_fake_openai_endpoint - - start_postgres: - db_name: litellm_test - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - run: - name: Seed database with real schema - command: | - docker run -d \ - -p 4001:4000 \ - -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ - -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \ - --name schema-seed \ - --add-host=host.docker.internal:host-gateway \ - -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --use_prisma_db_push - - wait_for_service: - url: http://localhost:4001 - timeout: "300" - - run: - name: Stop schema seed container - command: docker stop schema-seed && docker rm schema-seed - - run: - name: Run Docker container with bad schema and disabled updates - command: | - docker run -d \ - -p 4000:4000 \ - -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \ - -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ - -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ - -e DISABLE_SCHEMA_UPDATE="True" \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - --name my-app \ - --add-host=host.docker.internal:host-gateway \ - -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ - -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ - -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 - - wait_for_service: - url: http://localhost:4000 - timeout: "60" - - run: - name: Check container logs for expected message - command: | - echo "=== Printing Full Container Startup Logs ===" - LOG_OUTPUT="$(docker logs my-app 2>&1)" - printf '%s\n' "$LOG_OUTPUT" - echo "=== End of Full Container Startup Logs ===" - - if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then - echo "Expected message found in logs. Test passed." - else - echo "Expected message not found in logs. Test failed." - exit 1 - fi - - run: - name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/integration/configuration/test_basic_proxy_startup.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \ - -v \ - --junitxml=test-results/junit-2.xml \ - --durations=5" - no_output_timeout: 15m - - store_test_results: - path: test-results - - proxy_logging_guardrails_model_info_tests: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout_over_https - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - skip_if_unrelated_changes - - setup_google_dns - - install_uv - - install_rust - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_postgres - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - start_openai_record_replay_proxy - - start_fake_openai_endpoint - # Second Docker Container Run with Different Config - # NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license - - run: - name: Run Second Docker container - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e REDIS_HOST=$REDIS_HOST \ - -e REDIS_PASSWORD=$REDIS_PASSWORD \ - -e REDIS_PORT=$REDIS_PORT \ - -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - -e LITELLM_LICENSE="bad-license" \ - -e LITELLM_LOG=ERROR \ - --add-host host.docker.internal:host-gateway \ - --name my-app-3 \ - -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 - - - run: - name: Start outputting logs for second container - command: docker logs -f my-app-3 - background: true - - - wait_for_service: - url: http://localhost:4000 - timeout: "300" - - - run: - name: Run second round of tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/integration/configuration/test_basic_proxy_startup.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \ - -v \ - --junitxml=test-results/junit-2.xml \ - --durations=5" - no_output_timeout: 15m - - # Store test results - - store_test_results: - path: test-results - e2e_ui_build: - docker: - - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - resource_class: large - working_directory: ~/project - steps: - - checkout_over_https - - skip_if_unrelated_changes: - category: client - - setup_google_dns - - install_node - - restore_cache: - keys: - - ui-dashboard-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - - run: - name: Install dashboard dependencies - command: | - cd ui/litellm-dashboard - npm ci - - save_cache: - key: ui-dashboard-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - paths: - - ui/litellm-dashboard/node_modules - - run: - name: Build UI bundle - command: | - cd ui/litellm-dashboard - npm run build - mkdir -p ../../ui-bundle - mv out ../../ui-bundle/out - find ../../ui-bundle/out -name '*.html' ! -name 'index.html' | while read -r f; do - d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" - done - - persist_to_workspace: - root: ~/project - paths: - - ui-bundle/out - - e2e_ui_testing: - docker: - - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 - environment: - POSTGRES_USER: e2euser - POSTGRES_PASSWORD: e2epassword - POSTGRES_DB: litellm_e2e - resource_class: xlarge - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" - CI: "true" - # Boot the proxy with an external logout URL so proxyLogoutUrl.spec.ts can - # assert the redirect. Set at job level so both the proxy boot step and the - # Playwright step (whose skip guard reads this) see the same value. Safe for - # the rest of the suite: nothing else performs a logout. - PROXY_LOGOUT_URL: "https://www.example.com" - steps: - - checkout_over_https - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - skip_if_unrelated_changes: - category: client - - attach_workspace: - at: ~/project - - setup_google_dns - - install_node - - install_uv - - install_rust - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Python dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma - - save_cache: - key: v1-uv-cache-{{ checksum "uv.lock" }} - paths: - - ~/.cache/uv - - restore_cache: - keys: - - ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} - - run: - name: Install Node dependencies and Playwright - # The cimg/python:3.12-browsers image already ships the Chromium system - # libraries Playwright needs (libnss3, libatk-bridge2.0-0, libcups2, etc.). - # `--with-deps` triggers a redundant apt-get update + install that adds - # 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors, - # so we install just the browser binary. - command: | - cd tests/e2e/ui - npm ci - npx playwright install chromium - - save_cache: - key: ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} - paths: - - tests/e2e/ui/node_modules - - ~/.cache/ms-playwright - - run: - name: Use built UI bundle - command: | - rm -rf litellm/proxy/_experimental/out - cp -r ui-bundle/out litellm/proxy/_experimental/out - - wait_for_service: - url: tcp://localhost:5432 - timeout: "30" - - run: - name: Push Prisma schema - command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Seed database - command: | - PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ - -f tests/e2e/ui/fixtures/seed.sql - - run: - name: Start mock LLM server - command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py - background: true - - run: - name: Start mock Presidio server - command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py - background: true - - run: - name: Wait for mock Presidio server - command: | - for i in $(seq 1 30); do - if curl -sf http://127.0.0.1:8091/health >/dev/null 2>&1; then exit 0; fi - sleep 1 - done - echo "Mock Presidio server never answered /health on port 8091" >&2 - exit 1 - - run: - name: Start LiteLLM proxy - environment: - MOCK_LLM_URL: "http://127.0.0.1:8090/v1" - DISABLE_SCHEMA_UPDATE: "true" - SERVER_ROOT_PATH: "" - # PROXY_LOGOUT_URL is inherited from the job-level environment so the - # proxy and proxyLogoutUrl.spec.ts agree on the logout target. - # LITELLM_LICENSE is forwarded from the project env so premium-gated - # UI flows can be exercised. license.spec.ts asserts the resulting - # JWT carries premium_user=true; if it ever stops being passed, that - # test fails loudly rather than silently regressing premium coverage. - command: | - LITELLM_LICENSE="$LITELLM_LICENSE" \ - uv run --no-sync python -m litellm.proxy.proxy_cli \ - --config tests/e2e/ui/fixtures/config.yml \ - --port 4000 - background: true - - run: - name: Wait for proxy to be ready - command: | - for i in $(seq 1 60); do - HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true) - if [ "$HTTP_CODE" = "200" ]; then - echo "Proxy is ready" - exit 0 - fi - sleep 2 - done - echo "Proxy failed to start" - exit 1 - - run: - name: Run Playwright E2E tests - # Forward LITELLM_LICENSE so license.spec.ts can detect that the - # proxy was launched with a license and assert premium_user=true. - command: | - cd tests/e2e/ui - LITELLM_LICENSE="$LITELLM_LICENSE" \ - npx playwright test --config playwright.config.ts --workers=3 --retries=0 - no_output_timeout: 10m - - store_artifacts: - path: tests/e2e/ui/test-results - destination: e2e-test-results - - store_artifacts: - path: tests/e2e/ui/playwright-report - destination: e2e-playwright-report - - e2e_ui_testing_server_root_path: - docker: - - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 - environment: - POSTGRES_USER: e2euser - POSTGRES_PASSWORD: e2epassword - POSTGRES_DB: litellm_e2e - resource_class: large - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" - CI: "true" - # The whole job exercises the proxy mounted under a prefix. SERVER_ROOT_PATH - # is read both by the proxy at boot (to rewrite the built UI bundle in place) - # and by migration.serverRootPath.config.ts, which refuses to run without it. - SERVER_ROOT_PATH: "/litellm" - steps: - - checkout_over_https - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - skip_if_unrelated_changes: - category: client - - attach_workspace: - at: ~/project - - setup_google_dns - - install_node - - install_uv - - install_rust - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Python dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma - - save_cache: - key: v1-uv-cache-{{ checksum "uv.lock" }} - paths: - - ~/.cache/uv - - restore_cache: - keys: - - ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} - - run: - name: Install Node dependencies and Playwright - command: | - cd tests/e2e/ui - npm ci - npx playwright install chromium - - save_cache: - key: ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} - paths: - - tests/e2e/ui/node_modules - - ~/.cache/ms-playwright - - run: - name: Use built UI bundle - command: | - rm -rf litellm/proxy/_experimental/out - cp -r ui-bundle/out litellm/proxy/_experimental/out - - wait_for_service: - url: tcp://localhost:5432 - timeout: "30" - - run: - name: Push Prisma schema - command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Seed database - command: | - PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ - -f tests/e2e/ui/fixtures/seed.sql - - run: - name: Start mock LLM server - command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py - background: true - - run: - name: Start mock Presidio server - command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py - background: true - - run: - name: Wait for mock Presidio server - command: | - for i in $(seq 1 30); do - if curl -sf http://127.0.0.1:8091/health >/dev/null 2>&1; then exit 0; fi - sleep 1 - done - echo "Mock Presidio server never answered /health on port 8091" >&2 - exit 1 - - run: - name: Start LiteLLM proxy under a server root path - environment: - MOCK_LLM_URL: "http://127.0.0.1:8090/v1" - DISABLE_SCHEMA_UPDATE: "true" - # Output flows to this step's own log, so a boot crash is visible here - # rather than swallowed by a downstream readiness probe. - command: | - LITELLM_LICENSE="$LITELLM_LICENSE" \ - uv run --no-sync python -m litellm.proxy.proxy_cli \ - --config tests/e2e/ui/fixtures/config.yml \ - --port 4000 - background: true - - run: - name: Wait for prefixed proxy to be ready - command: | - for i in $(seq 1 60); do - HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer $LITELLM_MASTER_KEY" http://127.0.0.1:4000/litellm/health 2>/dev/null || true) - if [ "$HTTP_CODE" = "200" ]; then - echo "Prefixed proxy is ready" - exit 0 - fi - sleep 2 - done - echo "Prefixed proxy failed to start; see the 'Start LiteLLM proxy under a server root path' step for the boot log" - exit 1 - - run: - name: Run migration smoke under SERVER_ROOT_PATH - command: | - cd tests/e2e/ui - LITELLM_LICENSE="$LITELLM_LICENSE" \ - npx playwright test --config migration.serverRootPath.config.ts - no_output_timeout: 10m - - store_artifacts: - path: tests/e2e/ui/test-results - destination: e2e-server-root-path-test-results - - store_artifacts: - path: tests/e2e/ui/playwright-report - destination: e2e-server-root-path-playwright-report - - build_docker_database_image: - parameters: - migration_qualification: - type: boolean - default: false - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - when: - condition: << parameters.migration_qualification >> - steps: - - checkout_over_https: - revision: << pipeline.parameters.migration_source_sha >> - - unless: - condition: << parameters.migration_qualification >> - steps: - - checkout_over_https - - skip_if_unrelated_changes - - - run: - name: Build Docker image - environment: - MIGRATION_CANDIDATE_IMAGE: << pipeline.parameters.migration_candidate_image >> - command: | - if [ -n "$MIGRATION_CANDIDATE_IMAGE" ]; then - [[ "$MIGRATION_CANDIDATE_IMAGE" =~ ^ghcr.io/berriai/[a-z0-9._/-]+@sha256:[0-9a-f]{64}$ ]] || exit 1 - docker pull "$MIGRATION_CANDIDATE_IMAGE" - docker tag "$MIGRATION_CANDIDATE_IMAGE" litellm-docker-database:ci - else - docker build \ - --label org.opencontainers.image.revision="$(git rev-parse HEAD)" \ - -t litellm-docker-database:ci \ - -f Dockerfile . - fi - python3 .circleci/scripts/run_migration_tests.py record-image - - - run: - name: Save Docker image to workspace root - command: | - docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst - - - persist_to_workspace: - root: . - paths: - - litellm-docker-database.tar.zst - - migration-image.json - - migration_startup_tests: - parameters: - suite: - type: enum - enum: [startup, recovery, legacy, upgrade, shaped] - baseline: - type: boolean - default: false - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - environment: - LITELLM_MIGRATION_TESTS: "1" - LITELLM_MIGRATION_TEST_IMAGE: litellm-docker-database:ci - LITELLM_MIGRATION_BASELINE_IMAGE: << pipeline.parameters.migration_baseline_image >> - MIGRATION_TEST_ADMIN_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres - MIGRATION_TEST_CONTAINER_ADMIN_URL: postgresql://postgres:postgres@host.docker.internal:5432/postgres - MIGRATION_TEST_OUTPUT: /tmp/migration-results - PYTHONPATH: tests/e2e - steps: - - checkout_over_https: - revision: << pipeline.parameters.migration_source_sha >> - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - install_uv - - install_rust - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install test dependencies - command: uv sync --frozen --all-groups --all-extras --python 3.12 - - attach_workspace: - at: ~/project - - run: - name: Load the shared candidate and start PostgreSQL - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker run -d --name migration-postgres \ - -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres \ - -p 5432:5432 \ - postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - - when: - condition: << parameters.baseline >> - steps: - - run: - name: Pull the baseline release the upgrade starts from - environment: - BASELINE_IMAGE: << pipeline.parameters.migration_baseline_image >> - command: | - [[ "$BASELINE_IMAGE" =~ ^ghcr.io/berriai/[a-z0-9._/-]+(@sha256:[0-9a-f]{64}|:v[0-9][0-9a-z.-]*)$ ]] || exit 1 - docker pull "$BASELINE_IMAGE" - - run: - name: Run migration startup regressions - environment: - MIGRATION_TEST_SUITE: << parameters.suite >> - MIGRATION_CANDIDATE_IMAGE: << pipeline.parameters.migration_candidate_image >> - command: | - mkdir -p /tmp/migration-results - uv run --no-sync python .circleci/scripts/run_migration_tests.py - no_output_timeout: 15m - - store_test_results: - path: /tmp/migration-results/junit - - run: - name: Package migration diagnostics - when: always - command: | - mkdir -p /tmp/migration-artifacts - if [ -d /tmp/migration-results ]; then - tar -czf /tmp/migration-artifacts/diagnostics.tar.gz -C /tmp/migration-results . - if [ -f /tmp/migration-results/verdict.json ]; then - cp /tmp/migration-results/verdict.json /tmp/migration-artifacts/verdict.json - fi - fi - - store_artifacts: - path: /tmp/migration-artifacts - destination: migration-results - - run: - name: Remove migration test containers - when: always - command: | - docker ps -aq --filter label=litellm-migration-test=true | xargs -r docker rm -f - docker rm -f migration-postgres || true - - test_bad_database_url: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: medium - working_directory: ~/project - steps: - - checkout_over_https - - skip_if_unrelated_changes - - attach_workspace: - at: ~/project - - setup_google_dns - - start_postgres - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker tag litellm-docker-database:ci myapp:latest - - run: - name: Run Docker container with bad DATABASE_URL - command: | - set +e - docker run --name my-app \ - -p 4000:4000 \ - -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \ - -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ - -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \ - myapp:latest \ - --port 4000 > docker_output.log 2>&1 - echo "$?" > docker_exit_code - set -e - - run: - name: Display Docker logs - command: cat docker_output.log - - run: - name: Proxy must refuse to serve on an unreachable database - command: | - fail() { echo "FAILED: $1"; cat docker_output.log; exit 1; } - exit_code="$(cat docker_exit_code)" - [ "$exit_code" -ne 0 ] || fail "proxy exited 0 with an unreachable database" - grep -q "P1001" docker_output.log || fail "log does not name the unreachable database server" - ! grep -q "Application startup complete" docker_output.log || fail "proxy reached serving state" - ! docker exec my-app true 2>/dev/null || fail "container is still running" - echo "Proxy refused to serve (exit $exit_code) and never reached startup. Test passed." - - integration_contracts: - parameters: - suite: - type: string - mode: - type: enum - enum: [standard, replica] - default: standard - parallelism: - type: integer - default: 1 - resource_class: - type: enum - enum: [large, xlarge] - default: large - machine: - image: ubuntu-2204:2024.04.1 - resource_class: << parameters.resource_class >> - working_directory: ~/project - parallelism: << parameters.parallelism >> - steps: - - setup_litellm_test_deps - - when: - condition: - equal: [browser, << parameters.suite >>] - steps: - - install_node - - restore_cache: - keys: - - integration-ui-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }} - - run: - name: Install locked browser dependencies - command: | - cd ui/litellm-dashboard - npm ci - cd ../../tests/e2e/ui - npm ci - sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l \ - timeout --signal=TERM --kill-after=20s 6m node node_modules/@playwright/test/cli.js install-deps chromium - timeout --signal=TERM --kill-after=20s 3m node node_modules/@playwright/test/cli.js install chromium - - save_cache: - key: integration-ui-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }} - paths: - - ~/.npm - - ~/.cache/ms-playwright - - run: - name: Build the candidate dashboard - command: cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build - - when: - condition: - equal: [mcp, << parameters.suite >>] - steps: - - run: - name: Install MCP SDK1 peer - command: | - uv venv --python 3.12 .venv-mcp-peer - uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1' - echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV" - - when: - condition: - equal: [sdk, << parameters.suite >>] - steps: - - run: - name: Install redis-server 7.2.16 - command: | - mkdir -p /tmp/redis-bin - cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)" - docker cp "${cid}:/usr/local/bin/redis-server" /tmp/redis-bin/redis-server - docker rm "$cid" - /tmp/redis-bin/redis-server --version | grep -q 'v=7.2.16' - echo 'export PATH="/tmp/redis-bin:$PATH"' >> "$BASH_ENV" - - start_postgres: - image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - server_args: "-c max_connections=300 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000" - - start_redis - - run: - name: Run owned integration contracts - command: bash .circleci/scripts/run_integration.sh << parameters.suite >> << parameters.mode >> - no_output_timeout: 15m - - run: - name: Stop owned database and Redis - when: always - command: | - mkdir -p test-results/services-<< parameters.suite >>-<< parameters.mode >> - docker logs postgres-db > test-results/services-<< parameters.suite >>-<< parameters.mode >>/postgres.log 2>&1 || true - docker logs redis-cache > test-results/services-<< parameters.suite >>-<< parameters.mode >>/redis.log 2>&1 || true - docker rm -f postgres-db redis-cache - test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)" - - when: - condition: - not: - equal: [1, << parameters.parallelism >>] - steps: - - run: - name: Prepare this shard's execution record - command: | - results="test-results/integration-<< parameters.suite >>" - if [ "<< parameters.mode >>" = replica ]; then - results="$results-replica" - fi - mkdir -p "/tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >>/$CIRCLE_NODE_INDEX" - cp "$results/execution.json" "/tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >>/$CIRCLE_NODE_INDEX/execution.json" - if [ "<< parameters.suite >>" = mcp ]; then - mkdir -p "/tmp/integration-shards/mcp-standard/$CIRCLE_NODE_INDEX/coverage" - cp test-results/integration-mcp/coverage/data "/tmp/integration-shards/mcp-standard/$CIRCLE_NODE_INDEX/coverage/data.$CIRCLE_NODE_INDEX" - fi - - persist_to_workspace: - root: /tmp/integration-shards - paths: - - << parameters.suite >>-<< parameters.mode >> - - store_test_results: - path: test-results - - store_artifacts: - path: test-results - - integration_shard_coverage: - parameters: - suite: - type: enum - enum: [management, accounting, database, extensions, providers, mcp] - mode: - type: enum - enum: [standard, replica] - default: standard - shards: - type: integer - docker: - - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - resource_class: small - working_directory: ~/project - environment: - UV_PYTHON_DOWNLOADS: never - steps: - - checkout_over_https - - install_uv - - run: - name: Install the locked shard verifier and coverage dependencies - command: | - uv export --frozen --all-groups --all-extras --no-emit-project --no-emit-workspace --no-header --no-annotate \ - | awk '/^[^ ]/ { keep = $0 ~ /^(annotated-types|coverage|pydantic|pydantic-core|typing-extensions|typing-inspection)==/ } keep' \ - > /tmp/fan-in-requirements.txt - uv venv .venv --python 3.12 - uv pip install --no-config --python .venv --require-hashes -r /tmp/fan-in-requirements.txt - - attach_workspace: - at: /tmp/integration-shards - - run: - name: Verify every << parameters.suite >> case ran exactly once - command: | - .venv/bin/python -m tests.integration._support.sharding /tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >> \ - --shards << parameters.shards >> - - when: - condition: - equal: [mcp, << parameters.suite >>] - steps: - - run: - name: Combine coverage from every MCP shard - command: | - mkdir -p test-results/integration-mcp/coverage - export COVERAGE_FILE="$PWD/test-results/integration-mcp/coverage/data" - .venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml /tmp/integration-shards/mcp-standard/*/coverage - .venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > test-results/integration-mcp/coverage/coverage.txt - .venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d test-results/integration-mcp/coverage/html - - store_artifacts: - path: test-results/integration-mcp/coverage - - routing_parity: - parameters: - suite: - type: string - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - setup_litellm_test_deps - - run: - name: Check out base product code - environment: - ROUTING_PARITY_BASE: << pipeline.parameters.routing_parity_base >> - command: | - [[ "$ROUTING_PARITY_BASE" =~ ^[0-9a-f]{40}$ ]] || exit 1 - git fetch --depth 1 --no-filter origin "$ROUTING_PARITY_BASE" - git rm -r -f --quiet litellm enterprise litellm-proxy-extras - git checkout "$ROUTING_PARITY_BASE" -- litellm enterprise litellm-proxy-extras - git reset --quiet - test -f litellm/rust_bridge/_native.abi3.so - - start_postgres: - image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000" - - start_redis - - run: - name: Run base side - command: bash .circleci/scripts/run_integration.sh << parameters.suite >> parity base - no_output_timeout: 15m - - run: - name: Stop base database and Redis - when: always - command: | - mkdir -p test-results/services-<< parameters.suite >>-parity-base - docker logs postgres-db > test-results/services-<< parameters.suite >>-parity-base/postgres.log 2>&1 || true - docker logs redis-cache > test-results/services-<< parameters.suite >>-parity-base/redis.log 2>&1 || true - docker rm -f postgres-db redis-cache - test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)" - - run: - name: Check out head product code - command: | - git rm -r -f --quiet litellm enterprise litellm-proxy-extras - git checkout "$CIRCLE_SHA1" -- litellm enterprise litellm-proxy-extras - git reset --quiet - test -f litellm/rust_bridge/_native.abi3.so - - start_postgres: - image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000" - - start_redis - - run: - name: Run head side - command: bash .circleci/scripts/run_integration.sh << parameters.suite >> parity head - no_output_timeout: 15m - - run: - name: Stop head database and Redis - when: always - command: | - mkdir -p test-results/services-<< parameters.suite >>-parity-head - docker logs postgres-db > test-results/services-<< parameters.suite >>-parity-head/postgres.log 2>&1 || true - docker logs redis-cache > test-results/services-<< parameters.suite >>-parity-head/redis.log 2>&1 || true - docker rm -f postgres-db redis-cache - test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)" - - run: - name: Compare routing parity - command: PYTHONPATH="$PWD/tests" .venv/bin/python -m integration._support.routing check test-results/parity-<< parameters.suite >>/base test-results/parity-<< parameters.suite >>/head - - store_test_results: - path: test-results - - store_artifacts: - path: test-results - - postgres_suite: - parameters: - test_path: - type: string - seed: - type: boolean - coverage_flag: - type: string - default: "" - timeout_minutes: - type: integer - workers: - type: integer - default: 1 - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - environment: - DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test - steps: - - checkout_over_https - - skip_if_unrelated_changes - - setup_litellm_test_deps: - checkout: false - - start_postgres: - db_name: litellm_test - image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - - run: - name: Generate Prisma client - command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - - when: - condition: << parameters.seed >> - steps: - - run: - name: Seed database schema - command: uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Run << parameters.test_path >> - command: | - mkdir -p test-results - coverage_args=() - worker_args=() - if [ "<< parameters.workers >>" -gt 1 ]; then - worker_args=(-n << parameters.workers >> --dist=worksteal) - fi - if [ -n "<< parameters.coverage_flag >>" ]; then - coverage_args=(--cov=./litellm --cov-report=xml:coverage.xml) - fi - timeout --signal=TERM << parameters.timeout_minutes >>m uv run --no-sync pytest \ - << parameters.test_path >> -vv --tb=short --durations=10 -o junit_family=xunit1 \ - --junitxml=test-results/junit.xml "${coverage_args[@]}" "${worker_args[@]}" - - when: - condition: << parameters.coverage_flag >> - steps: - - install_codecov_cli - - run: - name: Upload coverage - when: always - command: | - [ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; } - codecov upload-process --disable-search --fail-on-error -f coverage.xml \ - -F << parameters.coverage_flag >> -C "$CIRCLE_SHA1" \ - -n "<< parameters.coverage_flag >>-${CIRCLE_BUILD_NUM}" --git-service github - - store_test_results: - path: test-results - - store_artifacts: - path: test-results - -workflows: - migration_startup_scheduled: - triggers: - - schedule: - cron: "17 0,6,12,18 * * *" - filters: - branches: - only: main - jobs: - - build_docker_database_image: - migration_qualification: true - - migration_startup_tests: - name: migration-startup - suite: startup - requires: [build_docker_database_image] - - migration_startup_tests: - name: migration-recovery - suite: recovery - requires: [build_docker_database_image] - - migration_startup_tests: - name: migration-legacy-and-pooling - suite: legacy - requires: [build_docker_database_image] - - migration_startup_tests: - name: migration-upgrade - suite: upgrade - baseline: true - requires: [build_docker_database_image] - - migration_startup_tests: - name: migration-upgrade-shaped - suite: shaped - baseline: true - requires: [build_docker_database_image] - integration: - jobs: - - integration_contracts: - name: integration-<< matrix.suite >> - filters: &integration_default_run not pipeline.parameters.run_migration_tests and pipeline.parameters.routing_parity_base == "" - matrix: - parameters: - suite: [sdk, cost, browser] - parallelism: 1 - - integration_contracts: - name: integration-security - suite: security - parallelism: 1 - resource_class: xlarge - filters: *integration_default_run - - integration_contracts: - name: integration-management - suite: management - parallelism: 4 - resource_class: xlarge - filters: *integration_default_run - - integration_shard_coverage: - name: integration-management-coverage - suite: management - shards: 4 - requires: [integration-management] - filters: *integration_default_run - - integration_contracts: - name: integration-accounting - suite: accounting - parallelism: 4 - resource_class: xlarge - filters: *integration_default_run - - integration_shard_coverage: - name: integration-accounting-coverage - suite: accounting - shards: 4 - requires: [integration-accounting] - filters: *integration_default_run - - integration_contracts: - name: integration-providers - suite: providers - parallelism: 4 - resource_class: xlarge - filters: *integration_default_run - - integration_shard_coverage: - name: integration-providers-coverage - suite: providers - shards: 4 - requires: - - integration-providers - filters: *integration_default_run - - integration_contracts: - name: integration-database - suite: database - parallelism: 2 - filters: *integration_default_run - - integration_shard_coverage: - name: integration-database-coverage - suite: database - shards: 2 - requires: [integration-database] - filters: *integration_default_run - - integration_contracts: - name: integration-mcp - suite: mcp - parallelism: 2 - filters: *integration_default_run - - integration_shard_coverage: - name: integration-mcp-coverage - suite: mcp - shards: 2 - requires: - - integration-mcp - filters: *integration_default_run - - integration_contracts: - name: integration-extensions - suite: extensions - parallelism: 8 - resource_class: xlarge - filters: *integration_default_run - - integration_shard_coverage: - name: integration-extensions-coverage - suite: extensions - shards: 8 - requires: - - integration-extensions - filters: *integration_default_run - - integration_contracts: - name: integration-management-replica - suite: management - mode: replica - parallelism: 4 - resource_class: xlarge - filters: *integration_default_run - - integration_shard_coverage: - name: integration-management-replica-coverage - suite: management - mode: replica - shards: 4 - requires: [integration-management-replica] - filters: *integration_default_run - - integration_contracts: - name: integration-database-replica - suite: database - mode: replica - parallelism: 2 - filters: *integration_default_run - - integration_shard_coverage: - name: integration-database-replica-coverage - suite: database - mode: replica - shards: 2 - requires: [integration-database-replica] - filters: *integration_default_run - - postgres_suite: - name: roi-database - test_path: tests/integration/database/test_roi_observed.py - seed: false - coverage_flag: roi-postgres - timeout_minutes: 10 - filters: *integration_default_run - - windows_release_wheel: - filters: *integration_default_run - - base_sdk_install: - filters: *integration_default_run - - build_docker_database_image: - filters: *integration_default_run - - proxy_logging_guardrails_model_info_tests: - requires: - - build_docker_database_image - filters: *integration_default_run - - db_migration_disable_update_check: - requires: - - build_docker_database_image - filters: *integration_default_run - - test_bad_database_url: - requires: - - build_docker_database_image - filters: *integration_default_run - - helm_chart_testing: - requires: - - build_docker_database_image - filters: *integration_default_run - - e2e_ui_build: - filters: *integration_default_run - - e2e_ui_testing: - requires: [e2e_ui_build] - filters: *integration_default_run - - e2e_ui_testing_server_root_path: - requires: [e2e_ui_build] - filters: *integration_default_run - - build_docker_database_image: - name: build_docker_database_image-migration - migration_qualification: true - filters: pipeline.parameters.run_migration_tests - - migration_startup_tests: - name: migration-startup - suite: startup - requires: [build_docker_database_image-migration] - filters: pipeline.parameters.run_migration_tests - - migration_startup_tests: - name: migration-recovery - suite: recovery - requires: [build_docker_database_image-migration] - filters: pipeline.parameters.run_migration_tests - - migration_startup_tests: - name: migration-legacy-and-pooling - suite: legacy - requires: [build_docker_database_image-migration] - filters: pipeline.parameters.run_migration_tests - - migration_startup_tests: - name: migration-upgrade - suite: upgrade - baseline: true - requires: [build_docker_database_image-migration] - filters: pipeline.parameters.run_migration_tests - - migration_startup_tests: - name: migration-upgrade-shaped - suite: shaped - baseline: true - requires: [build_docker_database_image-migration] - filters: pipeline.parameters.run_migration_tests - - routing_parity: - name: routing-parity-<< matrix.suite >> - matrix: - parameters: - suite: [management, accounting, database, providers, extensions, cost, mcp] - filters: pipeline.parameters.routing_parity_base != "" diff --git a/.circleci/scripts/run_migration_tests.py b/.circleci/scripts/run_migration_tests.py deleted file mode 100644 index 50d01f1d50b..00000000000 --- a/.circleci/scripts/run_migration_tests.py +++ /dev/null @@ -1,116 +0,0 @@ -from __future__ import annotations - -import json -import os -import re -import subprocess -import sys -from pathlib import Path -from typing import Final -from xml.etree import ElementTree - -SUITES: Final = { - "startup": (("test_startup.py",), 13), - "recovery": (("test_recovery.py",), 15), - "legacy": (("test_legacy.py", "test_pooling.py"), 11), - "upgrade": (("test_upgrade.py", "test_rolling_upgrade.py"), 5), - "shaped": (("test_shaped_database.py",), 1), -} - - -def successful_junit(path: Path, expected: int, exit_code: int) -> bool: - if exit_code != 0 or not path.is_file(): - return False - try: - root: Final = ElementTree.parse(path).getroot() - except ElementTree.ParseError: - return False - cases: Final = tuple(root.iter("testcase")) - identities: Final = frozenset((case.get("classname"), case.get("name")) for case in cases) - return len(cases) == len(identities) == expected and all( - not any(case.find(tag) is not None for tag in ("failure", "error", "skipped")) for case in cases - ) - - -def output(*command: str) -> str: - return subprocess.check_output(command, text=True, timeout=90).strip() - - -def record_image() -> None: - source: Final = output("git", "rev-parse", "HEAD") - image: Final = output("docker", "image", "inspect", "litellm-docker-database:ci", "--format", "{{.Id}}") - revision: Final = output( - "docker", - "image", - "inspect", - "litellm-docker-database:ci", - "--format", - '{{index .Config.Labels "org.opencontainers.image.revision"}}', - ) - assert re.fullmatch(r"[0-9a-f]{40}", source), "Invalid source revision" - assert revision == source, "Candidate image revision differs from the tested source" - Path("migration-image.json").write_text( - json.dumps( - { - "source_sha": source, - "image_id": image, - "candidate_image": os.environ.get("MIGRATION_CANDIDATE_IMAGE", ""), - } - ) - ) - - -def main() -> int: - suite: Final = os.environ["MIGRATION_TEST_SUITE"] - files, expected = SUITES[suite] - metadata: Final = json.loads(Path("migration-image.json").read_text()) - assert metadata["source_sha"] == output("git", "rev-parse", "HEAD"), "Image and test source revisions differ" - assert metadata["image_id"] == output( - "docker", "image", "inspect", os.environ["LITELLM_MIGRATION_TEST_IMAGE"], "--format", "{{.Id}}" - ), "Loaded image differs from the build output" - assert metadata["candidate_image"] == os.environ.get("MIGRATION_CANDIDATE_IMAGE", ""), "Wrong release candidate" - destination: Final = Path(os.environ["MIGRATION_TEST_OUTPUT"]) - junit: Final = destination / "junit" / "results.xml" - junit.parent.mkdir(parents=True, exist_ok=True) - result: Final = subprocess.run( - ( - sys.executable, - "-m", - "pytest", - *(f"tests/e2e/migrations/{name}" for name in files), - "-vv", - "--tb=short", - "--durations=10", - f"--junitxml={junit}", - "-o", - "addopts=", - "--reruns=0", - ), - check=False, - timeout=1200, - ) - passed: Final = successful_junit(junit, expected, result.returncode) - (destination / "verdict.json").write_text( - json.dumps( - { - **metadata, - "suite": suite, - "baseline_image": os.environ.get("LITELLM_MIGRATION_BASELINE_IMAGE", ""), - "expected_cases": expected, - "passed": passed, - "pytest_exit_code": result.returncode, - "test_revision": metadata["source_sha"], - "workflow_id": os.environ.get("CIRCLE_WORKFLOW_ID", ""), - "job_number": os.environ.get("CIRCLE_BUILD_NUM", ""), - }, - indent=2, - ) - ) - return 0 if passed else 1 - - -if __name__ == "__main__": - if sys.argv[1:] == ["record-image"]: - record_image() - else: - raise SystemExit(main()) diff --git a/.dockerignore b/.dockerignore index f0fdcf9d479..d8eb72a057f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,5 @@ docs litellm/proxy/discovery_endpoints/skill_scanner_adapter -.circleci .github tests .git diff --git a/.github/AGENTS.md b/.github/AGENTS.md new file mode 100644 index 00000000000..14bd461a8a0 --- /dev/null +++ b/.github/AGENTS.md @@ -0,0 +1,5 @@ +# CI + +Integration groups and replica runs remain in `workflows/test-integration.yml`. Do not duplicate them in another workflow. + +CI helpers used by existing workflows live in `scripts/ci/`. Jobs that need merge bases check out full history. Removing a CI job does not count its tests as covered. List tests that no remaining job invokes explicitly in `ci-coverage-allowlist.yml`. diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml index b0f9b72f0ee..59c6ab28c30 100644 --- a/.github/actions/detect-changes/action.yml +++ b/.github/actions/detect-changes/action.yml @@ -1,6 +1,6 @@ name: "Detect relevant changes" description: >- - Classify the pull request's changed files with .circleci/scripts/classify_changes.sh + Classify the pull request's changed files with .github/scripts/ci/classify_changes.sh and expose decision=run|skip for one category. backend means anything outside ui/, docs/ and markdown; ui means the dashboard sources alone. decision=skip lets callers short-circuit expensive steps while the job still completes successfully and satisfies diff --git a/.github/ci-coverage-allowlist.yml b/.github/ci-coverage-allowlist.yml index 76f8e3ef7e7..3a714187325 100644 --- a/.github/ci-coverage-allowlist.yml +++ b/.github/ci-coverage-allowlist.yml @@ -4,6 +4,16 @@ description: >- by a job nor listed here, so every entry below is a decision on the record. test_paths: + - reason: ran only in CircleCI, removed + paths: + - tests/e2e/migrations/test_legacy.py + - tests/e2e/migrations/test_pooling.py + - tests/e2e/migrations/test_recovery.py + - tests/e2e/migrations/test_rolling_upgrade.py + - tests/e2e/migrations/test_shaped_database.py + - tests/e2e/migrations/test_startup.py + - tests/e2e/migrations/test_upgrade.py + - tests/integration/database/test_roi_observed.py - reason: >- litellm.agent() end-to-end suite. It drives the real claude, codex, opencode and pi CLIs and deepagents against a live LiteLLM AI Gateway, so it needs those binaries on PATH plus @@ -19,7 +29,7 @@ test_paths: - tests/rust-python-harness - reason: >- Owned by the open pull request that moves the Rust/Python parity checks (#45135). It ran in - the CircleCI build_and_test job that globbed tests/test_*.py, which this migration deletes, + the retired build_and_test job that globbed tests/test_*.py, and that pull request gives it a new home paths: - tests/test_rust_python_harness.py diff --git a/.github/scripts/assert_ci_coverage.py b/.github/scripts/assert_ci_coverage.py index b442948c3ca..b443fd9e0d9 100644 --- a/.github/scripts/assert_ci_coverage.py +++ b/.github/scripts/assert_ci_coverage.py @@ -10,6 +10,7 @@ import warnings from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass from functools import lru_cache +from itertools import chain from types import MappingProxyType from typing import Final @@ -17,7 +18,6 @@ import yaml REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] WORKFLOW_DIR = REPO_ROOT / ".github" / "workflows" -CIRCLECI_CONFIG = REPO_ROOT / ".circleci" / "config.yml" ALLOWLIST_FILE = REPO_ROOT / ".github" / "ci-coverage-allowlist.yml" TESTS_ROOT = REPO_ROOT / "tests" @@ -25,14 +25,18 @@ ALLOWLIST_KEYS = frozenset({"description", "test_paths", "dockerfiles"}) PATH_FILTER_KEYS = frozenset({"paths", "paths-ignore"}) TEST_PATH_KEYS = frozenset({"test-path", "test-paths", "test_path"}) DOCKERFILE_INPUT_KEYS = frozenset({"file", "dockerfile"}) -TEST_RUNNER_RE = re.compile(r"\bpytest\b|\bcircleci tests\b|\bhelm unittest\b|\bplaywright test\b|\bpython[0-9.]*\s") +TEST_RUNNER_RE = re.compile(r"\bpytest\b|\bhelm unittest\b|\bplaywright test\b|\bpython[0-9.]*\s") IMAGE_BUILD_RE = re.compile(r"\bdocker\s+(?:buildx\s+)?build\b") TEST_TOKEN_RE = re.compile(r"tests/[A-Za-z0-9_./*?-]+") IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)") DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*") COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE) GLOB_CHARS = frozenset("*?") -DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType({"image-scan.yml": "tests/integration/database/image/"}) +DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType( + { + "image-scan.yml": "tests/integration/database/image/", + } +) # Trees whose jobs are sharded with no catch-all bucket, so every child that holds # tests has to be named by some shard or it runs nowhere. A child listed here is @@ -115,9 +119,7 @@ def _scalars(node: object, key: str) -> tuple[Scalar, ...]: def _config_files() -> tuple[pathlib.Path, ...]: - workflows = tuple(sorted(path for path in WORKFLOW_DIR.iterdir() if path.suffix in (".yml", ".yaml"))) - circleci = (CIRCLECI_CONFIG,) if CIRCLECI_CONFIG.is_file() else () - return workflows + circleci + return tuple(sorted(path for path in WORKFLOW_DIR.iterdir() if path.suffix in (".yml", ".yaml"))) def _all_scalars() -> tuple[Scalar, ...]: @@ -243,7 +245,6 @@ def _describe(paths: tuple[str, ...]) -> str: return f"{len(paths)} test file(s) invoked by no job: {names}{suffix}" -GLOB_CALL_RE = re.compile(r'circleci tests glob "([^"]+)"') KEYWORD_RE = re.compile(r"-k\s+\\?[\"']([^\"'\\]+)") @@ -316,16 +317,14 @@ def _keyword_terms( def _slices() -> tuple[Slice, ...]: - if not CIRCLECI_CONFIG.exists(): - return () test_files: Final = frozenset(_test_files()) - jobs: Final = yaml.safe_load(CIRCLECI_CONFIG.read_text()).get("jobs", {}) return tuple( - Slice(job=job, globs=globs, named=named, required=required, excluded=excluded, understood=understood) - for job, body in jobs.items() + Slice(job=f"{path.name}:{job}", globs=globs, named=named, required=required, excluded=excluded, understood=understood) + for path in _config_files() + for job, body in yaml.safe_load(path.read_text()).get("jobs", {}).items() for text in ("\n".join(_strings(body)),) if "pytest" in text - for globs in (tuple(GLOB_CALL_RE.findall(text)),) + for globs in (tuple(token for token in TEST_TOKEN_RE.findall(text) if GLOB_CHARS & set(token)),) for named in (frozenset(TEST_TOKEN_RE.findall(text)) & test_files,) for required, excluded, understood in ( _keyword_terms(tuple(KEYWORD_RE.findall(text)), attributable=len(globs) < 2), @@ -352,16 +351,11 @@ def _matchable_names(relative_path: str) -> frozenset[str]: def _workflow_named_tokens() -> frozenset[str]: - """Test tokens a GitHub Actions job names directly. - - A CircleCI `-k` that deselects a file no longer means the file runs nowhere once a - workflow names it, so the slice check has to credit those the same way the census does. - """ return _invoked_test_tokens( scalar for path in _config_files() - if path != CIRCLECI_CONFIG for scalar in _scalars(yaml.safe_load(path.read_text(encoding="utf-8")), path.name) + if not KEYWORD_RE.search(scalar.value) ) @@ -566,13 +560,28 @@ def _owns(workflow: str, relative_path: str) -> bool: return owned_root is not None and relative_path.startswith(owned_root) +def _expression_suites(value: str) -> tuple[str, ...]: + suites: Final = chain.from_iterable(map(json.loads, re.findall(r"'(\[[^']*\])'", value))) + return tuple(suite for suite in suites if isinstance(suite, str) and "{" not in suite) + + +def _matrix_suites(value: object) -> tuple[str, ...]: + if isinstance(value, list): + return tuple(suite for suite in value if isinstance(suite, str)) + if isinstance(value, str): + return _expression_suites(value) + return () + + def _tokens_outside_owner(relative_path: str, workflow_tokens: Mapping[str, frozenset[str]]) -> frozenset[str]: return frozenset[str]().union( *(tokens for workflow, tokens in workflow_tokens.items() if not _owns(workflow, relative_path)) ) -def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]: +def _integration_ownership( + repo_root: pathlib.Path = REPO_ROOT, *, allowlist: Allowlist | None = None +) -> tuple[frozenset[str], tuple[Finding, ...]]: runner: Final = repo_root / "tests/integration/run.py" if not runner.exists(): return frozenset(), () @@ -589,26 +598,19 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens browser_manifest: Final = repo_root / "tests/e2e/ui/tests/integrationCritical/expected.json" browser_nodes: Final = json.loads(browser_manifest.read_text()) if browser_manifest.exists() else () browser_paths: Final = frozenset(node.split("::", 1)[0] for node in browser_nodes) - circle_path: Final = repo_root / ".circleci/config.yml" - circle: Final = yaml.safe_load(circle_path.read_text()) if circle_path.exists() else {} - circle_test_path_tokens: Final = _invoked_test_tokens( - scalar for scalar in _scalars(circle, "config.yml") if scalar.key == "test_path" - ) - steps: Final = circle.get("jobs", {}).get("integration_contracts", {}).get("steps", ()) + integration_path: Final = repo_root / ".github/workflows/test-integration.yml" + integration: Final = yaml.safe_load(integration_path.read_text()) if integration_path.exists() else {} + integration_jobs: Final = integration.get("jobs", {}) invoked: Final = any( - ".circleci/scripts/run_integration.sh" in scalar.value - for scalar in _scalars(steps, "integration_contracts") - if scalar.key == "command" + ".github/scripts/ci/run_integration.sh" in scalar.value + for scalar in _scalars(integration_jobs, "integration") + if scalar.key == "run" ) scheduled: Final = frozenset( suite - for job in circle.get("workflows", {}).get("integration", {}).get("jobs", ()) - if isinstance(job, dict) and "integration_contracts" in job - for suite in job["integration_contracts"] - .get("matrix", {}) - .get("parameters", {}) - .get("suite", (job["integration_contracts"].get("suite"),)) - if isinstance(suite, str) + for body in integration_jobs.values() + if any(".github/scripts/ci/run_integration.sh" in scalar.value for scalar in _scalars(body, "integration")) + for suite in _matrix_suites(body.get("strategy", {}).get("matrix", {}).get("suite", ())) ) required: Final = (frozenset({"browser"}) if browser_paths else frozenset()) | frozenset( group @@ -630,19 +632,27 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens } ) gha_tokens: Final = frozenset[str]().union(*workflow_tokens.values()) - circle_owned: Final = ( + invoked_github_files: Final = frozenset( + path for path in github_files if any(_token_covers(token, path) for token in gha_tokens) + ) + grouped: Final = ( frozenset(str(path.relative_to(repo_root)) for path in integration_root.rglob("test_*.py")) - github_files ) findings: Final = ( tuple( Finding(path, "integration file outside GITHUB_FILES is selected by GitHub Actions") - for path in sorted(circle_owned) - if any(_token_covers(token, path) for token in gha_tokens) + for path in sorted(grouped) + if any( + _token_covers(token, path) + for workflow, tokens in workflow_tokens.items() + if workflow != "test-integration.yml" + for token in tokens + ) ) + tuple( Finding(path, "GitHub-owned integration contract has no invoking job") for path in sorted(github_files) - if not any(_token_covers(token, path) for token in gha_tokens | circle_test_path_tokens) + if path not in invoked_github_files and not (allowlist is not None and allowlist.covers_test(path)) ) + tuple( Finding(path, "GitHub-owned integration file is missing") @@ -658,6 +668,7 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens browser_commands: Final = tuple( scalar.value for path in (repo_root / ".github/workflows").glob("*.y*ml") + if path.name != "test-integration.yml" for scalar in _scalars(yaml.safe_load(path.read_text()), path.name) if scalar.key in {"run", "command"} ) @@ -690,14 +701,14 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens else () ) group_findings: Final = tuple( - Finding(group, "canonical integration group is not scheduled by CircleCI") + Finding(group, "canonical integration group is not scheduled by GitHub Actions") for group in sorted(required - scheduled) ) + tuple(Finding(path, "canonical node must have exactly one integration group") for path in sorted(ungrouped)) if not paths or not invoked or not scheduled: return frozenset(), findings + ( - Finding(str(runner.relative_to(repo_root)), "dedicated CircleCI runner is missing"), + Finding(str(runner.relative_to(repo_root)), "dedicated GitHub Actions runner is missing"), ) - return paths | browser_paths | github_files, findings + group_findings + browser_findings + exclusion_findings + return paths | browser_paths | invoked_github_files, findings + group_findings + browser_findings + exclusion_findings def main() -> int: @@ -709,7 +720,7 @@ def main() -> int: allowlist = _load_allowlist() scalars = _all_scalars() - integration_paths, ownership_findings = _integration_ownership() + integration_paths, ownership_findings = _integration_ownership(allowlist=allowlist) test_findings = _uncovered_tests( allowlist, _invoked_selections(scalars) + (Selection(included=integration_paths, ignored=frozenset()),), @@ -731,8 +742,8 @@ def main() -> int: ) if ownership_findings: browser_remedy = ( - " CircleCI's browser group owns the integrationCritical Playwright contracts, so no GitHub workflow may " - "select them and the default Playwright config must exclude them." + " The browser group in test-integration.yml owns the integrationCritical Playwright contracts. " + "Other workflows and the default Playwright config must exclude them." if any("browser" in finding.detail or "Playwright" in finding.detail for finding in ownership_findings) else "" ) diff --git a/.circleci/scripts/classify_changes.sh b/.github/scripts/ci/classify_changes.sh similarity index 86% rename from .circleci/scripts/classify_changes.sh rename to .github/scripts/ci/classify_changes.sh index a33695d0dbe..82851fa0a82 100755 --- a/.circleci/scripts/classify_changes.sh +++ b/.github/scripts/ci/classify_changes.sh @@ -14,22 +14,22 @@ outside_cost_map_set=false while IFS= read -r file || [ -n "$file" ]; do [ -n "$file" ] || continue case "$file" in - litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .github/workflows/test-unit.yml | .github/actions/detect-changes/* | .github/scripts/detect_changes.sh | .circleci/scripts/classify_changes.sh | tests/integration/run.py | pyproject.toml | uv.lock) + litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .github/workflows/test-unit.yml | .github/actions/detect-changes/* | .github/scripts/detect_changes.sh | .github/scripts/ci/classify_changes.sh | tests/integration/run.py | pyproject.toml | uv.lock) has_redis_compat=true ;; esac case "$file" in *.md | *.mdx) : ;; - pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .circleci/scripts/classify_changes.sh | tests/unit/test_circleci_path_filter.py | tests/unit/test_detect_changes.py) + pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .github/scripts/ci/classify_changes.sh | tests/unit/test_ci_path_filter.py | tests/unit/test_detect_changes.py) has_mcp_dependencies=true ;; esac case "$file" in - litellm-rust/* | litellm/rust_bridge/* | rust-toolchain.toml | pyproject.toml | uv.lock | tests/windows_tests/* | .circleci/*) + litellm-rust/* | litellm/rust_bridge/* | rust-toolchain.toml | pyproject.toml | uv.lock | tests/windows_tests/* | .github/scripts/ci/* | .github/actions/*) has_windows_release=true ;; esac case "$file" in ui/* | tests/e2e/ui/*) has_client=true ;; docs/* | *.md | *.mdx) : ;; - .github/* | .circleci/*) has_ci=true; has_backend=true ;; + .github/*) has_ci=true; has_backend=true ;; *) has_backend=true ;; esac case "$file" in diff --git a/.circleci/scripts/path_filter.sh b/.github/scripts/ci/path_filter.sh similarity index 78% rename from .circleci/scripts/path_filter.sh rename to .github/scripts/ci/path_filter.sh index c3cf6e566ff..e8ac6e5b918 100755 --- a/.circleci/scripts/path_filter.sh +++ b/.github/scripts/ci/path_filter.sh @@ -6,10 +6,12 @@ here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" run_full() { echo "path-filter[$category]: running job ($1)" + echo "decision=run" + [ -z "${GITHUB_OUTPUT:-}" ] || echo "decision=run" >> "$GITHUB_OUTPUT" exit 0 } -[ -n "${CIRCLE_PULL_REQUEST:-}" ] || run_full "not a pull request" +[ "${GITHUB_EVENT_NAME:-}" = pull_request ] || run_full "not a pull request" candidate_bases="${PATH_FILTER_BASE_BRANCH:-main}" merge_base="" @@ -36,5 +38,6 @@ if [ "$decision" = run ]; then run_full "$category-relevant changes detected" fi -echo "path-filter[$category]: only unrelated changes detected; halting job as successful" -circleci-agent step halt +echo "path-filter[$category]: only unrelated changes detected" +echo "decision=skip" +[ -z "${GITHUB_OUTPUT:-}" ] || echo "decision=skip" >> "$GITHUB_OUTPUT" diff --git a/.circleci/scripts/prepare_replica_roles.py b/.github/scripts/ci/prepare_replica_roles.py similarity index 100% rename from .circleci/scripts/prepare_replica_roles.py rename to .github/scripts/ci/prepare_replica_roles.py diff --git a/.circleci/scripts/run_integration.sh b/.github/scripts/ci/run_integration.sh similarity index 92% rename from .circleci/scripts/run_integration.sh rename to .github/scripts/ci/run_integration.sh index fb91bcce77f..ed405634796 100644 --- a/.circleci/scripts/run_integration.sh +++ b/.github/scripts/ci/run_integration.sh @@ -30,7 +30,7 @@ egress_cgroup=litellm-integration cleanup() { original_status=$? trap - EXIT INT TERM - sudo .venv/bin/python .circleci/scripts/stop_integration_processes.py \ + sudo .venv/bin/python .github/scripts/ci/stop_integration_processes.py \ "$integration_identity" "$(id -u)" "$proxy_pid" "$peer_pid" "$upstream_pid" \ > "$results/process-cleanup.txt" 2>&1 || original_status=1 for owned_pid in "$peer_pid" "$proxy_pid" "$upstream_pid"; do @@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out" test -f "$LITELLM_UI_PATH/index.html" fi -export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')" +export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')" export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED" uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1 @@ -93,7 +93,7 @@ export INTEGRATION_PROXY_DATABASE_URL="" export INTEGRATION_PROXY_READ_REPLICA_URL="" export INTEGRATION_ROUTING="" if [ "$mode" = replica ] || [ "$mode" = parity ]; then - .venv/bin/python .circleci/scripts/prepare_replica_roles.py > "$results/prepare-replica-roles.log" 2>&1 + .venv/bin/python .github/scripts/ci/prepare_replica_roles.py > "$results/prepare-replica-roles.log" 2>&1 export INTEGRATION_PROXY_DATABASE_URL="postgresql://litellm_writer:litellm-writer@127.0.0.1:5432/circle_test" export INTEGRATION_PROXY_READ_REPLICA_URL="postgresql://litellm_reader:litellm-reader@127.0.0.1:5432/circle_test" fi @@ -202,7 +202,7 @@ start_proxy() { if [ "$isolated_workers" = false ]; then start_proxy 4000 proxy.log proxy_pid="$launched_pid" - .venv/bin/python .circleci/scripts/wait_integration_services.py + .venv/bin/python .github/scripts/ci/wait_integration_services.py curl --noproxy '*' -sSf -X POST "$INTEGRATION_PROXY_URL/config/update" \ -H "Authorization: Bearer $LITELLM_MASTER_KEY" -H 'Content-Type: application/json' \ -d '{"router_settings": {"num_retries": 0}}' > "$results/seed-router-settings.json" @@ -210,7 +210,7 @@ if [ "$isolated_workers" = false ]; then export INTEGRATION_PEER_URL=http://127.0.0.1:4001 start_proxy 4001 peer.log peer_pid="$launched_pid" - .venv/bin/python .circleci/scripts/wait_integration_services.py + .venv/bin/python .github/scripts/ci/wait_integration_services.py fi fi @@ -233,14 +233,13 @@ if [ "$suite" = browser ]; then E2E_UI_BASE_URL="$E2E_UI_BASE_URL" E2E_UI_ARTIFACT_DIR="$E2E_UI_ARTIFACT_DIR" \ LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" CI=true \ node tests/e2e/ui/node_modules/@playwright/test/cli.js test --config tests/e2e/ui/integration.config.ts - .venv/bin/python .circleci/scripts/verify_integration_browser.py "$results/browser-results.json" + .venv/bin/python .github/scripts/ci/verify_integration_browser.py "$results/browser-results.json" exit 0 fi shard_arguments=() -if [ "${CIRCLE_NODE_TOTAL:-1}" -gt 1 ]; then - .venv/bin/python tests/integration/run.py "$suite" --list \ - | circleci tests split --split-by=timings --timings-type=filename > "$results/node-files.txt" +if [ -n "${INTEGRATION_SHARD_FILES:-}" ]; then + cp "$INTEGRATION_SHARD_FILES" "$results/node-files.txt" shard_arguments=(--shard-files "$results/node-files.txt") fi @@ -265,7 +264,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \ "${license_env[@]}" \ INTEGRATION_SEED="$INTEGRATION_SEED" \ INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \ - INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \ + INTEGRATION_CASE_SHARDS="${INTEGRATION_CASE_SHARDS:-1}" INTEGRATION_CASE_SHARD_INDEX="${INTEGRATION_CASE_SHARD_INDEX:-0}" \ LITELLM_LOCAL_MODEL_COST_MAP=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \ INTEGRATION_PROXY_DATABASE_URL="$INTEGRATION_PROXY_DATABASE_URL" \ INTEGRATION_PROXY_READ_REPLICA_URL="$INTEGRATION_PROXY_READ_REPLICA_URL" \ diff --git a/.circleci/scripts/stop_integration_processes.py b/.github/scripts/ci/stop_integration_processes.py similarity index 100% rename from .circleci/scripts/stop_integration_processes.py rename to .github/scripts/ci/stop_integration_processes.py diff --git a/.circleci/scripts/verify_integration_browser.py b/.github/scripts/ci/verify_integration_browser.py similarity index 94% rename from .circleci/scripts/verify_integration_browser.py rename to .github/scripts/ci/verify_integration_browser.py index 4468fadbcde..452a6cd43ae 100644 --- a/.circleci/scripts/verify_integration_browser.py +++ b/.github/scripts/ci/verify_integration_browser.py @@ -31,7 +31,7 @@ def main() -> None: result: Final = json.loads(Path(sys.argv[1]).read_text()) assert not result.get("errors"), result.get("errors") expected: Final = json.loads( - (Path(__file__).resolve().parents[2] / "tests/e2e/ui/tests/integrationCritical/expected.json").read_text() + (Path(__file__).resolve().parents[3] / "tests/e2e/ui/tests/integrationCritical/expected.json").read_text() ) assert expected and result["stats"]["expected"] == len(expected) assert all(result["stats"][name] == 0 for name in ("unexpected", "flaky", "skipped")) @@ -41,7 +41,7 @@ def main() -> None: suites: Final = TypeAdapter(list[BrowserSuite]).validate_python(result["suites"], strict=True) specs: Final = tuple(spec for suite in suites for spec in cases(suite)) - repository: Final = Path(__file__).resolve().parents[2] + repository: Final = Path(__file__).resolve().parents[3] report_root: Final = Path(result["config"]["rootDir"]) assert report_root.is_absolute(), "Playwright rootDir must be explicit" observed: Final = tuple( diff --git a/.circleci/scripts/wait_integration_services.py b/.github/scripts/ci/wait_integration_services.py similarity index 100% rename from .circleci/scripts/wait_integration_services.py rename to .github/scripts/ci/wait_integration_services.py diff --git a/.github/scripts/detect_changes.sh b/.github/scripts/detect_changes.sh index 2d427c92fb5..acc19be57f1 100755 --- a/.github/scripts/detect_changes.sh +++ b/.github/scripts/detect_changes.sh @@ -16,7 +16,7 @@ run_full() { } here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -classify="${here}/../../.circleci/scripts/classify_changes.sh" +classify="${here}/ci/classify_changes.sh" [ -n "${PR_NUMBER:-}" ] || run_full "not a pull_request event" [ -n "${REPO:-}" ] || run_full "no repository in the environment" diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index 73159bd64d2..8be7b89e076 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -55,7 +55,6 @@ jobs: services: postgres: image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - command: -c max_connections=1000 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -81,6 +80,17 @@ jobs: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: persist-credentials: false + - name: Configure PostgreSQL for integration workers + env: + POSTGRES_CONTAINER: ${{ job.services.postgres.id }} + run: | + docker exec "$POSTGRES_CONTAINER" sh -c 'printf "%s\n" "max_connections = 1000" "shared_preload_libraries = pg_stat_statements" "pg_stat_statements.track = all" "pg_stat_statements.max = 20000" >> "$PGDATA/postgresql.conf"' + docker restart "$POSTGRES_CONTAINER" + for attempt in $(seq 1 30); do + if docker exec "$POSTGRES_CONTAINER" pg_isready -U postgres; then exit 0; fi + sleep 1 + done + exit 1 - name: Set up Python timeout-minutes: 3 @@ -155,8 +165,8 @@ jobs: LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }} run: | case "$SUITE" in - *-replica) bash .circleci/scripts/run_integration.sh "${SUITE%-replica}" replica ;; - *) bash .circleci/scripts/run_integration.sh "$SUITE" ;; + *-replica) bash .github/scripts/ci/run_integration.sh "${SUITE%-replica}" replica ;; + *) bash .github/scripts/ci/run_integration.sh "$SUITE" ;; esac - name: Verify every case ran exactly once diff --git a/.github/workflows/test-mcp-dependency-resolution.yml b/.github/workflows/test-mcp-dependency-resolution.yml index 20efb351985..90589cf1e6f 100644 --- a/.github/workflows/test-mcp-dependency-resolution.yml +++ b/.github/workflows/test-mcp-dependency-resolution.yml @@ -28,7 +28,7 @@ on: - ".github/actions/cache-cargo-build/**" - ".github/scripts/detect_changes.sh" - ".github/scripts/uv_sync_with_retries.sh" - - ".circleci/scripts/classify_changes.sh" + - ".github/scripts/ci/classify_changes.sh" permissions: contents: read diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index d5fd755ba18..09138bd6c14 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -20,7 +20,7 @@ import json import os import subprocess import sys -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import Final @@ -28,8 +28,13 @@ from typing import Final COST_MAP_PATH: Final = "model_prices_and_context_window.json" BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json" PINS_PATH: Final = "ci_cd/cost_map_pins.json" +PINS_PATHS: Final = ( + "litellm/scripts/ci_cd/cost_map_pins.json", + "scripts/ci_cd/cost_map_pins.json", + "ci_cd/cost_map_pins.json", +) GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH) -CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH) +CHECKED_PATHS: Final = (*GUARDED_PATHS, *PINS_PATHS) BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_" PRICE_SYNC_BRANCH_PREFIX: Final = "litellm-providers/price-sync" TARGET_ENV: Final = "COST_MAP_GUARD_TARGET" @@ -193,7 +198,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str def pin_author_failures(head_ref: str, author_type: str, changed_files: Sequence[str]) -> tuple[str, ...]: - if PINS_PATH not in changed_files: + if not any(path in changed_files for path in PINS_PATHS): return () if head_ref.startswith(PRICE_SYNC_BRANCH_PREFIX): return (f"price sync PRs may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",) @@ -251,8 +256,11 @@ def _git(*args: str) -> str | None: return result.stdout if result.returncode == 0 else None -def snapshot(revision: str) -> Snapshot: - return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS)) +def snapshot(revision: str, *, read_git: Callable[[str, str], str | None] = _git) -> Snapshot: + return Snapshot( + *(read_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS), + next((text for path in PINS_PATHS if (text := read_git("show", f"{revision}:{path}")) is not None), ""), + ) def merged_tree(merge_base: str, target: str, head: str) -> str | MergeFailed | None: diff --git a/codecov.yaml b/codecov.yaml index 521088868bd..419a28ce6c6 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -6,7 +6,7 @@ codecov: ignore: - "litellm-rust/**" -# Uploads are flagged per workflow tier (GHA) or "circleci". carryforward makes +# Uploads are flagged per workflow tier. carryforward makes # a re-upload of a flag replace its prior session instead of accumulating a # conflicting one, and lets a commit reuse a flag from its parent when that flag # was not re-uploaded. Required because the same commit can receive the diff --git a/helm/litellm/ci/postgres.yaml b/helm/litellm/ci/postgres.yaml index aa210e932b0..01472950b3b 100644 --- a/helm/litellm/ci/postgres.yaml +++ b/helm/litellm/ci/postgres.yaml @@ -1,4 +1,4 @@ -# Throwaway PostgreSQL for the CircleCI helm_chart_testing job. Applied with +# Throwaway PostgreSQL for the local chart smoke test. Applied with # kubectl before `helm install`; the chart itself ships no database. apiVersion: v1 kind: Secret diff --git a/helm/litellm/ci/test-values.yaml b/helm/litellm/ci/test-values.yaml index 40c68489ed3..850dd48ee63 100644 --- a/helm/litellm/ci/test-values.yaml +++ b/helm/litellm/ci/test-values.yaml @@ -1,4 +1,4 @@ -# Values for the CircleCI helm_chart_testing job: a monolith install on a +# Values for the local chart smoke test: a monolith install on a # kind cluster from the image built by the pipeline (image.repository / tag / # pullPolicy are passed with --set), backed by the throwaway PostgreSQL in # ci/postgres.yaml so the migrations Job and the proxy have a database. diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 1774442cddc..138e203aba9 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -102,7 +102,7 @@ def reset_vcr_diag_dir() -> None: pass -# CircleCI truncates a step's retrievable output to the last ~400 KB. The +# CI truncates a step's retrievable output to the last ~400 KB. The # diagnostic log is emitted right *before* the final pytest summary line but # *after* the VCR CLASSIFICATION SUMMARY, so an unbounded dump (the body/key # matchers log one block per *episode comparison*, even on an eventual HIT) diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index 008a8f80cf0..ccdc616c243 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -30,7 +30,7 @@ deferred. It needs credentials Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family or behavior area. If you add a new folder, you must add a line here describing what kind of tests belong in it, so the layout stays self-describing. `gateway/` is the exception: it holds proxy configuration only and never tests -- `migrations/` - isolated Docker startup, concurrent migration, crash recovery, and legacy database compatibility. The CircleCI migration workflow enables `LITELLM_MIGRATION_TESTS=1`. These tests own their proxy containers and databases, so they do not use the shared proxy preflight or shared database cleanup +- `migrations/` - isolated Docker startup, concurrent migration, crash recovery, and legacy database compatibility. Set `LITELLM_MIGRATION_TESTS=1` to run this suite manually. No CI workflow invokes it. These tests own their proxy containers and databases, so they do not use the shared proxy preflight or shared database cleanup - `llm_translation/` - Azure realtime translation is explicitly skipped pending investigation of intermittent missing transcripts and remains unqualified. OpenAI translation and other Azure realtime cases remain enabled. LLM endpoint and provider-translation behavior: passthrough, custom pricing, OCR, and the non-chat inference endpoints (`/v1/responses`, `/v1/messages`, `/embeddings`, `/v1/rerank`, `/v1/audio/speech`, `/v1/images/generations`), each against a deployment the test creates via `/model/new` and deletes on teardown - `access_control/` - the gateway's authorization and error-shape contract: per-key model allow-lists, route-group permissions (`allowed_routes`), and unknown-model validation diff --git a/tests/e2e/ui/fixtures/mock_llm_server/server.py b/tests/e2e/ui/fixtures/mock_llm_server/server.py index 1e3dcecaff3..c2390abce34 100644 --- a/tests/e2e/ui/fixtures/mock_llm_server/server.py +++ b/tests/e2e/ui/fixtures/mock_llm_server/server.py @@ -130,8 +130,7 @@ async def systemone(request: Request): if __name__ == "__main__": # The port is overridable so two checkouts can run the harness at the same - # time; the default keeps every existing caller (run_e2e.sh, the CircleCI - # job, the e2e chart's sidecar) working untouched. + # time; the default keeps run_e2e.sh and the e2e chart's sidecar working untouched. # # The HOST is deliberately NOT configurable. Binding loopback is what makes # this reachable at 127.0.0.1:8090 from inside the proxy's own pod, which is diff --git a/tests/e2e/ui/integration.config.ts b/tests/e2e/ui/integration.config.ts index 55ccb860512..f40cd1d3b85 100644 --- a/tests/e2e/ui/integration.config.ts +++ b/tests/e2e/ui/integration.config.ts @@ -2,9 +2,6 @@ import { defineConfig, devices } from "@playwright/test"; import * as path from "path"; import { ARTIFACT_DIR, UI_BASE_URL } from "./constants"; -if (process.env.GITHUB_ACTIONS === "true") - throw new Error("Integration contracts are owned by CircleCI"); - export default defineConfig({ testDir: "./tests/integrationCritical", testMatch: "*.spec.ts", diff --git a/tests/e2e/ui/run_e2e.sh b/tests/e2e/ui/run_e2e.sh index 97cbe5fcf59..183b1e3dbc6 100755 --- a/tests/e2e/ui/run_e2e.sh +++ b/tests/e2e/ui/run_e2e.sh @@ -36,7 +36,7 @@ PROXY_LOG="" # Ports, overridable so two checkouts can run this harness at the same time -- # otherwise a second run aborts on "port 4000 is in use" and the only way out is # to stop someone else's stack. Defaults are the historical values, so an unset -# environment behaves exactly as before (CI, the CircleCI job and the docs all +# environment behaves exactly as before (CI and the docs # assume 4000/5432/8090). PROXY_PORT="${PROXY_PORT:-4000}" POSTGRES_PORT="${POSTGRES_PORT:-5432}" diff --git a/tests/e2e/ui/tests/auth/proxyLogoutUrl.spec.ts b/tests/e2e/ui/tests/auth/proxyLogoutUrl.spec.ts index 3a79ce82717..2fd8972b021 100644 --- a/tests/e2e/ui/tests/auth/proxyLogoutUrl.spec.ts +++ b/tests/e2e/ui/tests/auth/proxyLogoutUrl.spec.ts @@ -3,9 +3,9 @@ import { Role, users } from "../../fixtures/users"; import { logInThroughLoginPage } from "../../helpers/userOnboarding"; /** - * Runs as part of the standard e2e suite: both `run_e2e.sh` and the CircleCI - * `e2e_ui_testing` job boot the proxy with PROXY_LOGOUT_URL=https://www.example.com - * and export the same value to this Playwright process. The spec reads it to + * Runs as part of the standard e2e suite. `run_e2e.sh` boots the proxy with + * PROXY_LOGOUT_URL=https://www.example.com and exports the same value to + * this Playwright process. The spec reads it to * know where the browser is expected to land. * * The skip guard below is a safety net for environments that launch the proxy diff --git a/tests/e2e/ui/tests/proxy-admin/license.spec.ts b/tests/e2e/ui/tests/proxy-admin/license.spec.ts index 37a0e324f27..cfc098b56e4 100644 --- a/tests/e2e/ui/tests/proxy-admin/license.spec.ts +++ b/tests/e2e/ui/tests/proxy-admin/license.spec.ts @@ -4,11 +4,11 @@ import { ADMIN_STORAGE_PATH } from "../../constants"; /** * Sanity check that LITELLM_LICENSE is being forwarded to the proxy when set - * in the environment (e.g. CircleCI's `e2e_ui_testing` job). The login JWT's + * in the environment (e.g. `run_e2e.sh`). The login JWT's * `premium_user` claim is the same value the dashboard reads to enable * premium-gated UI surfaces (Team-BYOK switch, etc.), so asserting it here * catches any future regression where the env var stops being plumbed - * through `run_e2e.sh` / `.circleci/config.yml`. + * through `run_e2e.sh`. * * Skips locally when no license is configured. */ diff --git a/tests/integration/AGENTS.md b/tests/integration/AGENTS.md index e5068d24e88..b430ab553a5 100644 --- a/tests/integration/AGENTS.md +++ b/tests/integration/AGENTS.md @@ -26,20 +26,20 @@ directory that a `GROUPS` entry in `run.py` selects. There is no manifest and no tests. A product bug the test exposes is `pytest.skip("BUG: ")` at the top of the body, not a fix in the test and not a deletion. Needs no network, proxy, DB or Redis: `tests/unit` -CircleCI's integration groups run every file here except the OCI and sandbox tests, which no CI runs, and +GitHub Actions integration groups run every file here except the OCI and sandbox tests, which no CI runs, and the files in `GITHUB_FILES` in `run.py`. `test-integration.yml` runs the same groups through the same `run_integration.sh` on GitHub Actions, on main every three hours from 8am to 8pm Pacific Monday to Saturday, -and on a pull request when someone adds the `run-ci` label. Neither runs every pull request, so a new file here +and on a pull request when someone adds the `run-ci` label. These jobs do not run every pull request, so a new file here first runs on main after its PR merges, or earlier if the PR gets `run-ci`. Wanting a file to run before merge is no reason to change CI, and a new dedicated workflow for it is the same mistake as a shared shard. Write the file, leave `GITHUB_FILES` and every workflow alone, and tell the user it first runs on main. `GITHUB_FILES` is only for a file that -needs what CircleCI's groups cannot give it, like the images `image-scan.yml` builds or the plain pytest -run CircleCI's `roi-database` job gives `test_roi_observed.py`. Add a file there only in that case, +needs what GitHub Actions groups cannot give it, like the images `image-scan.yml` builds. `test_roi_observed.py` previously needed a dedicated plain pytest +job and now has no CI owner. It is listed explicitly in the coverage allowlist. Add a file to `GITHUB_FILES` only when needed, together with its dedicated job, and list a new dedicated GitHub workflow with the directory it owns in `DEDICATED_INTEGRATION_WORKFLOWS` in `.github/scripts/assert_ci_coverage.py`. The `ci-coverage` check -fails a file here that a GitHub workflow runs unless it is in `GITHUB_FILES` and that workflow owns its -directory (#46780) +fails a file here that another workflow selects unless that workflow owns its path in +`DEDICATED_INTEGRATION_WORKFLOWS` (#46780). ## Fixtures that boot a proxy @@ -50,4 +50,4 @@ on one xdist worker, so the proxy they share boots there once and stops when tha file. A test that requests one more of those fixtures than its neighbors gets its own group and can boot the proxy again on another worker, so keep that set the same across the tests that share a proxy. An unpinned session fixture boots a copy on every worker that runs one of its tests and keeps them all until -the node ends, and those copies ran CircleCI integration-extensions nodes out of memory +the node ends, and those copies ran GitHub Actions integration-extensions nodes out of memory diff --git a/tests/integration/README.md b/tests/integration/README.md index a1d224749b3..1f9399c8b22 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,24 +1,24 @@ # Integration contracts -These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite, and `.github/workflows/test-integration.yml` runs the same groups through `run_integration.sh` on GitHub Actions for `run-ci` pull requests and scheduled main runs. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls +These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. `.github/workflows/test-integration.yml` runs the groups through `.github/scripts/ci/run_integration.sh` for `run-ci` pull requests and scheduled main runs. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls -The ROI database contracts in `database/test_roi_observed.py` run as plain pytest outside `run_integration.sh` in CircleCI's `roi-database` Postgres job. The job uploads coverage with the `roi-postgres` flag. These contracts own temporary databases and script only the external provider transport +The ROI database contracts in `database/test_roi_observed.py` run as plain pytest outside `run_integration.sh`. They own temporary databases and script only the external provider transport. No remaining CI job invokes them. They are listed explicitly in `.github/ci-coverage-allowlist.yml`. -The Redis cluster IAM contracts in `sdk/test_redis_cluster_iam_auth.py` start their own `redis-server` and mock the cloud credential clients, so CircleCI's `sdk` group installs `redis-server` before it runs them +The Redis cluster IAM contracts in `sdk/test_redis_cluster_iam_auth.py` start their own `redis-server` and mock the cloud credential clients, so the GitHub Actions `sdk` group installs `redis-server` before it runs them The `cost` group is driven by the `CostTrackingTestCase` literals under `cost_calculation/cases/.py` (gathered by `cost_calculation/catalog.py`), with the scripted cost map in `cost_calculation/cost_map.py`. Each case has a name, contract ID, cost-map model, optional deployment overrides, request body, tagged response and exact or recount expectations. Request bodies use `$MODEL` for the registered proxy model, while responses use `$REQUEST_ID` for the per-run scenario ID. To add a case, add a cost-map entry when the model is new, add the request body and exact provider response data, and add hand-computed expected values. The upstream serves each stored response for any path under `/`, while the test-owned cost map is served over loopback through `LITELLM_MODEL_COST_MAP_URL` -Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `mcp`, `sdk` or `cost` to run a selected group. The group to directory mapping is the `GROUPS` literal at the top of `run.py`; a new directory needs a `GROUPS` entry and an `OWNED_DIRECTORIES` entry in `_support/manifest.py`. Set `INTEGRATION_WORKERS` above 1 to run a group under pytest-xdist. Every group run by more than one worker gives each worker a private database, Redis, scripted upstream process, fake provider and proxy, with a peer and read-only database role where required. There is no opt-in list and no shared proxy, upstream, database or Redis in that mode: `run_integration.sh` does not start them and does not hand their addresses to the run, so a test can only reach its own worker's services. The pytest controller owns these services and closes them after worker failures. Their cases use pytest-xdist load groups inside the files selected by CircleCI timing splits. Module and class fixtures are grouped automatically, so authors do not need scheduling markers. Function fixtures remain independently schedulable. Tests that own an independent mutable sink for every case should use function fixtures, so a shared module fixture does not serialize them or share pending observations. Nested harness probes discard the parent run's shard, worker and result settings, then supply their own selections. MCP keeps its explicit load groups. Automatic group suffixes are removed from JUnit and execution records to preserve case identities. Owned proxy helpers wait for a healthy response from every configured, live worker before yielding. Serial runs keep the supplied deployment, including routing parity runs. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. When that deployment runs more than one proxy worker, set `INTEGRATION_PROXY_WORKERS` to the count so a test that writes a model and then calls it waits out the config reload interval, the only cross-worker convergence bound the wire exposes. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate +Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `mcp`, `sdk` or `cost` to run a selected group. The group to directory mapping is the `GROUPS` literal at the top of `run.py`; a new directory needs a `GROUPS` entry and an `OWNED_DIRECTORIES` entry in `_support/manifest.py`. Set `INTEGRATION_WORKERS` above 1 to run a group under pytest-xdist. Every group run by more than one worker gives each worker a private database, Redis, scripted upstream process, fake provider and proxy, with a peer and read-only database role where required. There is no opt-in list and no shared proxy, upstream, database or Redis in that mode: `run_integration.sh` does not start them and does not hand their addresses to the run, so a test can only reach its own worker's services. The pytest controller owns these services and closes them after worker failures. Their cases use pytest-xdist load groups within each group's selected files. Module and class fixtures are grouped automatically, so authors do not need scheduling markers. Function fixtures remain independently schedulable. Tests that own an independent mutable sink for every case should use function fixtures, so a shared module fixture does not serialize them or share pending observations. Nested harness probes discard the parent run's shard, worker and result settings, then supply their own selections. MCP keeps its explicit load groups. Automatic group suffixes are removed from JUnit and execution records to preserve case identities. Owned proxy helpers wait for a healthy response from every configured, live worker before yielding. Serial runs keep the supplied deployment, including routing parity runs. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. When that deployment runs more than one proxy worker, set `INTEGRATION_PROXY_WORKERS` to the count so a test that writes a model and then calls it waits out the config reload interval, the only cross-worker convergence bound the wire exposes. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate Management, MCP and accounting also require `INTEGRATION_PEER_URL`, `REDIS_HOST` and `REDIS_PORT`. Under several workers each worker gets a second directly addressed proxy sharing only that worker's stores, and a single-worker run uses the two proxies `run_integration.sh` starts. The test-only CLI wrapper supplies enterprise route entitlement, following the existing behavior suite's convention. It does not qualify license validation; run it with one worker and no reload -The generated lifecycle models use 20 examples, eight steps, generation and shrinking, with isolated resources per example. HTTP operation caps include generation and shrinking and exempt cleanup. Local qualification defaults to seed 4106601 and canonical order; CircleCI derives exploration and ordering seeds from the checked-out revision and workflow ID. The ordering seed shuffles files, scoped fixture configurations and cases within each configuration. Ordering keeps shared module and class configurations together where their dependencies permit it. Dependent fixtures with their own module parameters can still rebuild an outer configuration, with every case retained +The generated lifecycle models use 20 examples, eight steps, generation and shrinking, with isolated resources per example. HTTP operation caps include generation and shrinking and exempt cleanup. Local qualification defaults to seed 4106601 and canonical order; GitHub Actions derives exploration and ordering seeds from the checked-out revision and workflow ID. The ordering seed shuffles files, scoped fixture configurations and cases within each configuration. Ordering keeps shared module and class configurations together where their dependencies permit it. Dependent fixtures with their own module parameters can still rebuild an outer configuration, with every case retained -CircleCI uses its built-in `circleci tests split --split-by=timings --timings-type=filename` to assign test files using the uploaded JUnit timings. Each file stays on one node, preserving its module and parametrized fixtures. Jobs default to one node. Parallelism should use the smallest node count that keeps the complete job, including setup and cleanup, within twenty minutes. A group that already finishes within that target on one node stays on one node. Management, accounting, providers and management replica currently use four nodes. Extensions uses eight nodes; database and MCP use two, and fast suites use one. No job is configured above eight nodes. A single slow file can still determine a suite's runtime; split such a file along independent fixture boundaries rather than adding more nodes +GitHub Actions runs one job per suite and uses pytest-xdist workers inside that job. `INTEGRATION_WORKERS` controls the worker count, with defaults in `test-integration.yml`. Each worker owns its services. A slow file can still determine a suite's runtime, so split it along independent fixture boundaries rather than adding more workers -Every node collects the same candidate files before applying CircleCI's assigned file list. Its execution record retains the candidate inventory and the selected, passed and skipped cases. An empty assigned file list produces an empty shard, not a full-suite fallback. Every parallel group has a dependent census job that rejects an empty candidate inventory, missing shards, omitted cases and duplicate execution. Standard and replica runs keep separate workspace records +Every group records its candidate inventory and the selected, passed and skipped cases. A census step rejects an empty candidate inventory, omitted cases and duplicate execution. Standard and replica runs keep separate records -Future change-based selection belongs before timing splitting. Pass the selected group files or node IDs to `run.py --list` to list their files, then give those filenames to CircleCI. Run the same candidate selection on every node with `--shard-files --shards --shard-index `. The census compares execution with the candidate inventory, while full-suite runs remain the default. CircleCI determines scheduling from timing history; timing data never selects or omits candidates. Use `--seed` and `--order-seed` to reproduce test generation and order. The installed Hypothesis version, settings, seeds and collected order are recorded beside the execution manifest +Future change-based selection belongs before worker scheduling. Pass the selected group files or node IDs to `run.py --list` to list their files. An external shard assignment can use `INTEGRATION_SHARD_FILES` with `INTEGRATION_CASE_SHARDS` and `INTEGRATION_CASE_SHARD_INDEX`. An empty assigned file list produces an empty shard, not a full-suite fallback. The current GitHub Actions jobs use one shard with index zero and use pytest-xdist within it. Use `--seed` and `--order-seed` to reproduce test generation and order. The installed Hypothesis version, settings, seeds and collected order are recorded beside the execution manifest A future selector must explicitly skip a group when no candidates apply; omitting positional candidates means the full group. Keep full runs on the default branch and scheduled workflows, and select the full affected groups when shared fixtures, CI configuration or an unknown dependency changes @@ -30,9 +30,9 @@ Owned proxy workers configure a 32-thread executor before application startup, p Every owned listener's default port comes from `allocate_port()` in `_support/ports.py`. That covers owned proxies, scripted upstreams, fake providers, OTLP sinks, database relays and Redis, including peers such as `wire_server` that bind and keep their own sockets. Keeping those listeners out of the ephemeral range prevents them from taking a port Prisma selected with a bind-and-close probe before its query engine starts. Each pytest process claims a block of 200 ports between 10000 and 32767 by holding an exclusive lock file per block in the system temp directory, so two runs on one machine, such as parallel worktrees, never share a block, and the OS drops the lock when the process exits. The range sits above the fixed service ports and below the Linux and macOS ephemeral ranges, so a port-0 bind or an outbound connection never lands in a block. Ports inside a block go out in order and are never handed out twice in a run, a port that something outside the harness already holds is skipped, and a process that uses up its block claims the next free one. A test that restarts a server on the same port allocates that port up front instead of reading it back from the first server. TQ012 in `scripts/check_test_quality.py` rejects reading a port back from a port-0 bind anywhere under `tests/integration` -The CircleCI workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. `GITHUB_FILES` in `run.py` lists the files excluded from the `run_integration.sh` selection +The GitHub Actions workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. `GITHUB_FILES` in `run.py` lists the files excluded from the `run_integration.sh` selection -There is no hand-maintained list of test cases. A positional argument is a file of the group or a pytest node id inside one (`path::test[param]`), so one cell of a parametrized file can run alone. The runner fails only when pytest fails, when collection errors, or when a selected file collects zero tests. Older tests still carry `@pytest.mark.covers(...)` decorators; the marker stays registered so they collect, but the IDs are not checked against anything and new tests should not use it. The GitHub Actions coverage census reads the `GROUPS` literal in `run.py` and treats every `tests/integration//test_*.py` file in a scheduled group as owned by CircleCI +There is no hand-maintained list of test cases. A positional argument is a file of the group or a pytest node id inside one (`path::test[param]`), so one cell of a parametrized file can run alone. The runner fails only when pytest fails, when collection errors, or when a selected file collects zero tests. Older tests still carry `@pytest.mark.covers(...)` decorators; the marker stays registered so they collect, but the IDs are not checked against anything and new tests should not use it. The GitHub Actions coverage census reads the `GROUPS` literal in `run.py` and treats every `tests/integration//test_*.py` file in a scheduled group as owned by GitHub Actions Provider sentinels currently use the controlled server, not live recordings. The provider shard also runs the existing strict replay controls for changed requests, exhausted interactions, leftover interactions and no provider connection. Future recorded scenarios must use that replay-only implementation; missing recordings cannot fall back to a real provider. `GET /__observations` returns every request the upstream recorded since the last `DELETE /__observations`, and reading it never clears it. An autouse fixture resets it before each test, so a test that needs a clean slate between phases sends the DELETE itself. A check that a request never reached the upstream asserts on the same read that shows a request sent after it did reach the upstream, so the absence cannot pass on an empty or reset log @@ -46,20 +46,20 @@ Accounting cases compare persisted input and output cost components against lite Provider contracts exercise actual TCP requests with synthetic credentials and local protocol peers. The S3 verifier uses independently implemented equations, a published known-answer vector, a fixed signing clock and deliberately invalid signed requests. Bedrock cases clear ambient AWS credential sources and check the literal model path, loaded role references, STS requests and bearer-only behavior -CircleCI runs providers on four isolated nodes using its file timing splitter. Each node owns its services. The dependent providers coverage job verifies all four execution records against the candidate case inventory and rejects missing, duplicate, failed or incomplete execution. The node's assigned filenames are saved in `node-files.txt` beside its execution record +GitHub Actions runs the providers group with 32 pytest workers, each with its own services. The census step verifies the execution record against the candidate inventory and rejects missing, duplicate, failed or incomplete execution -Streaming checks send real HTTP transfer chunks, including one-byte partitions, fragmented tools, incomplete transfers and a cancellation barrier. They assert meaningful text, tool arguments, final usage and persisted cost. The Redis recovery case owns a separate database and Redis process, uses the supported one-second circuit-breaker recovery setting, waits for the real subscriber and verifies response data in Redis after restart. CircleCI reuses its existing Redis image for that extra process; it never pulls an image during tests +Streaming checks send real HTTP transfer chunks, including one-byte partitions, fragmented tools, incomplete transfers and a cancellation barrier. They assert meaningful text, tool arguments, final usage and persisted cost. The Redis recovery case owns a separate database and Redis process, uses the supported one-second circuit-breaker recovery setting, waits for the real subscriber and verifies response data in Redis after restart. GitHub Actions reuses its existing Redis image for that extra process; it never pulls an image during tests The `messages_endpoint/` directory holds `/v1/messages` endpoint contracts: native-provider backends under `providers/` (`anthropic`, `bedrock`, `gemini`) and the translation bridges (`responses_bridge`, `chat_bridge`) at the top level. It runs in the providers shard; `run.py` selects test files recursively under each scheduled directory -The sdk shard exercises the SDK's own HTTP clients against local protocol peers with no gateway in the path, so a case here fails only when the client library or its wire behavior changes. The HTTP/2 case runs a hypercorn TLS peer offering h2 and http/1.1 over ALPN, drives the sync and async httpx handlers at it with `LITELLM_HTTP2` off and on, and asserts the version both the client and the peer observed on the wire. Put a test here only when it needs no proxy or database. CircleCI starts a local Redis for this shard like the others, so SDK-side caching cases that need a real Redis server belong here too; a case that reaches the gateway belongs in one of the other shards +The sdk shard exercises the SDK's own HTTP clients against local protocol peers with no gateway in the path, so a case here fails only when the client library or its wire behavior changes. The HTTP/2 case runs a hypercorn TLS peer offering h2 and http/1.1 over ALPN, drives the sync and async httpx handlers at it with `LITELLM_HTTP2` off and on, and asserts the version both the client and the peer observed on the wire. Put a test here only when it needs no proxy or database. GitHub Actions starts a local Redis for this shard like the others, so SDK-side caching cases that need a real Redis server belong here too; a case that reaches the gateway belongs in one of the other shards -The extensions shard uses the built-in generic callback and guardrail transports. It checks callback correlation and credential exclusion, guardrail rewriting and denial, retained OpenAI consumers and A2A wire versions. CircleCI runs it on eight xlarge nodes using its file timing splitter. Each node runs its assigned files with four pytest workers, each owning its database, Redis, upstream and proxy. The dependent coverage job verifies all eight execution records against the candidate case inventory. Tests must not assume a particular set of sibling files +The extensions shard uses the built-in generic callback and guardrail transports. It checks callback correlation and credential exclusion, guardrail rewriting and denial, retained OpenAI consumers and A2A wire versions. GitHub Actions runs it with 48 pytest workers, each owning its database, Redis, upstream and proxy. The census step verifies execution against the candidate inventory. Tests must not assume a particular set of sibling files The security shard uses four pytest workers on an xlarge machine, each with its own isolated services like every other multi-worker group. Route sweeps use a new HTTP client for every call and share only the TLS verification context, so authentication and response cookies cannot cross callers. The canary rig supplies owned OpenAI and Anthropic endpoints and synthetic credentials and rejects attempted external egress The two mcp nodes own separate services and use the same case census as providers and extensions. Their coverage data is combined into one MCP report after both nodes finish. Each node runs the MCP gateway against SDK peers owned by each test (`_support/mcp.py`): streamable HTTP, SSE and stdio peers, an OpenAPI-spec app, and an OAuth 2.1 authorization-server double. Every peer records the requests it receives so a test can assert what reached the peer, not only what the proxy answered. The shard runs with `INTEGRATION_WORKERS` set and with `INTEGRATION_COVERAGE=1`, which starts every worker's proxy and peer under `coverage run --parallel-mode` limited to the MCP modules and stores `coverage.txt` plus an HTML report with the job artifacts. `mcp_coverage.toml` measures lines through `sys.monitoring`, because the CPython 3.12 trace function that branch measurement needs made a covered proxy take about four times as long to become ready. A test that fails because the product is wrong is skipped with `pytest.skip("BUG: ")` so the skip list in `execution.json` is the open MCP bug list -Browser contracts live in `tests/e2e/ui/tests/integrationCritical` and run only through `tests/e2e/ui/integration.config.ts`. The expected browser results are listed in `expected.json` in that directory and checked by `.circleci/scripts/verify_integration_browser.py`. The CircleCI browser shard builds the checked-out dashboard, starts the owned proxy with that build, and verifies one exact browser result without retries or skips. The default Playwright selection excludes this directory. The focused project flow asserts the submitted create and clear values, fresh SQL state and actual blocked/restored serving while preserving model restrictions +Browser contracts live in `tests/e2e/ui/tests/integrationCritical` and run only through `tests/e2e/ui/integration.config.ts`. The expected browser results are listed in `expected.json` in that directory and checked by `.github/scripts/ci/verify_integration_browser.py`. The GitHub Actions browser shard builds the checked-out dashboard, starts the owned proxy with that build, and verifies one exact browser result without retries or skips. The default Playwright selection excludes this directory. The focused project flow asserts the submitted create and clear values, fresh SQL state and actual blocked/restored serving while preserving model restrictions -Two always-on `-replica` CircleCI jobs (management, database) run their groups in replica mode, where every proxy connects through a real `litellm_writer` role and a real read-only `litellm_reader` role against the same PostgreSQL. Nothing is captured there: the job passes when the tests pass, and a write routed to the read-only reader fails the test that issued it. A deeper check runs on demand as the `routing_parity` workflow, triggered through the CircleCI API v2 pipeline endpoint on the PR branch with `{"parameters": {"routing_parity_base": "<40-hex merge-base sha>"}}`. The workflow fans out over the seven groups, and each `routing-parity-` job runs its own group twice against the same test harness, once with `litellm/`, `enterprise/`, and `litellm-proxy-extras/` checked out from the base revision and once from the head, with a pytest plugin snapshotting `pg_stat_statements` into `routing-observed.json` per side. The `check` step then compares the two observations and writes `routing-diff.txt`: a statement seen on both sides fails when its role set changed, globally or for the same test (per-test capture is skipped under xdist), unless it is listed in `tests/integration/routing/either_role.json`, where each entry names the statement and a one-line reason it legitimately runs on whichever role asks for it, printed under `== either role ==`. Queries seen on only one side are listed, never failed, `pg_stat_statements` evictions and a role that never ran a statement are failures +Two `-replica` GitHub Actions suites (management, database) run their groups in replica mode, where every proxy connects through a real `litellm_writer` role and a real read-only `litellm_reader` role against the same PostgreSQL. Nothing is captured there: the job passes when the tests pass, and a write routed to the read-only reader fails the test that issued it. Routing parity support remains in the local harness, but no CI workflow runs it. diff --git a/tests/integration/_support/redis_process.py b/tests/integration/_support/redis_process.py index 1a325d73659..494d13c750e 100644 --- a/tests/integration/_support/redis_process.py +++ b/tests/integration/_support/redis_process.py @@ -152,7 +152,7 @@ def owned_redis( ], text=True, ).strip() - assert host, "CircleCI owned Redis container has no address" + assert host, "CI-owned Redis container has no address" prefix: Final = ("docker", "exec", "redis-cache", "redis-server") output: Final = Path(inherited.get("INTEGRATION_RESULTS_DIR", str(directory))) output.mkdir(parents=True, exist_ok=True) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d605bd3d7d7..8c0c463c217 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -28,7 +28,6 @@ from tests.integration._support.provider import SharedProvider, shared_provider from tests.integration._support.routing import RoutingPlugin from tests.integration._support.sdk_models import build_sdk_models from tests.integration._support.workers import gateway_profile, start_worker_stacks, worker_stack -from tests.integration.run import GITHUB_FILES if TYPE_CHECKING: from tests.integration._support.actors import ActorSet, GatewayClient, ManagementDatabase, ScratchResources @@ -216,11 +215,6 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item for item in items if item.path.is_relative_to(root) and item.path.relative_to(root).parts[0] in OWNED_DIRECTORIES ) - circleci_only: Final = tuple( - item for item in owned if item.path.relative_to(root.parents[1]).as_posix() not in GITHUB_FILES - ) - if circleci_only and os.environ.get("GITHUB_ACTIONS") == "true": - raise pytest.UsageError("Integration contracts are owned by CircleCI") for item in owned: item.add_marker(pytest.mark.integration) config.stash[INVENTORY] = tuple(_case_identity(item.nodeid) for item in owned) @@ -230,7 +224,7 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item raise pytest.UsageError("Integration shard index must be within the positive shard count") files: Final = config.stash[SHARD_FILES] if count > 1 and files is None: - raise pytest.UsageError("Parallel integration runs require CircleCI's shard file selection") + raise pytest.UsageError("Parallel integration runs require a shard file selection") if files is not None: available: Final = frozenset(item.nodeid.split("::", 1)[0] for item in owned) for file in sorted(files - available - config.stash.get(FAILED_COLLECTIONS, frozenset())): diff --git a/tests/integration/database/test_harness_lifecycle.py b/tests/integration/database/test_harness_lifecycle.py index 4cb01dfeef6..724ba5412d2 100644 --- a/tests/integration/database/test_harness_lifecycle.py +++ b/tests/integration/database/test_harness_lifecycle.py @@ -79,8 +79,7 @@ def _nested_pytest_environment(environment: Mapping[str, str]) -> dict[str, str] "INTEGRATION_COVERAGE_FILE", "INTEGRATION_CASE_SHARDS", "INTEGRATION_CASE_SHARD_INDEX", - "CIRCLE_NODE_TOTAL", - "CIRCLE_NODE_INDEX", + "INTEGRATION_SHARD_FILES", "PYTEST_XDIST_WORKER", "PYTEST_XDIST_WORKER_COUNT", "PYTEST_XDIST_TESTRUNUID", @@ -750,8 +749,6 @@ def test_direct_and_dependent_consumers_reuse_the_same_module_parameter(tmp_path **os.environ, "INTEGRATION_CASE_SHARDS": "2", "INTEGRATION_CASE_SHARD_INDEX": "1", - "CIRCLE_NODE_TOTAL": "2", - "CIRCLE_NODE_INDEX": "1", } ), capture_output=True, diff --git a/tests/unit/test_assert_ci_coverage.py b/tests/unit/test_assert_ci_coverage.py index 87c56f1a42b..4127432bd3d 100644 --- a/tests/unit/test_assert_ci_coverage.py +++ b/tests/unit/test_assert_ci_coverage.py @@ -26,7 +26,25 @@ sys.modules[_spec.name] = coverage # @dataclass(slots=True) rebuilds via sys.mo _spec.loader.exec_module(coverage) -def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path: Path) -> None: +def _write_group_workflow(root: Path, suites: tuple[str, ...] = ("management",)) -> Path: + workflow: Final = root / ".github/workflows/test-integration.yml" + workflow.parent.mkdir(parents=True, exist_ok=True) + workflow.write_text( + yaml.safe_dump( + { + "jobs": { + "integration": { + "strategy": {"matrix": {"suite": list(suites)}}, + "steps": [{"run": "bash .github/scripts/ci/run_integration.sh $SUITE"}], + } + } + } + ) + ) + return workflow + + +def test_integration_groups_require_exclusive_scheduled_github_owner(tmp_path: Path) -> None: test_path: Final = "tests/integration/management/test_contract.py" test_file: Final = tmp_path / test_path test_file.parent.mkdir(parents=True) @@ -37,39 +55,18 @@ def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path: ) paths, findings = coverage._integration_ownership(tmp_path) assert not paths - assert [finding.detail for finding in findings] == ["dedicated CircleCI runner is missing"] - circle: Final = tmp_path / ".circleci/config.yml" - circle.parent.mkdir() - circle.write_text( - yaml.safe_dump( - { - "jobs": { - "integration_contracts": { - "steps": [{"run": {"command": "bash .circleci/scripts/run_integration.sh management"}}] - } - }, - "workflows": {"integration": {"jobs": [{"integration_contracts": {"suite": "management"}}]}}, - } - ) - ) + assert [finding.detail for finding in findings] == ["dedicated GitHub Actions runner is missing"] + _write_group_workflow(tmp_path) paths, findings = coverage._integration_ownership(tmp_path) assert paths == frozenset({test_path}) assert findings == () - configured: Final = yaml.safe_load(circle.read_text()) - configured["workflows"]["integration"]["jobs"] = [ - {"integration_contracts": {"matrix": {"parameters": {"suite": ["providers"]}}}} - ] - circle.write_text(yaml.safe_dump(configured)) + _write_group_workflow(tmp_path, ("providers",)) _, findings = coverage._integration_ownership(tmp_path) assert [(finding.subject, finding.detail) for finding in findings] == [ - ("management", "canonical integration group is not scheduled by CircleCI") + ("management", "canonical integration group is not scheduled by GitHub Actions") ] - configured["workflows"]["integration"]["jobs"][0]["integration_contracts"]["matrix"]["parameters"]["suite"] = [ - "management" - ] - circle.write_text(yaml.safe_dump(configured)) + _write_group_workflow(tmp_path) workflow: Final = tmp_path / ".github/workflows/test.yml" - workflow.parent.mkdir(parents=True) workflow.write_text(yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": "pytest tests/integration"}]}}})) _, findings = coverage._integration_ownership(tmp_path) assert [(finding.subject, finding.detail) for finding in findings] == [ @@ -98,12 +95,9 @@ def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path: assert [(finding.subject, finding.detail) for finding in missing_invocation] == [ (github_path, "GitHub-owned integration contract has no invoking job") ] - circle_config: Final = yaml.safe_load(circle.read_text()) - circle_config["jobs"]["postgres_suite"] = {"parameters": {"test_path": {"type": "string"}}} - circle_config["workflows"]["integration"]["jobs"].append({"postgres_suite": {"test_path": github_path}}) - circle.write_text(yaml.safe_dump(circle_config)) - _, circle_invocation = coverage._integration_ownership(tmp_path) - assert circle_invocation == () + dedicated.write_text(yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": f"pytest {github_path}"}]}}})) + _, restored_invocation = coverage._integration_ownership(tmp_path) + assert restored_invocation == () def _write_integration_tree(root: Path, github_path: str) -> None: @@ -117,20 +111,24 @@ def _write_integration_tree(root: Path, github_path: str) -> None: 'GROUPS: Final = MappingProxyType({"management": ("management",)})\n' f"GITHUB_FILES: Final = frozenset({{{github_path!r}}})\n" ) - (root / ".circleci").mkdir() - (root / ".circleci/config.yml").write_text( - yaml.safe_dump( - { - "jobs": { - "integration_contracts": { - "steps": [{"run": {"command": "bash .circleci/scripts/run_integration.sh management"}}] - } - }, - "workflows": {"integration": {"jobs": [{"integration_contracts": {"suite": "management"}}]}}, - } - ) - ) - (root / ".github/workflows").mkdir(parents=True) + _write_group_workflow(root) + + +@pytest.mark.parametrize( + ("matrix", "expected"), + ( + (["management", "database"], ("management", "database")), + ([], ()), + (None, ()), + ("${{ inputs.suite }}", ()), + ( + "${{ fromJSON(inputs.suite == 'all' && '[\"management\",\"database\"]' || format('[\"{0}\"]', inputs.suite)) }}", + ("management", "database"), + ), + ), +) +def test_integration_matrix_accounts_for_default_suites(matrix: object, expected: tuple[str, ...]) -> None: + assert coverage._matrix_suites(matrix) == expected @pytest.mark.parametrize( @@ -172,20 +170,18 @@ def test_an_integration_file_outside_every_group_cannot_run_on_a_github_shard(tm assert not [finding for finding in before if finding.subject == ungrouped] -def test_circleci_postgres_suites_count_their_test_path_as_invoked() -> None: +def test_workflow_matrices_count_their_test_paths_as_invoked() -> None: config: Final = { - "workflows": { - "integration": { - "jobs": [ - {"postgres_suite": {"name": "proxy-behavior", "test_path": "tests/proxy_behavior", "seed": True}}, - { - "postgres_suite": { - "name": "roi-database", - "test_path": "tests/integration/database/test_roi_observed.py", - "seed": False, - } - }, - ] + "jobs": { + "postgres": { + "strategy": { + "matrix": { + "include": [ + {"test-path": "tests/proxy_behavior"}, + {"test-path": "tests/integration/database/test_roi_observed.py"}, + ] + } + } } } } @@ -194,15 +190,27 @@ def test_circleci_postgres_suites_count_their_test_path_as_invoked() -> None: ) -def test_every_circleci_postgres_suite_is_credited_on_the_repo_as_it_stands() -> None: - circle: Final = yaml.safe_load(coverage.CIRCLECI_CONFIG.read_text()) - paths: Final = tuple( - job["postgres_suite"]["test_path"] - for job in circle["workflows"]["integration"]["jobs"] - if isinstance(job, dict) and "postgres_suite" in job +def test_allowlisting_an_unrun_integration_contract_does_not_claim_it_runs(tmp_path: Path) -> None: + path: Final = "tests/integration/database/test_roi_observed.py" + _write_integration_tree(tmp_path, path) + paths, findings = coverage._integration_ownership(tmp_path) + assert path not in paths + assert [(finding.subject, finding.detail) for finding in findings] == [ + (path, "GitHub-owned integration contract has no invoking job") + ] + allowlist: Final = coverage.Allowlist( + test_paths=(coverage.AllowEntry(paths=(path,), reason="ran only in CircleCI, removed"),), dockerfiles=() ) - assert paths == ("tests/integration/database/test_roi_observed.py",) - assert set(paths) <= coverage._invoked_test_tokens(coverage._all_scalars()) + allowed_paths, allowed_findings = coverage._integration_ownership(tmp_path, allowlist=allowlist) + assert path not in allowed_paths + assert allowed_findings == () + (tmp_path / ".github/workflows/test-unit.yml").write_text( + yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": f"pytest {path}"}]}}}) + ) + _, wrong_owner = coverage._integration_ownership(tmp_path, allowlist=allowlist) + assert [(finding.subject, finding.detail) for finding in wrong_owner] == [ + (path, "GitHub-owned integration contract runs outside a dedicated integration workflow") + ] def test_an_ancestor_directory_covers_a_file_but_does_not_name_it(): @@ -419,7 +427,7 @@ def test_an_excluded_term_matching_only_an_inner_name_leaves_the_file_claimed(): assert slice_.claims("tests/x/test_router.py", frozenset({"test_cache_key"})) is True -def test_character_class_globs_match_the_letter_shards_circleci_uses(): +def test_character_class_globs_match_letter_shards(): # tests/local_testing is split by first letter; without character-class support every # file in it looks unglobbed, and the slice guard would report the whole directory. glob = "tests/local_testing/**/test_[a-mA-M]*.py" @@ -509,12 +517,15 @@ def test_a_workflow_that_names_a_file_clears_it_from_the_slice_check(): assert any(coverage._token_covers(token, "tests/unit/proxy/test_proxy_server_custom_logger.py") for token in named) -def test_the_slice_check_credits_only_workflows_never_the_circleci_config(): - named = coverage._workflow_named_tokens() - circleci_only = "tests/proxy_admin_ui_tests" - assert not any(coverage._token_covers(token, f"{circleci_only}/test_key_management.py") for token in named), ( - "a tree only CircleCI globs must not be credited to a workflow" - ) +def test_the_slice_check_ignores_configs_outside_the_workflow_directory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + workflows: Final = tmp_path / "workflows" + workflows.mkdir() + (workflows / "test.yml").write_text("jobs:\n test:\n steps:\n - run: pytest tests/selected\n") + (tmp_path / "ignored.yml").write_text("jobs:\n test:\n steps:\n - run: pytest tests/unrun\n") + monkeypatch.setattr(coverage, "WORKFLOW_DIR", workflows) + assert coverage._workflow_named_tokens() == frozenset({"tests/selected"}) @pytest.mark.parametrize("selector", ("test_selected.py", "test_selected.py::test_redis_auth")) @@ -527,7 +538,6 @@ def test_a_workflow_does_not_credit_a_file_it_never_names( f"jobs:\n test:\n steps:\n - run: uv run pytest tests/local_testing/{selector}\n" ) monkeypatch.setattr(coverage, "WORKFLOW_DIR", workflows) - monkeypatch.setattr(coverage, "CIRCLECI_CONFIG", tmp_path / "circleci.yml") named: Final = coverage._workflow_named_tokens() assert named == frozenset({"tests/local_testing/test_selected.py"}) @@ -543,16 +553,14 @@ def test_slice_audit_rechecks_changed_and_new_files_between_calls( workflows.mkdir(parents=True) tests: Final = tmp_path / "tests/local_testing" tests.mkdir(parents=True) - circle: Final = tmp_path / ".circleci/config.yml" - circle.parent.mkdir() - circle.write_text( + (workflows / "test.yml").write_text( yaml.safe_dump( { "jobs": { selector: { "steps": [ { - "run": f'circleci tests glob "tests/local_testing/test_*.py" | xargs pytest -k "{selector}"' + "run": f'pytest tests/local_testing/test_*.py -k "{selector}"' } ] } @@ -564,7 +572,6 @@ def test_slice_audit_rechecks_changed_and_new_files_between_calls( monkeypatch.setattr(coverage, "REPO_ROOT", tmp_path) monkeypatch.setattr(coverage, "TESTS_ROOT", tmp_path / "tests") monkeypatch.setattr(coverage, "WORKFLOW_DIR", workflows) - monkeypatch.setattr(coverage, "CIRCLECI_CONFIG", circle) allowlist: Final = coverage.Allowlist((), ()) existing: Final = tests / "test_existing.py" existing.write_text("def test_selected(): pass\n") diff --git a/tests/unit/test_circleci_path_filter.py b/tests/unit/test_ci_path_filter.py similarity index 90% rename from tests/unit/test_circleci_path_filter.py rename to tests/unit/test_ci_path_filter.py index 9f695960517..1e110b62434 100644 --- a/tests/unit/test_circleci_path_filter.py +++ b/tests/unit/test_ci_path_filter.py @@ -1,6 +1,6 @@ -"""Regression tests for CircleCI change-based job gating. +"""Regression tests for change-based job gating. -`.circleci/scripts/classify_changes.sh` is the pure decision function behind +`.github/scripts/ci/classify_changes.sh` is the pure decision function behind `path_filter.sh`: given the list of files a PR changed (on stdin) and a job category, it prints `run` or `skip`. The gating contract we lock in here: @@ -24,7 +24,7 @@ from pathlib import Path import pytest -SCRIPTS_DIR = Path(__file__).resolve().parents[2] / ".circleci" / "scripts" +SCRIPTS_DIR = Path(__file__).resolve().parents[2] / ".github" / "scripts" / "ci" SCRIPT = SCRIPTS_DIR / "classify_changes.sh" PATH_FILTER = SCRIPTS_DIR / "path_filter.sh" @@ -56,7 +56,7 @@ CI = [".github/workflows/test-unit.yml"] ("mcp-dependencies", ["litellm-proxy-extras/pyproject.toml"], "run"), ("mcp-dependencies", ["scripts/check_mcp_sdk_install.py"], "run"), ("mcp-dependencies", [".github/workflows/test-mcp-dependency-resolution.yml"], "run"), - ("mcp-dependencies", [".circleci/scripts/classify_changes.sh"], "run"), + ("mcp-dependencies", [".github/scripts/ci/classify_changes.sh"], "run"), ("mcp-dependencies", ["litellm/llms/openai/chat/gpt_transformation.py"], "skip"), ("mcp-dependencies", DOCS + CLIENT, "skip"), ("windows-release", ["litellm-rust/crates/inference/src/lib.rs"], "run"), @@ -65,7 +65,7 @@ CI = [".github/workflows/test-unit.yml"] ("windows-release", ["pyproject.toml"], "run"), ("windows-release", ["uv.lock"], "run"), ("windows-release", ["tests/windows_tests/check_windows_wheel_install.py"], "run"), - ("windows-release", [".circleci/config.yml"], "run"), + ("windows-release", [".github/scripts/ci/classify_changes.sh"], "run"), ("windows-release", ["litellm/main.py"], "skip"), ("windows-release", ["tests/unit/test_utils.py"], "skip"), ("windows-release", ["ui/litellm-dashboard/src/App.tsx"], "skip"), @@ -126,7 +126,7 @@ def test_classify_decisions(category: str, changed: list[str], expected: str) -> (".github/workflows/test-unit.yml", "run"), ("tests/integration/run.py", "run"), ("uv.lock", "run"), - (".circleci/config.yml", "skip"), + (".github/workflows/test-integration.yml", "skip"), ("litellm/router.py", "skip"), ("docs/my-website/redis.md", "skip"), ), @@ -158,7 +158,7 @@ def test_markdown_under_dot_github_is_still_docs() -> None: def test_ui_and_client_diverge_on_a_backend_only_change() -> None: - """`client` gates CircleCI's dashboard end-to-end jobs, which drive a real + """`client` gates the dashboard end-to-end jobs, which drive a real proxy and so must run on backend changes. `ui` gates the dashboard build and its unit tests, which cannot see the backend at all.""" assert classify("client", BACKEND) == "run" @@ -202,16 +202,9 @@ def _pr_repo(tmp_path: Path, feature_files: dict[str, str]) -> Path: def _run_path_filter(work: Path, tmp_path: Path, category: str, scripts_dir: Path, is_pr: bool = True): - bin_dir = tmp_path / "bin" - bin_dir.mkdir(exist_ok=True) - stub = bin_dir / "circleci-agent" - stub.write_text("#!/usr/bin/env bash\necho \"[stub] circleci-agent $*\"\nexit 0\n") - stub.chmod(0o755) env = dict(os.environ) - env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" - env.pop("CIRCLE_PULL_REQUEST", None) - if is_pr: - env["CIRCLE_PULL_REQUEST"] = "https://github.com/x/y/pull/1" + env["GITHUB_EVENT_NAME"] = "pull_request" if is_pr else "push" + env["GITHUB_OUTPUT"] = str(tmp_path / "output") return subprocess.run( ["bash", str(scripts_dir / "path_filter.sh"), category], cwd=work, @@ -221,11 +214,12 @@ def _run_path_filter(work: Path, tmp_path: Path, category: str, scripts_dir: Pat ) -def test_path_filter_halts_docs_only_pr(tmp_path: Path) -> None: +def test_path_filter_skips_docs_only_pr(tmp_path: Path) -> None: work = _pr_repo(tmp_path, {"README.md": "# docs\n"}) result = _run_path_filter(work, tmp_path, "backend", SCRIPTS_DIR) assert result.returncode == 0 - assert "circleci-agent step halt" in result.stdout + assert "decision=skip" in result.stdout + assert (tmp_path / "output").read_text() == "decision=skip\n" def test_path_filter_runs_backend_pr(tmp_path: Path) -> None: diff --git a/tests/unit/test_circleci_checkout.py b/tests/unit/test_circleci_checkout.py deleted file mode 100644 index 347024838f7..00000000000 --- a/tests/unit/test_circleci_checkout.py +++ /dev/null @@ -1,310 +0,0 @@ -import os -import shutil -import subprocess -from collections.abc import Iterator, Mapping -from dataclasses import dataclass -from pathlib import Path -from typing import Final - -import pytest -import yaml -from pydantic import BaseModel, ConfigDict - -CIRCLECI: Final = Path(__file__).resolve().parents[2] / ".circleci" -CONFIG: Final = CIRCLECI / "config.yml" -PUBLIC_REPO: Final = "https://github.com/BerriAI/litellm.git" -LINUX_SHELL: Final = ("bash", "-eo", "pipefail", "-c") -WINDOWS_SHELL: Final = ("bash", "-c") -PRODUCT_DIRS: Final = ("litellm", "enterprise", "litellm-proxy-extras") - - -class _Run(BaseModel): - model_config = ConfigDict(frozen=True) - command: str - - -class _Step(BaseModel): - model_config = ConfigDict(frozen=True) - run: _Run - - -class _Command(BaseModel): - model_config = ConfigDict(frozen=True) - steps: tuple[_Step] - - -class _Commands(BaseModel): - model_config = ConfigDict(frozen=True) - checkout_over_https: _Command - - -class _NamedRun(BaseModel): - model_config = ConfigDict(frozen=True) - name: str = "" - command: str - - -class _JobStep(BaseModel): - model_config = ConfigDict(frozen=True) - run: _NamedRun | str | None = None - - -class _Job(BaseModel): - model_config = ConfigDict(frozen=True) - steps: tuple[_JobStep | str, ...] - - -class _Jobs(BaseModel): - model_config = ConfigDict(frozen=True) - routing_parity: _Job - - -class _Config(BaseModel): - model_config = ConfigDict(frozen=True) - commands: _Commands - jobs: _Jobs - - -class _TraceEvent(BaseModel): - model_config = ConfigDict(frozen=True) - event: str - name: str = "" - - -@dataclass(frozen=True, slots=True) -class _Upstream: - url: str - older: str - newer: str - - -def _step_name(step: object) -> str: - if isinstance(step, str): - return step - if isinstance(step, dict) and len(step) == 1: - return str(next(iter(step))) - return "" - - -def _step_names(node: object) -> Iterator[str]: - if isinstance(node, dict): - for key, value in node.items(): - if key in ("steps", "pre-steps", "post-steps") and isinstance(value, list): - yield from map(_step_name, value) - yield from _step_names(value) - if isinstance(node, list): - for item in node: - yield from _step_names(item) - - -def _config() -> _Config: - return _Config.model_validate(yaml.safe_load(CONFIG.read_text())) - - -def _routing_parity_step(name: str) -> str: - return next( - step.run.command - for step in _config().jobs.routing_parity.steps - if isinstance(step, _JobStep) and isinstance(step.run, _NamedRun) and step.run.name == name - ) - - -def _fetches(trace: Path) -> int: - events: Final = tuple(map(_TraceEvent.model_validate_json, trace.read_text().splitlines())) - return sum(event.event == "cmd_name" and event.name == "fetch" for event in events) - - -def _git(cwd: Path, *args: str) -> str: - return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout.strip() - - -def _commit(work: Path, files: Mapping[str, str], message: str) -> str: - for name, content in files.items(): - (work / name).parent.mkdir(parents=True, exist_ok=True) - (work / name).write_text(content) - _git(work, "add", "-A") - _git(work, "commit", "-q", "--allow-empty", "-m", message) - return _git(work, "rev-parse", "HEAD") - - -def _new_upstream(tmp_path: Path) -> tuple[Path, Path]: - remote: Final = tmp_path / "remote.git" - work: Final = tmp_path / "work" - work.mkdir() - _git(tmp_path, "init", "-q", "--bare", str(remote)) - _git(remote, "config", "uploadpack.allowFilter", "true") - _git(work, "init", "-q", "-b", "main") - _git(work, "config", "user.email", "t@t") - _git(work, "config", "user.name", "t") - return remote, work - - -def _publish_two_commits(tmp_path: Path) -> _Upstream: - remote, work = _new_upstream(tmp_path) - older: Final = _commit(work, {"app.py": "x = 1\n"}, "pipeline commit") - newer: Final = _commit(work, {"app.py": "x = 2\n"}, "main moved on") - _git(work, "push", "-q", str(remote), "main") - return _Upstream(url=remote.as_uri(), older=older, newer=newer) - - -def _publish_docs_only_pr(tmp_path: Path) -> _Upstream: - remote, work = _new_upstream(tmp_path) - main: Final = _commit(work, {"litellm_inference.py": "x\n"}, "base") - _git(work, "checkout", "-q", "-b", "litellm_feature") - head: Final = _commit(work, {"README.md": "# docs\n"}, "docs") - _git(work, "push", "-q", str(remote), "main", "litellm_feature") - return _Upstream(url=remote.as_uri(), older=main, newer=head) - - -def _product(version: str) -> dict[str, str]: - return {f"{directory}/code.py": f"{version} {directory}\n" for directory in PRODUCT_DIRS} - - -def _publish_parity_base(tmp_path: Path, base_on_main: bool) -> _Upstream: - remote, work = _new_upstream(tmp_path) - fork_point: Final = _commit(work, _product("fork point"), "fork point") - base: Final = _commit(work, _product("base"), "release") - _git(work, "branch", "-q", "stable") - _git(work, "reset", "-q", "--hard", base if base_on_main else fork_point) - head: Final = _commit(work, _product("head"), "pipeline commit") - _git(work, "push", "-q", str(remote), "main", "stable") - return _Upstream(url=remote.as_uri(), older=base, newer=head) - - -def _stub(bin_dir: Path, name: str, script: str) -> None: - bin_dir.mkdir(exist_ok=True) - (bin_dir / name).write_text(f"#!/usr/bin/env bash\n{script}\n") - (bin_dir / name).chmod(0o755) - - -def _job_env(tmp_path: Path, upstream: _Upstream) -> dict[str, str]: - return { - "PATH": f"{tmp_path / 'bin'}{os.pathsep}{os.environ['PATH']}", - "HOME": str(tmp_path), - "GIT_CONFIG_NOSYSTEM": "1", - "GIT_TERMINAL_PROMPT": "0", - "GIT_SSH_COMMAND": "false", - "GIT_CONFIG_COUNT": "1", - "GIT_CONFIG_KEY_0": f"url.{upstream.url}.insteadOf", - "GIT_CONFIG_VALUE_0": PUBLIC_REPO, - } - - -def _run_checkout( - tmp_path: Path, - upstream: _Upstream, - revisions: Mapping[str, str], - shell: tuple[str, ...] = LINUX_SHELL, -) -> subprocess.CompletedProcess[str]: - job: Final = tmp_path / "job" - job.mkdir() - command: Final = _config().commands.checkout_over_https - return subprocess.run( - [*shell, command.steps[0].run.command], - cwd=job, - env={**_job_env(tmp_path, upstream), **revisions}, - capture_output=True, - text=True, - ) - - -def test_every_clone_goes_over_https_so_no_job_needs_the_ssh_checkout_key() -> None: - names: Final = tuple(_step_names(yaml.safe_load(CONFIG.read_text()))) - assert "checkout_over_https" in names - assert "checkout" not in names - - -@pytest.mark.parametrize( - "revisions", - ( - {"CIRCLE_SHA1": "older"}, - {"CIRCLE_SHA1": "older", "REQUESTED_REVISION": ""}, - {"CIRCLE_SHA1": "newer", "REQUESTED_REVISION": "older"}, - ), -) -def test_a_job_checks_out_its_own_commit_from_the_public_https_url_after_main_moves( - tmp_path: Path, revisions: Mapping[str, str] -) -> None: - upstream: Final = _publish_two_commits(tmp_path) - shas: Final = {"older": upstream.older, "newer": upstream.newer, "": ""} - result: Final = _run_checkout(tmp_path, upstream, {name: shas[value] for name, value in revisions.items()}) - assert result.returncode == 0, result.stderr - assert _git(tmp_path / "job", "rev-parse", "HEAD") == upstream.older - - -@pytest.mark.parametrize("shell", (LINUX_SHELL, WINDOWS_SHELL), ids=("linux", "windows-bash-exe")) -def test_a_revision_that_is_not_a_full_sha_is_refused_before_git_runs(tmp_path: Path, shell: tuple[str, ...]) -> None: - upstream: Final = _publish_two_commits(tmp_path) - result: Final = _run_checkout( - tmp_path, upstream, {"CIRCLE_SHA1": upstream.newer, "REQUESTED_REVISION": "main"}, shell - ) - assert result.returncode != 0 - assert not (tmp_path / "job" / ".git").exists() - - -def test_a_docs_only_pr_job_still_skips_backend_work_after_the_checkout(tmp_path: Path) -> None: - upstream: Final = _publish_docs_only_pr(tmp_path) - _stub(tmp_path / "bin", "circleci-agent", 'echo "circleci-agent $*"') - checkout: Final = _run_checkout(tmp_path, upstream, {"CIRCLE_SHA1": upstream.newer}) - assert checkout.returncode == 0, checkout.stderr - path_filter: Final = subprocess.run( - ["bash", str(CIRCLECI / "scripts" / "path_filter.sh"), "backend"], - cwd=tmp_path / "job", - env={**_job_env(tmp_path, upstream), "CIRCLE_PULL_REQUEST": "https://github.com/BerriAI/litellm/pull/1"}, - capture_output=True, - text=True, - ) - assert path_filter.returncode == 0, path_filter.stderr - assert "circleci-agent step halt" in path_filter.stdout - - -@pytest.mark.parametrize("shell", (LINUX_SHELL, WINDOWS_SHELL), ids=("linux", "windows-bash-exe")) -@pytest.mark.parametrize( - ("failing_step", "failures", "checkouts", "checked_out"), - (("fetch", 2, 1, True), ("fetch", 3, 0, False), ("checkout", 2, 3, True), ("checkout", 3, 3, False)), -) -def test_a_failed_fetch_or_blob_download_is_retried_and_the_step_stops_on_its_error_once_retries_run_out( - tmp_path: Path, shell: tuple[str, ...], failing_step: str, failures: int, checkouts: int, checked_out: bool -) -> None: - upstream: Final = _publish_two_commits(tmp_path) - calls: Final = tmp_path / "calls" - _stub(tmp_path / "bin", "sleep", f'echo sleep >> "{calls}"') - _stub( - tmp_path / "bin", - "git", - f'''echo "$1" >> "{calls}" -if [ "$1" = {failing_step} ] && [ "$(grep -c '^{failing_step}$' "{calls}")" -le {failures} ]; then exit 128; fi -exec "{shutil.which("git")}" "$@"''', - ) - result: Final = _run_checkout(tmp_path, upstream, {"CIRCLE_SHA1": upstream.older}, shell) - assert (result.returncode == 0) is checked_out, result.stderr - steps: Final = calls.read_text().splitlines() - assert steps.count("fetch") == min(failures + 1, 3) - assert steps.count("sleep") == steps.count("fetch") - 1 - assert steps.count("checkout") == checkouts - head: Final = subprocess.run(["git", "rev-parse", "HEAD"], cwd=tmp_path / "job", capture_output=True, text=True) - assert (head.stdout.strip() == upstream.older) is checked_out - - -@pytest.mark.parametrize("base_on_main", (True, False), ids=("main-commit", "release-branch-commit")) -def test_routing_parity_downloads_the_base_files_in_one_fetch_instead_of_one_per_missing_file( - tmp_path: Path, base_on_main: bool -) -> None: - upstream: Final = _publish_parity_base(tmp_path, base_on_main) - checkout: Final = _run_checkout(tmp_path, upstream, {"CIRCLE_SHA1": upstream.newer}) - assert checkout.returncode == 0, checkout.stderr - job: Final = tmp_path / "job" - (job / "litellm" / "rust_bridge").mkdir() - (job / "litellm" / "rust_bridge" / "_native.abi3.so").write_text("built for this job\n") - trace: Final = tmp_path / "trace.json" - result: Final = subprocess.run( - [*LINUX_SHELL, _routing_parity_step("Check out base product code")], - cwd=job, - env={**_job_env(tmp_path, upstream), "ROUTING_PARITY_BASE": upstream.older, "GIT_TRACE2_EVENT": str(trace)}, - capture_output=True, - text=True, - ) - assert result.returncode == 0, result.stderr - assert tuple((job / name).read_text() for name in _product("base")) == tuple(_product("base").values()) - assert _fetches(trace) == 1 - assert _git(job, "rev-parse", "HEAD") == upstream.newer diff --git a/tests/unit/test_circleci_run_migration_tests.py b/tests/unit/test_circleci_run_migration_tests.py deleted file mode 100644 index 30fe7383c07..00000000000 --- a/tests/unit/test_circleci_run_migration_tests.py +++ /dev/null @@ -1,36 +0,0 @@ -import importlib.util -from pathlib import Path -from typing import Final - -import pytest - -SCRIPT: Final = Path(__file__).resolve().parents[2] / ".circleci/scripts/run_migration_tests.py" -SPEC: Final = importlib.util.spec_from_file_location("migration_ci", SCRIPT) -assert SPEC is not None and SPEC.loader is not None -MODULE: Final = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(MODULE) - - -@pytest.mark.parametrize( - "xml,expected,exit_code,passed", - ( - ('', 2, 0, True), - ('', 2, 0, False), - ("", 1, 0, False), - ("", 1, 0, False), - ("", 1, 0, False), - ("", 1, 1, False), - ("", 1, 5, False), - ("", 1, 0, False), - ("', 2, 0, False), - ), -) -def test_only_a_complete_passing_suite_can_certify_an_image( - tmp_path: Path, xml: str | None, expected: int, exit_code: int, passed: bool -) -> None: - path: Final = tmp_path / "results.xml" - if xml is not None: - path.write_text(xml) - assert MODULE.successful_junit(path, expected, exit_code) is passed diff --git a/tests/unit/test_circleci_rust_toolchain.py b/tests/unit/test_circleci_rust_toolchain.py deleted file mode 100644 index 562fb5db032..00000000000 --- a/tests/unit/test_circleci_rust_toolchain.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Static guardrails for how CircleCI provisions Rust. - -The root package builds `litellm-rust` through maturin, so any job that runs -`uv sync` or `uv build` compiles the bridge. The `cimg/python` images ship no -Rust toolchain, and when cargo is missing maturin's `puccinialin` helper -quietly provisions one itself: it fetches `rustup-init` from the unversioned -`https://static.rust-lang.org/rustup/dist//` path with no checksum and -installs a floating `stable` toolchain. uv suppresses build-backend output on a -successful sync, so that happens with nothing in the job log to show for it, -and the compiler a job builds with changes whenever upstream publishes. - -Two invariants are pinned here: - - 1. No step list (job or reusable command) reaches a `uv sync` / `uv build` - without a Rust toolchain already provisioned ahead of it. That is the - `install_rust` command on Linux and `install_windows_toolchain` on Windows, - so the check accepts any command or step that installs a pinned rustup. A new job that syncs without one - falls back to the unpinned path, which is exactly the regression a static - check catches at PR time and a green CI run does not. - 2. Both installers pin what they download: an explicit rustup version, a - verified SHA-256, and the exact toolchain in `rust-toolchain.toml`. -""" - -from __future__ import annotations - -import re -from pathlib import Path -from typing import Final - -import pytest -import yaml - -REPO_ROOT = Path(__file__).resolve().parents[2] -CONFIG = REPO_ROOT / ".circleci" / "config.yml" -TOOLCHAIN: Final = REPO_ROOT / "rust-toolchain.toml" - -BUILDS_WORKSPACE = re.compile(r"\buv\s+(?:sync|build)\b") -RUSTUP_ARCHIVE_URL = re.compile(r"https://static\.rust-lang\.org/rustup/archive/\d+\.\d+\.\d+/") -EXACT_TOOLCHAIN = re.compile(r"--default-toolchain\s+\"?(\d+\.\d+\.\d+)\"?") -TOOLCHAIN_CHANNEL: Final = re.compile(r'^channel = "(\d+\.\d+\.\d+)"$', re.MULTILINE) - - -def _config() -> dict[str, object]: - return yaml.safe_load(CONFIG.read_text()) - - -def _step_text(step: object) -> str: - """Flatten one step into the shell text it runs, or '' for a command reference.""" - if isinstance(step, dict): - run = step.get("run") - if isinstance(run, str): - return run - if isinstance(run, dict): - command = run.get("command") - return command if isinstance(command, str) else "" - return "" - - -def _pinned_toolchain() -> str: - match: Final = TOOLCHAIN_CHANNEL.search(TOOLCHAIN.read_text()) - assert match is not None, "rust-toolchain.toml must pin an exact channel" - return match.group(1) - - -def _without_comments(text: str) -> str: - return "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#")) - - -def _installs_pinned_rustup(step: object) -> bool: - text = _step_text(step) - return "rustup-init" in text and ("sha256sum" in text or "SHA256" in text) - - -def _provisioning_commands() -> frozenset[str]: - return frozenset( - name.removeprefix("command ") - for name, steps in _step_lists().items() - if name.startswith("command ") and any(_installs_pinned_rustup(step) for step in steps) - ) - - -def _provisions_rust(step: object, provisioning_commands: frozenset[str]) -> bool: - return (isinstance(step, str) and step in provisioning_commands) or _installs_pinned_rustup(step) - - -def _step_lists() -> dict[str, list[object]]: - config = _config() - lists: dict[str, list[object]] = {} - for kind in ("jobs", "commands"): - section = config.get(kind) - if not isinstance(section, dict): - continue - for name, body in section.items(): - steps = body.get("steps") if isinstance(body, dict) else None - if isinstance(steps, list): - lists[f"{kind[:-1]} {name}"] = steps - return lists - - -def _first_unprovisioned_build(steps: list[object], provisioning_commands: frozenset[str]) -> str | None: - """Return the shell text of the first workspace build reached without Rust, if any.""" - rust_ready = False - for step in steps: - if _provisions_rust(step, provisioning_commands): - rust_ready = True - text = _step_text(step) - if BUILDS_WORKSPACE.search(_without_comments(text)) and not rust_ready: - return text - return None - - -def test_step_lists_exist() -> None: - lists = _step_lists() - assert "command install_rust" in lists - building = { - name - for name, steps in lists.items() - if any(BUILDS_WORKSPACE.search(_without_comments(_step_text(s))) for s in steps) - } - assert "command setup_litellm_test_deps" in building, f"found {sorted(building)}" - - -def test_no_workspace_build_without_a_provisioned_rust_toolchain() -> None: - provisioning_commands: Final = _provisioning_commands() - assert {"install_rust", "install_windows_toolchain"} <= provisioning_commands - offenders = { - name: build - for name, steps in _step_lists().items() - if (build := _first_unprovisioned_build(steps, provisioning_commands)) is not None - } - assert not offenders, ( - "these CircleCI step lists run `uv sync`/`uv build` with no Rust toolchain provisioned first, " - "so maturin will download an unpinned rustup and a floating toolchain instead: " - f"{ {name: build.strip().splitlines()[0] for name, build in offenders.items()} }" - ) - - -@pytest.fixture(name="install_rust_command") -def _install_rust_command() -> str: - steps = _step_lists()["command install_rust"] - return "\n".join(_step_text(step) for step in steps) - - -def test_install_rust_pins_the_rustup_version_in_the_url(install_rust_command: str) -> None: - assert RUSTUP_ARCHIVE_URL.search(install_rust_command), ( - "install_rust must download rustup-init from a version-pinned /rustup/archive// URL; " - "the /rustup/dist/ path always serves whatever rustup is current" - ) - assert "/rustup/dist/" not in install_rust_command - - -def test_install_rust_verifies_the_installer_checksum(install_rust_command: str) -> None: - assert "sha256sum -c" in install_rust_command - assert re.search(r"RUSTUP_SHA256=[0-9a-f]{64}\b", install_rust_command), ( - "install_rust must compare the downloaded installer against a hardcoded SHA-256 " - "taken from rust-lang's published .sha256 sidecar" - ) - checksum_index = install_rust_command.index("sha256sum -c") - execute_index = install_rust_command.index("/tmp/rustup-init -y") - assert checksum_index < execute_index, "the checksum must be verified before the installer is executed" - - -def test_install_rust_pins_an_exact_toolchain_version(install_rust_command: str) -> None: - match: Final = EXACT_TOOLCHAIN.search(install_rust_command) - assert match is not None, ( - "install_rust must pin an exact toolchain version (e.g. 1.98.0); a channel name like " - "stable/beta/nightly makes the compiler drift with whatever upstream published that day" - ) - assert match.group(1) == _pinned_toolchain() - - -def test_windows_installer_matches_the_repo_toolchain() -> None: - windows_steps: Final = _step_lists()["command install_windows_toolchain"] - windows_command: Final = "\n".join(_step_text(step) for step in windows_steps) - match: Final = EXACT_TOOLCHAIN.search(windows_command) - assert match is not None - assert match.group(1) == _pinned_toolchain() diff --git a/tests/unit/test_cost_map_guard.py b/tests/unit/test_cost_map_guard.py index 2602149b623..ae3f2f6caa0 100644 --- a/tests/unit/test_cost_map_guard.py +++ b/tests/unit/test_cost_map_guard.py @@ -355,6 +355,35 @@ def test_checked_in_pins_hold_in_the_checked_in_cost_map() -> None: assert guard.pin_failures(pins, cost_map) == () +@pytest.mark.parametrize("pins_path", guard.PINS_PATHS) +@pytest.mark.parametrize("pins", (PINS, "", "not JSON")) +def test_relocated_pins_keep_the_same_validation(pins_path: str, pins: str) -> None: + cost_map: Final = _serialize(BASE_MAP) + blobs: Final = { + f"base:{guard.COST_MAP_PATH}": cost_map, + f"base:{guard.BACKUP_PATH}": cost_map, + f"head:{guard.COST_MAP_PATH}": cost_map, + f"head:{guard.BACKUP_PATH}": cost_map, + f"base:{guard.PINS_PATH}": PINS, + f"head:{pins_path}": pins, + } + + def read_git(_operation: str, blob: str) -> str | None: + return blobs.get(blob) + + base: Final = guard.snapshot("base", read_git=read_git) + head: Final = guard.snapshot("head", read_git=read_git) + assert base == _snapshot(BASE_MAP, pins=PINS) + assert head == _snapshot(BASE_MAP, pins=pins) + failures: Final = _failures(head, changed_files=(guard.PINS_PATH, pins_path), bot=False, base=base) + assert bool(failures) is (pins != PINS), failures + + +@pytest.mark.parametrize("pins_path", guard.PINS_PATHS) +def test_relocated_pins_cannot_be_changed_by_bots(pins_path: str) -> None: + assert guard.pin_author_failures("litellm_fix_pricing", "Bot", (pins_path,)) == (BOT_AUTHOR_PIN_FAILURE,) + + def _commit(repo: Path, cost_map: dict[str, object], message: str, pins: str | None = None) -> str: text = _serialize(cost_map) pins_file = repo / guard.PINS_PATH diff --git a/tests/unit/test_detect_changes.py b/tests/unit/test_detect_changes.py index 44a4bfdf448..8f41d6498f0 100644 --- a/tests/unit/test_detect_changes.py +++ b/tests/unit/test_detect_changes.py @@ -21,7 +21,7 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] SCRIPT = REPO_ROOT / ".github" / "scripts" / "detect_changes.sh" -CLASSIFIER = REPO_ROOT / ".circleci" / "scripts" / "classify_changes.sh" +CLASSIFIER = REPO_ROOT / ".github" / "scripts" / "ci" / "classify_changes.sh" UI_FILE = "ui/litellm-dashboard/src/components/Teams.tsx" BACKEND_FILE = "litellm/proxy/proxy_server.py" @@ -65,9 +65,9 @@ def _scripts_tree(tmp_path: Path, classifier_body: str | None = None) -> Path: """Copy the scripts into a throwaway tree, preserving their relative layout.""" root = tmp_path / "tree" (root / ".github" / "scripts").mkdir(parents=True) - (root / ".circleci" / "scripts").mkdir(parents=True) + (root / ".github" / "scripts" / "ci").mkdir(parents=True) shutil.copy(SCRIPT, root / ".github" / "scripts" / SCRIPT.name) - target = root / ".circleci" / "scripts" / CLASSIFIER.name + target = root / ".github" / "scripts" / "ci" / CLASSIFIER.name if classifier_body is None: shutil.copy(CLASSIFIER, target) else: diff --git a/tests/unit/test_vcr_conftest_common_banner.py b/tests/unit/test_vcr_conftest_common_banner.py index 30a65abba1d..515d031afa9 100644 --- a/tests/unit/test_vcr_conftest_common_banner.py +++ b/tests/unit/test_vcr_conftest_common_banner.py @@ -145,7 +145,7 @@ def test_banner_silent_when_vcr_disabled(monkeypatch, health_reset, patch_capaci # --------------------------------------------------------------------------- -# Diagnostic-log dedup + cap. CircleCI truncates step output to the last +# Diagnostic-log dedup + cap. GitHub Actions truncates step output to the last # ~400 KB; an unbounded diagnostic dump pushes the VCR classification summary # out of the retrievable window, so the dump must dedupe and cap. # ---------------------------------------------------------------------------