From 742e0353ad4c6f81d8c5311e91a13bdaed057b19 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 27 Mar 2026 06:36:11 -0700 Subject: [PATCH] feat: add github cosign for releases --- .github/workflows/publish_to_pypi.yml | 56 +++++++ .github/workflows/release_docker.yml | 149 ++++++++++++++++++ .../docs/proxy/docker_quick_start.md | 53 +++++++ docs/my-website/docs/proxy/release_cycle.md | 1 + 4 files changed, 259 insertions(+) create mode 100644 .github/workflows/release_docker.yml diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml index 8f675bb3075..4f6c9bf7949 100644 --- a/.github/workflows/publish_to_pypi.yml +++ b/.github/workflows/publish_to_pypi.yml @@ -134,3 +134,59 @@ jobs: - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + + - name: Sign artifacts with sigstore + run: | + pip install sigstore==4.2.0 + python -m sigstore sign dist/* + + - name: Upload sigstore bundles + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: sigstore-bundles-${{ needs.preflight-checks.outputs.version }} + path: dist/*.sigstore.json + if-no-files-found: error + + verify-pypi-signatures: + name: Verify PyPI signatures + needs: [preflight-checks, publish-litellm] + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + + steps: + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Download sigstore bundles + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: sigstore-bundles-${{ needs.preflight-checks.outputs.version }} + path: verify/ + + - name: Download published package from PyPI + run: | + pip download --no-deps --no-binary :all: \ + litellm==${{ needs.preflight-checks.outputs.version }} \ + -d verify/ + pip download --no-deps --only-binary :all: \ + litellm==${{ needs.preflight-checks.outputs.version }} \ + -d verify/ + + - name: Verify signatures + run: | + pip install sigstore==4.2.0 + cd verify + for artifact in *.tar.gz *.whl; do + [ -f "$artifact" ] || continue + echo "Verifying $artifact ..." + python -m sigstore verify identity \ + --bundle "${artifact}.sigstore.json" \ + --cert-identity "https://github.com/BerriAI/litellm/.github/workflows/publish_to_pypi.yml@refs/heads/main" \ + --cert-oidc-issuer "https://token.actions.githubusercontent.com" \ + "$artifact" + echo "OK: $artifact verified" + done diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml new file mode 100644 index 00000000000..8cebe6a4cbb --- /dev/null +++ b/.github/workflows/release_docker.yml @@ -0,0 +1,149 @@ +name: Release Docker Images + +on: + workflow_dispatch: + inputs: + tag: + description: "Image tag (e.g. v1.83.0-nightly)" + required: true + type: string + commit_hash: + description: "Commit SHA to build from" + required: true + type: string + +concurrency: + group: release-docker-${{ github.event.inputs.tag }} + cancel-in-progress: false + +permissions: {} + +jobs: + build-and-push: + name: Build ${{ matrix.image }} + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + packages: write + contents: read + id-token: write + strategy: + fail-fast: false + matrix: + include: + - image: litellm + dockerfile: Dockerfile + - image: litellm-database + dockerfile: docker/Dockerfile.database + - image: litellm-non_root + dockerfile: docker/Dockerfile.non_root + steps: + - name: Checkout repo + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + ref: ${{ github.event.inputs.commit_hash }} + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 + with: + images: ghcr.io/berriai/${{ matrix.image }} + tags: | + type=raw,value=${{ github.event.inputs.tag }} + type=raw,value=main-latest + + - name: Build and push + id: push + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + with: + context: . + file: ${{ matrix.dockerfile }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Install cosign + uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v4.1.1 + + - name: Sign image with cosign (keyless) + env: + DIGEST: ${{ steps.push.outputs.digest }} + IMAGE: ghcr.io/berriai/${{ matrix.image }} + run: cosign sign --yes "${IMAGE}@${DIGEST}" + + - name: Generate SBOM + uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0 + with: + image: ghcr.io/berriai/${{ matrix.image }}@${{ steps.push.outputs.digest }} + format: spdx-json + output-file: sbom-${{ matrix.image }}.spdx.json + + - name: Attest SBOM with cosign + env: + DIGEST: ${{ steps.push.outputs.digest }} + IMAGE: ghcr.io/berriai/${{ matrix.image }} + run: | + cosign attest --yes \ + --predicate sbom-${{ matrix.image }}.spdx.json \ + --type spdxjson \ + "${IMAGE}@${DIGEST}" + + - name: Upload SBOM as artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: sbom-${{ matrix.image }} + path: sbom-${{ matrix.image }}.spdx.json + + verify-signatures: + name: Verify ${{ matrix.image }} signature + needs: build-and-push + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + packages: read + strategy: + fail-fast: false + matrix: + include: + - image: litellm + - image: litellm-database + - image: litellm-non_root + + steps: + - name: Install cosign + uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v4.1.1 + + - name: Log in to GHCR (read-only) + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Verify image signature + run: | + cosign verify \ + --certificate-identity-regexp "https://github.com/BerriAI/litellm/" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + "ghcr.io/berriai/${{ matrix.image }}:${{ github.event.inputs.tag }}" + + - name: Verify SBOM attestation + run: | + cosign verify-attestation \ + --certificate-identity-regexp "https://github.com/BerriAI/litellm/" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + --type spdxjson \ + "ghcr.io/berriai/${{ matrix.image }}:${{ github.event.inputs.tag }}" diff --git a/docs/my-website/docs/proxy/docker_quick_start.md b/docs/my-website/docs/proxy/docker_quick_start.md index 58a56604751..cf2a12d6633 100644 --- a/docs/my-website/docs/proxy/docker_quick_start.md +++ b/docs/my-website/docs/proxy/docker_quick_start.md @@ -865,6 +865,59 @@ LiteLLM Proxy uses the [LiteLLM Python SDK](https://docs.litellm.ai/docs/routing `litellm_settings` are module-level params for the LiteLLM Python SDK (equivalent to doing `litellm.` on the SDK). You can see all params [here](https://github.com/BerriAI/litellm/blob/208fe6cb90937f73e0def5c97ccb2359bf8a467b/litellm/__init__.py#L114) +## Verify Image & Package Signatures + +All official LiteLLM Docker images and PyPI packages are signed using [Sigstore](https://www.sigstore.dev/) keyless signing. This lets you cryptographically verify that an artifact was built by the BerriAI/litellm CI pipeline — not tampered with or uploaded by a compromised account. + +### Install cosign + +```bash +# macOS +brew install cosign + +# Linux +curl -fsSL https://github.com/sigstore/cosign/releases/latest/download/cosign-linux-amd64 -o /usr/local/bin/cosign +chmod +x /usr/local/bin/cosign +``` + +### Verify a Docker image + +```bash +cosign verify \ + --certificate-identity-regexp "https://github.com/BerriAI/litellm/" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + ghcr.io/berriai/litellm:main-latest +``` + +This also works for other image variants (`litellm-database`, `litellm-non_root`). + +### Verify SBOM attestation + +Each image includes a signed SBOM (Software Bill of Materials) attestation: + +```bash +cosign verify-attestation \ + --certificate-identity-regexp "https://github.com/BerriAI/litellm/" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + --type spdxjson \ + ghcr.io/berriai/litellm:main-latest +``` + +### Verify a PyPI package + +Download the `.sigstore.json` bundle from the [GitHub Actions workflow artifacts](https://github.com/BerriAI/litellm/actions/workflows/publish_to_pypi.yml), then: + +```bash +pip install sigstore +python -m sigstore verify identity \ + --bundle litellm-1.83.0.tar.gz.sigstore.json \ + --cert-identity "https://github.com/BerriAI/litellm/.github/workflows/publish_to_pypi.yml@refs/heads/main" \ + --cert-oidc-issuer "https://token.actions.githubusercontent.com" \ + litellm-1.83.0.tar.gz +``` + +If verification succeeds, you can be confident the artifact was built by the official BerriAI/litellm CI — not by a compromised credential or registry. + ## Support & Talk with founders - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) diff --git a/docs/my-website/docs/proxy/release_cycle.md b/docs/my-website/docs/proxy/release_cycle.md index b3e056b0243..fe1ecce3298 100644 --- a/docs/my-website/docs/proxy/release_cycle.md +++ b/docs/my-website/docs/proxy/release_cycle.md @@ -8,6 +8,7 @@ Litellm Proxy has the following release cycle: In production, we recommend using the latest `v1.x.x:main-stable` release. +All Docker images and PyPI packages are signed with [Sigstore](https://www.sigstore.dev/) keyless signing. See [Verify Image & Package Signatures](./docker_quick_start.md#verify-image--package-signatures) for instructions. Follow our release notes [here](https://github.com/BerriAI/litellm/releases).