From 8520626e7aecad5ebe33e66d1fd5e955b32085eb Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 3 Oct 2026 17:54:17 -0700 Subject: [PATCH] fix(lens): preserve approved worker digests and harden its image (#44467) * fix(lens): pin worker dependencies and support approved image digests * fix(lens): include locked dependencies and release identity in build context * fix(lens): select the dev worker package for SHA-tagged charts --- .github/workflows/image-scan.yml | 76 +++++++++- .github/workflows/lens-worker.yml | 4 +- deploy/lens/Dockerfile | 24 ++- deploy/lens/Dockerfile.dockerignore | 4 + deploy/lens/README.md | 9 +- deploy/lens/requirements.in | 2 + deploy/lens/requirements.lock | 172 ++++++++++++++++++++++ helm/litellm/templates/_helpers.tpl | 13 +- helm/litellm/tests/lens_worker_tests.yaml | 60 ++++++++ helm/litellm/values.yaml | 1 + litellm/proxy/lens/release.py | 3 +- tests/unit/proxy/lens/test_release.py | 7 + 12 files changed, 365 insertions(+), 10 deletions(-) create mode 100644 deploy/lens/requirements.in create mode 100644 deploy/lens/requirements.lock diff --git a/.github/workflows/image-scan.yml b/.github/workflows/image-scan.yml index 8fb11f40370..e72b8230232 100644 --- a/.github/workflows/image-scan.yml +++ b/.github/workflows/image-scan.yml @@ -16,7 +16,7 @@ on: - backend/Dockerfile - backend/main.py - deploy/lens/** - - litellm/proxy/lens/release.py + - litellm/proxy/lens/** - tests/e2e/migrations/lens_compose_smoke.sh - docker/component_entrypoint.sh - docker/entrypoint.sh @@ -40,6 +40,80 @@ concurrency: cancel-in-progress: true jobs: + lens-worker-image: + name: lens-worker-image (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + timeout-minutes: 15 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-latest + grype_sha256: edda0968d8827daab01d32b3cd7de192ae0915005e7bbfcfef9e68e79bc43343 + - arch: arm64 + runner: ubuntu-24.04-arm + grype_sha256: 553e4c36d9d61349830ba6034d43b8700a7f10576d3e2f4981c0fd2b96086465 + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + - name: Build the release worker + env: + RELEASE_TAG: sha-${{ github.sha }} + run: docker build --build-arg LITELLM_RELEASE_TAG="${RELEASE_TAG}" -f deploy/lens/Dockerfile -t lens-worker-scan . + - name: Verify the standalone worker on a read-only filesystem + env: + RELEASE_TAG: sha-${{ github.sha }} + run: | + docker run --rm --network none --read-only --cap-drop ALL \ + --tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \ + -e EXPECTED_RELEASE_TAG="${RELEASE_TAG}" --entrypoint python lens-worker-scan -c ' + import os + import lens.worker + from lens.release import release_tag + from lens.trace_store import trace_store + assert os.getuid() == 65532 + assert release_tag() == os.environ["EXPECTED_RELEASE_TAG"] + with trace_store() as store: + assert store.count() == 0 + ' + - name: Reject a dependency whose hash has changed + run: | + docker build --target builder -f deploy/lens/Dockerfile -t lens-worker-deps . + sed -E 's/sha256:[0-9a-f]{64}/sha256:0000000000000000000000000000000000000000000000000000000000000000/g' \ + deploy/lens/requirements.lock > "$RUNNER_TEMP/tampered.lock" + if docker run --rm -v "$RUNNER_TEMP/tampered.lock:/tmp/tampered.lock:ro" \ + --entrypoint uv lens-worker-deps pip sync --python /app/.venv/bin/python \ + --require-hashes --only-binary :all: --reinstall --no-cache /tmp/tampered.lock \ + > "$RUNNER_TEMP/hash-check.log" 2>&1; then + echo "::error::Dependency hash mismatch was accepted" + exit 1 + fi + cat "$RUNNER_TEMP/hash-check.log" + grep -qi 'hash mismatch' "$RUNNER_TEMP/hash-check.log" + - name: Download Grype v0.114.0 + env: + ARCH: ${{ matrix.arch }} + GRYPE_SHA256: ${{ matrix.grype_sha256 }} + run: | + curl -fsSL --retry 3 -o "$RUNNER_TEMP/grype.tar.gz" \ + "https://github.com/anchore/grype/releases/download/v0.114.0/grype_0.114.0_linux_${ARCH}.tar.gz" + echo "${GRYPE_SHA256} $RUNNER_TEMP/grype.tar.gz" | sha256sum -c - + tar xzf "$RUNNER_TEMP/grype.tar.gz" -C "$RUNNER_TEMP" grype + chmod +x "$RUNNER_TEMP/grype" + - name: Scan the worker for fixable HIGH/CRITICAL CVEs + env: + GRYPE_MATCH_PYTHON_USING_CPES: "true" + run: | + "$RUNNER_TEMP/grype" lens-worker-scan \ + --config .grype.yaml --only-fixed --fail-on high --output table + image-scan: name: image-scan runs-on: ubuntu-latest diff --git a/.github/workflows/lens-worker.yml b/.github/workflows/lens-worker.yml index e3c47f35c36..896a598decd 100644 --- a/.github/workflows/lens-worker.yml +++ b/.github/workflows/lens-worker.yml @@ -61,11 +61,11 @@ jobs: -v "$PWD/tests/proxy_behavior/lens/worker_storage_smoke.py:/app/storage_smoke.py:ro" \ --entrypoint python lens-worker:${{ github.sha }} /app/storage_smoke.py - name: Publish versioned Lens worker - if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' + if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' && github.ref == 'refs/heads/main' env: REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} REGISTRY_USER: ${{ github.actor }} - IMAGE: ghcr.io/berriai/litellm-lens-worker:sha-${{ github.sha }} + IMAGE: ghcr.io/berriai/litellm-lens-worker-dev:sha-${{ github.sha }} run: | printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io -u "$REGISTRY_USER" --password-stdin docker tag lens-worker:${{ github.sha }} "$IMAGE" diff --git a/deploy/lens/Dockerfile b/deploy/lens/Dockerfile index 3d7ddbe832f..84c45291c42 100644 --- a/deploy/lens/Dockerfile +++ b/deploy/lens/Dockerfile @@ -1,9 +1,27 @@ -FROM python:3.12-slim +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a + +FROM $UV_IMAGE AS uvbin + +FROM $LITELLM_BUILD_IMAGE AS builder +COPY --from=uvbin /uv /usr/local/bin/uv +RUN apk add --no-cache python-3.13 +ENV UV_PYTHON_DOWNLOADS=0 UV_LINK_MODE=copy +WORKDIR /app +COPY deploy/lens/requirements.lock /tmp/requirements.lock +RUN uv venv --python python3.13 /app/.venv && \ + uv pip sync --python /app/.venv/bin/python --require-hashes --only-binary :all: /tmp/requirements.lock + +FROM $LITELLM_RUNTIME_IMAGE AS runtime ARG LITELLM_RELEASE_TAG="" RUN : "${LITELLM_RELEASE_TAG:?Pass --build-arg LITELLM_RELEASE_TAG matching the gateway}" -ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} +RUN apk add --no-cache python-3.13 +ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} \ + PATH="/app/.venv/bin:${PATH}" \ + PYTHONDONTWRITEBYTECODE=1 WORKDIR /app -RUN pip install --no-cache-dir httpx==0.28.1 pydantic==2.11.7 +COPY --from=builder /app/.venv /app/.venv COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py litellm/proxy/lens/release.py /app/lens/ COPY litellm/proxy/lens/prompts/ /app/lens/prompts/ USER 65532:65532 diff --git a/deploy/lens/Dockerfile.dockerignore b/deploy/lens/Dockerfile.dockerignore index 70fe9c83b6d..801c89d9dfe 100644 --- a/deploy/lens/Dockerfile.dockerignore +++ b/deploy/lens/Dockerfile.dockerignore @@ -1,4 +1,7 @@ ** +!deploy/ +!deploy/lens/ +!deploy/lens/requirements.lock !litellm/ !litellm/proxy/ !litellm/proxy/lens/ @@ -7,5 +10,6 @@ !litellm/proxy/lens/trace_store.py !litellm/proxy/lens/analysis.py !litellm/proxy/lens/worker.py +!litellm/proxy/lens/release.py !litellm/proxy/lens/prompts/ !litellm/proxy/lens/prompts/** diff --git a/deploy/lens/README.md b/deploy/lens/README.md index afd2f409e21..1c67864adc4 100644 --- a/deploy/lens/README.md +++ b/deploy/lens/README.md @@ -53,7 +53,7 @@ lensWorker: key: token ``` -The worker image defaults to the chart's application version, and the chart connects it to the backend service. Keep these values and the Secret when upgrading the chart so the gateway and worker upgrade together. `lensWorker.replicaCount` controls simultaneous investigations. To use a private registry or external proxy, set `lensWorker.image.repository`, `lensWorker.image.tag`, and `lensWorker.url`. The dashboard uses the chart's worker image for standalone install commands too +Published release charts pin the worker's approved image digest. Source charts without a digest default to the chart's application version. The chart connects the worker to the backend service. Keep these values and the Secret when upgrading the chart so the gateway and worker upgrade together. `lensWorker.replicaCount` controls simultaneous investigations. To use a private registry or external proxy, set `lensWorker.image.repository`, `lensWorker.image.digest` (or `tag` for a source build), and `lensWorker.url`. A digest takes precedence over the tag. The dashboard uses the chart's worker image for standalone install commands too ## Standalone worker @@ -193,4 +193,9 @@ The dashboard reads its image from the running gateway. `LENS_WORKER_IMAGE` over For source development, use `make lens-dev`, which gives the proxy and source worker the same commit identity. For custom containers, build both from the same checkout with `--build-arg LITELLM_RELEASE_TAG=sha-$(git rev-parse HEAD)` and set the proxy's `LENS_WORKER_IMAGE` to the worker image you built. An unlabelled custom build refuses worker setup and claims instead of guessing from the Python package version. Normal package-index installations use their installed release version -The hourly development pipeline pins all component images to the same selected commit and publishes its chart only after every build and worker smoke test succeeds. The public commit-tagged worker workflow publishes on Lens-related changes, so an arbitrary `main` commit may require building your own pair; do not substitute the newest available worker +The hourly development pipeline pins all component images to the same selected commit and publishes its chart only after every build and worker smoke test succeeds. The public commit-tagged worker workflow publishes to `ghcr.io/berriai/litellm-lens-worker-dev` on Lens-related changes, so an arbitrary `main` commit may require building your own pair; do not substitute the newest available worker + + +## Worker dependencies + +The worker uses the same digest-pinned Wolfi base and Python version as the component images. Python dependencies and their hashes are locked in `deploy/lens/requirements.lock`. To update them, edit `deploy/lens/requirements.in`, then run `uv pip compile --universal --python-version 3.13 --generate-hashes --no-emit-index-url deploy/lens/requirements.in -o deploy/lens/requirements.lock`. The image installs only the locked wheels with hash verification. CI builds and scans both native architectures diff --git a/deploy/lens/requirements.in b/deploy/lens/requirements.in new file mode 100644 index 00000000000..3122d7bd6f2 --- /dev/null +++ b/deploy/lens/requirements.in @@ -0,0 +1,2 @@ +httpx==0.28.1 +pydantic==2.13.4 diff --git a/deploy/lens/requirements.lock b/deploy/lens/requirements.lock new file mode 100644 index 00000000000..a895b6d645e --- /dev/null +++ b/deploy/lens/requirements.lock @@ -0,0 +1,172 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --universal --python-version 3.13 --generate-hashes --no-emit-index-url deploy/lens/requirements.in -o deploy/lens/requirements.lock +annotated-types==0.8.0 \ + --hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \ + --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 + # via pydantic +anyio==4.15.1 \ + --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \ + --hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94 + # via httpx +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via + # httpcore + # httpx +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via httpcore +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via -r deploy/lens/requirements.in +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via + # anyio + # httpx +pydantic==2.13.4 \ + --hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \ + --hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6 + # via -r deploy/lens/requirements.in +pydantic-core==2.46.4 \ + --hash=sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0 \ + --hash=sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262 \ + --hash=sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda \ + --hash=sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0 \ + --hash=sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e \ + --hash=sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b \ + --hash=sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594 \ + --hash=sha256:10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29 \ + --hash=sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2 \ + --hash=sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c \ + --hash=sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d \ + --hash=sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398 \ + --hash=sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d \ + --hash=sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3 \ + --hash=sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f \ + --hash=sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb \ + --hash=sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7 \ + --hash=sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5 \ + --hash=sha256:228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9 \ + --hash=sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462 \ + --hash=sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4 \ + --hash=sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b \ + --hash=sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d \ + --hash=sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df \ + --hash=sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2 \ + --hash=sha256:3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0 \ + --hash=sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519 \ + --hash=sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd \ + --hash=sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7 \ + --hash=sha256:3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac \ + --hash=sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6 \ + --hash=sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565 \ + --hash=sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898 \ + --hash=sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb \ + --hash=sha256:432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928 \ + --hash=sha256:4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6 \ + --hash=sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3 \ + --hash=sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a \ + --hash=sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596 \ + --hash=sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987 \ + --hash=sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e \ + --hash=sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d \ + --hash=sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712 \ + --hash=sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008 \ + --hash=sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd \ + --hash=sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1 \ + --hash=sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be \ + --hash=sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea \ + --hash=sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292 \ + --hash=sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33 \ + --hash=sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3 \ + --hash=sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4 \ + --hash=sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b \ + --hash=sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826 \ + --hash=sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac \ + --hash=sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7 \ + --hash=sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d \ + --hash=sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf \ + --hash=sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4 \ + --hash=sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc \ + --hash=sha256:8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15 \ + --hash=sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3 \ + --hash=sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b \ + --hash=sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914 \ + --hash=sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04 \ + --hash=sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c \ + --hash=sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b \ + --hash=sha256:91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9 \ + --hash=sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce \ + --hash=sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4 \ + --hash=sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a \ + --hash=sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f \ + --hash=sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424 \ + --hash=sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894 \ + --hash=sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9 \ + --hash=sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76 \ + --hash=sha256:9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201 \ + --hash=sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb \ + --hash=sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109 \ + --hash=sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4 \ + --hash=sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848 \ + --hash=sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526 \ + --hash=sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0 \ + --hash=sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01 \ + --hash=sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458 \ + --hash=sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e \ + --hash=sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba \ + --hash=sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a \ + --hash=sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39 \ + --hash=sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c \ + --hash=sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000 \ + --hash=sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b \ + --hash=sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf \ + --hash=sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4 \ + --hash=sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd \ + --hash=sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28 \ + --hash=sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9 \ + --hash=sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30 \ + --hash=sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983 \ + --hash=sha256:d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1 \ + --hash=sha256:d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76 \ + --hash=sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5 \ + --hash=sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4 \ + --hash=sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7 \ + --hash=sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c \ + --hash=sha256:e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066 \ + --hash=sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3 \ + --hash=sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02 \ + --hash=sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89 \ + --hash=sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50 \ + --hash=sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76 \ + --hash=sha256:f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49 \ + --hash=sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b \ + --hash=sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d \ + --hash=sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7 \ + --hash=sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4 \ + --hash=sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c \ + --hash=sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e \ + --hash=sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff \ + --hash=sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae + # via pydantic +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # anyio + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.4 \ + --hash=sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47 \ + --hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147 + # via pydantic diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 2b7fdfb5fd7..eb7433c279a 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -472,10 +472,21 @@ collector containers through an emptyDir. Empty when the sidecar is off or gateway.collector.address is a tcp://127.0.0.1: address. */}} {{- define "litellm.lensWorker.image" -}} +{{- if .Values.lensWorker.image.digest -}} +{{- if not (regexMatch "^sha256:[0-9a-f]{64}$" .Values.lensWorker.image.digest) -}} +{{- fail "lensWorker.image.digest must be sha256 followed by 64 lowercase hex characters" -}} +{{- end -}} +{{- printf "%s@%s" .Values.lensWorker.image.repository .Values.lensWorker.image.digest -}} +{{- else -}} {{- $backendTag := .Values.backend.image.tag | default .Chart.AppVersion -}} {{- $releaseTag := ternary (printf "v%s" $backendTag) $backendTag (regexMatch "^[0-9]" $backendTag) -}} {{- $tag := .Values.lensWorker.image.tag | default $releaseTag -}} -{{- printf "%s:%s" .Values.lensWorker.image.repository $tag -}} +{{- $repository := .Values.lensWorker.image.repository -}} +{{- if and (hasPrefix "sha-" $tag) (eq $repository "ghcr.io/berriai/litellm-lens-worker") -}} +{{- $repository = "ghcr.io/berriai/litellm-lens-worker-dev" -}} +{{- end -}} +{{- printf "%s:%s" $repository $tag -}} +{{- end -}} {{- end -}} {{- define "litellm.gateway.collectorSocketDir" -}} diff --git a/helm/litellm/tests/lens_worker_tests.yaml b/helm/litellm/tests/lens_worker_tests.yaml index 5230f14efb4..9a83a4af6a4 100644 --- a/helm/litellm/tests/lens_worker_tests.yaml +++ b/helm/litellm/tests/lens_worker_tests.yaml @@ -6,6 +6,66 @@ templates: values: - ./values/required.yaml tests: + - it: installs the development package for a source commit + template: lens/deployment.yaml + set: + backend.image.tag: sha-0123456789abcdef + lensWorker.enabled: true + lensWorker.tokenSecret.name: lens-credential + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm-lens-worker-dev:sha-0123456789abcdef + - it: advertises the development package for standalone source workers + template: backend/deployment.yaml + set: + backend.image.tag: sha-0123456789abcdef + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LENS_WORKER_IMAGE + value: ghcr.io/berriai/litellm-lens-worker-dev:sha-0123456789abcdef + - it: preserves an explicit private source image repository + template: lens/deployment.yaml + set: + backend.image.tag: sha-0123456789abcdef + lensWorker.enabled: true + lensWorker.tokenSecret.name: lens-credential + lensWorker.image.repository: registry.example/lens-worker + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/lens-worker:sha-0123456789abcdef + - it: pins the worker to its approved digest even when its tag changes + template: lens/deployment.yaml + set: + lensWorker.enabled: true + lensWorker.tokenSecret.name: lens-credential + lensWorker.image.tag: replaced-release + lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm-lens-worker@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + - it: advertises the approved digest to standalone installers + template: backend/deployment.yaml + set: + lensWorker.image.tag: replaced-release + lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LENS_WORKER_IMAGE + value: ghcr.io/berriai/litellm-lens-worker@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + - it: refuses a malformed digest instead of falling back to the tag + template: backend/deployment.yaml + set: + lensWorker.image.digest: sha256:invalid + asserts: + - failedTemplate: + errorMessage: lensWorker.image.digest must be sha256 followed by 64 lowercase hex characters - it: keeps the worker opt in template: lens/deployment.yaml asserts: diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 6964dfd2b6e..cf3334f6156 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -636,6 +636,7 @@ lensWorker: image: repository: ghcr.io/berriai/litellm-lens-worker tag: "" + digest: "" pullPolicy: IfNotPresent tokenSecret: name: "" diff --git a/litellm/proxy/lens/release.py b/litellm/proxy/lens/release.py index 0338858a6a5..c5256e3ed4e 100644 --- a/litellm/proxy/lens/release.py +++ b/litellm/proxy/lens/release.py @@ -32,4 +32,5 @@ def worker_image() -> str: override: Final = os.environ.get("LENS_WORKER_IMAGE", "") if override: return override - return f"ghcr.io/berriai/litellm-lens-worker:{tag}" + package: Final = "litellm-lens-worker-dev" if tag.startswith("sha-") else "litellm-lens-worker" + return f"ghcr.io/berriai/{package}:{tag}" diff --git a/tests/unit/proxy/lens/test_release.py b/tests/unit/proxy/lens/test_release.py index 95d637465c4..a47a84744a7 100644 --- a/tests/unit/proxy/lens/test_release.py +++ b/tests/unit/proxy/lens/test_release.py @@ -21,6 +21,13 @@ def test_private_registry_override_keeps_its_exact_digest(monkeypatch: pytest.Mo assert worker_image() == image +def test_source_build_uses_the_separate_development_package(monkeypatch: pytest.MonkeyPatch) -> None: + tag: Final = "sha-" + "a" * 40 + monkeypatch.setenv("LITELLM_RELEASE_TAG", tag) + monkeypatch.delenv("LENS_WORKER_IMAGE", raising=False) + assert worker_image() == f"ghcr.io/berriai/litellm-lens-worker-dev:{tag}" + + @pytest.mark.parametrize( "installed,expected", (("1.2.3", "v1.2.3"), ("1.2.3rc4", "v1.2.3-rc.4"), ("1.2.3.dev5", "v1.2.3-dev.5")),