mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
1f68cc5649
* refactor(rust-bridge): require the native extension _native ships in every wheel and the sdist will not install without it, yet the bridge loaded it optionally and carried None paths that could fail open for fork safety, exception mapping and OTLP errors. Delete loader.py and import _native directly. Tests inject a fake module through NativeBinding(source=...) or with_source() instead of patching the loader The install smoke checks now verify the compiled module, and tests fail instead of skipping when _native is missing. The interactions spec canary installs the project so it has the extension Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(ci): build Rust bridge in Windows smoke tests --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
163 lines
7.1 KiB
Docker
163 lines
7.1 KiB
Docker
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
|
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
|
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
|
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
|
|
ARG PGBOUNCER_VERSION=1.26.0
|
|
ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d
|
|
|
|
FROM $UV_IMAGE AS uvbin
|
|
|
|
FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder
|
|
ARG PGBOUNCER_VERSION
|
|
ARG PGBOUNCER_SHA256
|
|
USER root
|
|
RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl
|
|
WORKDIR /build
|
|
RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \
|
|
echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \
|
|
tar xzf pgbouncer.tar.gz --strip-components=1 && \
|
|
./configure --prefix=/usr/local --with-openssl=/usr && \
|
|
make -j"$(nproc)" pgbouncer && \
|
|
install -m 0755 pgbouncer /usr/local/bin/pgbouncer
|
|
|
|
# ---------- Rust extension ----------
|
|
# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes
|
|
FROM $LITELLM_BUILD_IMAGE AS rust-native
|
|
|
|
WORKDIR /app
|
|
USER root
|
|
|
|
COPY --from=uvbin /uv /uvx /usr/local/bin/
|
|
|
|
RUN for i in 1 2 3; do \
|
|
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \
|
|
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
|
sleep 5; \
|
|
done
|
|
|
|
ENV UV_PYTHON_DOWNLOADS=0
|
|
|
|
COPY pyproject.toml README.md rust-toolchain.toml ./
|
|
COPY .cargo ./.cargo
|
|
COPY litellm-rust ./litellm-rust
|
|
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
mkdir -p litellm/rust_bridge /wheels && \
|
|
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
|
|
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
|
|
python -c "import maturin; maturin.build_wheel('/wheels')"
|
|
|
|
# ---------- Builder ----------
|
|
FROM $LITELLM_BUILD_IMAGE AS builder
|
|
|
|
WORKDIR /app
|
|
USER root
|
|
|
|
COPY --from=uvbin /uv /uvx /usr/local/bin/
|
|
|
|
# nodejs/npm so `prisma generate` uses Wolfi's Node via PRISMA_USE_GLOBAL_NODE
|
|
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
|
|
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
|
|
RUN for i in 1 2 3; do \
|
|
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
|
|
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
|
sleep 5; \
|
|
done
|
|
|
|
# UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start.
|
|
# UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a
|
|
# BuildKit cache mount (different filesystem).
|
|
# UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of
|
|
# silently pulling a managed interpreter.
|
|
# PRISMA_USE_GLOBAL_NODE explicit (matches default) so an env override can't
|
|
# silently re-enable nodeenv's Node download.
|
|
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
|
|
UV_LINK_MODE=copy \
|
|
UV_COMPILE_BYTECODE=1 \
|
|
UV_PYTHON_DOWNLOADS=0 \
|
|
PRISMA_USE_GLOBAL_NODE=true \
|
|
PATH="/app/.venv/bin:${PATH}"
|
|
|
|
# Stage 1 — install dependencies only.
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
|
|
--mount=type=bind,source=uv.lock,target=uv.lock \
|
|
--mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \
|
|
--mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \
|
|
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
|
|
--extra proxy \
|
|
--extra proxy-runtime \
|
|
--group admin-mcp \
|
|
--extra extra_proxy \
|
|
--extra semantic-router \
|
|
--extra bedrock-realtime \
|
|
--python python3.13
|
|
|
|
# Stage 2 — copy source and install the workspace members.
|
|
COPY . .
|
|
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
uv sync --frozen --no-install-project --no-default-groups --no-editable \
|
|
--extra proxy \
|
|
--extra proxy-runtime \
|
|
--group admin-mcp \
|
|
--extra extra_proxy \
|
|
--extra semantic-router \
|
|
--extra bedrock-realtime \
|
|
--python python3.13
|
|
|
|
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
|
|
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
|
|
|
|
# PYTHONPATH=/app makes the source tree shadow the installed package, so the
|
|
# compiled Rust extension must live next to the source or it is never imported.
|
|
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
|
|
|
|
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
|
|
npm_config_cache=/root/.npm \
|
|
prisma generate --schema=./schema.prisma
|
|
|
|
RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component_entrypoint.sh
|
|
|
|
# ---------- Runtime ----------
|
|
FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
|
|
|
USER root
|
|
|
|
RUN for i in 1 2 3; do \
|
|
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic libevent && break; \
|
|
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
|
sleep 5; \
|
|
done
|
|
|
|
# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with
|
|
# /home/nonroot. We run the proxy as that user.
|
|
WORKDIR /app
|
|
ENV HOME=/home/nonroot \
|
|
PATH="/app/.venv/bin:${PATH}" \
|
|
PYTHONPATH="/app" \
|
|
PYTHONDONTWRITEBYTECODE=1 \
|
|
PYTHONUNBUFFERED=1 \
|
|
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
|
|
|
|
COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
|
|
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
|
|
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
|
|
COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway
|
|
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
|
|
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
|
|
COPY --from=builder /opt/prisma /opt/prisma
|
|
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
|
|
|
|
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
|
|
find /app/.venv -type d -path "*/tornado/test" -delete && \
|
|
chmod -R a+rX /opt/prisma && \
|
|
python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \
|
|
python -c "import litellm, litellm.rust_bridge._native; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__"
|
|
|
|
USER nonroot
|
|
|
|
EXPOSE 4000/tcp
|
|
|
|
ENTRYPOINT ["sh", "-c", "exec /app/docker/component_entrypoint.sh python -m gateway.launch --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"]
|
|
CMD ["--host", "0.0.0.0", "--port", "4000"]
|