scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

[Feature] Download Prisma binaries at build time instead of at runtime for Security Restricted environments (#17695)

Browse source 
* Use config file to enable prometheus metrics

* Revert "Use config file to enable prometheus metrics"

This reverts commit 15ae36e171.

* Improve hardened stack and Prisma offline flow

* Document hardened compose usage

* Remove undesired change in fastapi-sso

* Restore dashboard lockfile

* Remove unecessary tempdirs

* Document hardened/offline Docker validation flow
This commit is contained in:
Mateo Di Loreto 2025-12-16 12:55:53 -03:00 • committed by GitHub
parent 8f976df651
commit 107ea9043a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 284 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
docker-compose.hardened.yml Normal file
View file

@ -0,0 +1,46 @@
services:
# Hardened stack: for testing the proxy under non-root, read-only, proxy-enforced constraints.
# Keep this file focused on hardening/QA scenarios; leave the main docker-compose.yml for default dev usage.
litellm:
build:
context: .
dockerfile: docker/Dockerfile.non_root
target: runtime
args:
PROXY_EXTRAS_SOURCE: "local"
depends_on:
- squid
user: "101:101"
group_add:
- "2345"
read_only: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
tmpfs:
- /app/cache:rw,noexec,nosuid,nodev,size=128m,uid=101,gid=101,mode=1777
- /app/migrations:rw,noexec,nosuid,nodev,size=64m,uid=101,gid=101,mode=1777
volumes:
- ./proxy_server_config.yaml:/app/config.yaml:ro
environment:
LITELLM_NON_ROOT: "true"
PRISMA_BINARY_CACHE_DIR: "/app/cache/prisma-python/binaries"
XDG_CACHE_HOME: "/app/cache"
LITELLM_MIGRATION_DIR: "/app/migrations"
HTTP_PROXY: "http://squid:3128"
HTTPS_PROXY: "http://squid:3128"
NO_PROXY: "localhost,127.0.0.1,db"
command:
- "--port"
- "4000"
- "--config"
- "/app/config.yaml"
squid:
image: sameersbn/squid:3.5.27-2
restart: unless-stopped
ports:
- "3128:3128"
tmpfs:
- /var/spool/squid:rw,noexec,nosuid,nodev,size=64m
- /var/log/squid:rw,noexec,nosuid,nodev,size=16m

255
docker/Dockerfile.non_root
View file

@ -1,154 +1,183 @@
# Base images # Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
ARG PROXY_EXTRAS_SOURCE=published
# ----------------- # -----------------
# Builder Stage # Builder Stage
# ----------------- # -----------------
FROM $LITELLM_BUILD_IMAGE AS builder FROM $LITELLM_BUILD_IMAGE AS builder
ARG PROXY_EXTRAS_SOURCE
WORKDIR /app WORKDIR /app
# Install build dependencies including Node.js for UI build
USER root USER root
# Install build dependencies with retry logic (includes node for UI build)
RUN for i in 1 2 3; do \ RUN for i in 1 2 3; do \
apk add --no-cache \ apk add --no-cache \
python3 \ python3 \
py3-pip \ py3-pip \
clang \ clang \
llvm \ llvm \
lld \ lld \
gcc \ gcc \
linux-headers \ linux-headers \
build-base \ build-base \
bash \ bash \
nodejs \ nodejs \
npm && break || sleep 5; \ npm && break || sleep 5; \
done \ done \
&& pip install --no-cache-dir --upgrade pip build && pip install --no-cache-dir --upgrade pip build
# Copy project files # Cache Python dependencies
COPY requirements.txt .
RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt \
&& pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.9.0"
# Copy source after dependency layers
COPY . . COPY . .
# Set LITELLM_NON_ROOT flag for build time # Set non-root flag for build time consistency
ENV LITELLM_NON_ROOT=true ENV LITELLM_NON_ROOT=true
# Build Admin UI # Build Admin UI using the upstream command order while keeping a single RUN layer
RUN mkdir -p /tmp/litellm_ui RUN mkdir -p /tmp/litellm_ui && \
npm install -g npm@latest && npm cache clean --force && \
cd /app/ui/litellm-dashboard && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi && \
rm -f package-lock.json && \
npm install --legacy-peer-deps && \
npm run build && \
cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/ && \
mkdir -p /tmp/litellm_assets && \
cp /app/litellm/proxy/logo.jpg /tmp/litellm_assets/logo.jpg && \
( cd /tmp/litellm_ui && \
for html_file in *.html; do \
if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \
folder_name="${html_file%.html}" && \
mkdir -p "$folder_name" && \
mv "$html_file" "$folder_name/index.html"; \
fi; \
done ) && \
cd /app/ui/litellm-dashboard && rm -rf ./out
RUN npm install -g npm@latest && npm cache clean --force # Build litellm wheel and place it in wheels dir (replace any PyPI wheels)
RUN cd /app/ui/litellm-dashboard && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi
RUN cd /app/ui/litellm-dashboard && rm -f package-lock.json
RUN cd /app/ui/litellm-dashboard && npm install --legacy-peer-deps
RUN cd /app/ui/litellm-dashboard && npm run build
RUN cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/
RUN mkdir -p /tmp/litellm_assets && cp /app/litellm/proxy/logo.jpg /tmp/litellm_assets/logo.jpg
RUN cd /tmp/litellm_ui && \
for html_file in *.html; do \
if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \
folder_name="${html_file%.html}" && \
mkdir -p "$folder_name" && \
mv "$html_file" "$folder_name/index.html"; \
fi; \
done
RUN cd /app/ui/litellm-dashboard && rm -rf ./out
# Build package and wheel dependencies
RUN rm -rf dist/* && python -m build && \ RUN rm -rf dist/* && python -m build && \
pip install dist/*.whl && \ rm -f /wheels/litellm-*.whl && \
pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt cp dist/*.whl /wheels/
# Optionally build local litellm-proxy-extras wheel
RUN if [ "$PROXY_EXTRAS_SOURCE" = "local" ]; then \
cd /app/litellm-proxy-extras && rm -rf dist && python -m build && \
cp dist/*.whl /wheels/; \
fi
# Pre-cache Prisma binaries in the builder stage
ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
XDG_CACHE_HOME=/app/.cache \
PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}"
RUN pip install --no-cache-dir prisma==0.11.0 nodejs-bin==18.4.0a4 \
&& mkdir -p /app/.cache/npm
RUN NPM_CONFIG_CACHE=/app/.cache/npm \
python -c "import prisma.cli.prisma as p; p.ensure_cached()"
RUN prisma generate && \
prisma --version && \
prisma migrate diff --from-empty --to-schema-datamodel ./schema.prisma --script > /dev/null 2>&1 || true
# ----------------- # -----------------
# Runtime Stage # Runtime Stage
# ----------------- # -----------------
FROM $LITELLM_RUNTIME_IMAGE AS runtime FROM $LITELLM_RUNTIME_IMAGE AS runtime
ARG PROXY_EXTRAS_SOURCE
WORKDIR /app WORKDIR /app
# Install runtime dependencies
USER root USER root
RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done \
&& for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done
# Copy only necessary artifacts from builder stage for runtime # Install runtime dependencies with retry
COPY . . RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done \
&& for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done
# Copy artifacts from builder
COPY --from=builder /app/requirements.txt /app/requirements.txt
COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/ COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
COPY --from=builder /app/schema.prisma /app/schema.prisma COPY --from=builder /app/schema.prisma /app/
COPY --from=builder /app/dist/*.whl .
COPY --from=builder /wheels/ /wheels/ COPY --from=builder /wheels/ /wheels/
COPY --from=builder /tmp/litellm_ui /tmp/litellm_ui COPY --from=builder /tmp/litellm_ui /tmp/litellm_ui
COPY --from=builder /tmp/litellm_assets /tmp/litellm_assets COPY --from=builder /tmp/litellm_assets /tmp/litellm_assets
COPY --from=builder /app/.cache /app/.cache
COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
COPY --from=builder \
/usr/lib/python3.13/site-packages/nodejs* \
/usr/lib/python3.13/site-packages/prisma* \
/usr/lib/python3.13/site-packages/tomlkit* \
/usr/lib/python3.13/site-packages/nodeenv* \
/usr/lib/python3.13/site-packages/
COPY --from=builder /usr/bin/prisma /usr/bin/prisma
# Install package from wheel and dependencies # Final runtime environment configuration
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \ ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
&& rm -f *.whl \ PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
&& rm -rf /wheels HOME=/app \
LITELLM_NON_ROOT=true \
XDG_CACHE_HOME=/app/.cache
# Remove test files and keys from dependencies # Install packages from wheels and optional extras without network
RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
find /usr/lib -type d -path "*/tornado/test" -delete pip install --no-index --find-links=/wheels/ /wheels/litellm-*-py3-none-any.whl && \
pip install --no-index --find-links=/wheels/ --no-deps semantic_router==0.1.11 && \
pip install --no-index --find-links=/wheels/ aurelio-sdk==0.0.19 && \
if [ "$PROXY_EXTRAS_SOURCE" = "local" ]; then \
if ls /wheels/litellm_proxy_extras-*.whl >/dev/null 2>&1; then \
pip install --no-index --find-links=/wheels/ /wheels/litellm_proxy_extras-*.whl; \
else \
echo "litellm_proxy_extras wheel not found; skipping local install"; \
fi; \
fi
# Install semantic_router and aurelio-sdk using script # Permissions, cleanup, and Prisma prep
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
mkdir -p /nonexistent /.npm /tmp/litellm_assets /tmp/litellm_ui && \
chown -R nobody:nogroup /app /tmp/litellm_ui /tmp/litellm_assets /nonexistent /.npm && \
pip uninstall jwt -y || true && \
pip uninstall PyJWT -y || true && \
pip install --no-index --find-links=/wheels/ PyJWT==2.10.1 --no-cache-dir && \
rm -rf /wheels && \
PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
chown -R nobody:nogroup $PRISMA_PATH && \
LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
[ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH && \
LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g=u $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+w $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+rX $PRISMA_PATH && \
chmod -R g+rX /app/.cache && \
mkdir -p /tmp/.npm /nonexistent /.npm && \
prisma generate
# Ensure correct JWT library is used (pyjwt not jwt) # Switch to non-root user for runtime
RUN pip uninstall jwt -y && \
pip uninstall PyJWT -y && \
pip install PyJWT==2.9.0 --no-cache-dir
# Set Prisma cache directories
ENV PRISMA_BINARY_CACHE_DIR=/nonexistent
ENV NPM_CONFIG_CACHE=/.npm
# Install prisma and make entrypoints executable
RUN pip install --no-cache-dir prisma && \
chmod +x docker/entrypoint.sh && \
chmod +x docker/prod_entrypoint.sh
# Create directories and set permissions for non-root user
RUN mkdir -p /nonexistent /.npm /tmp/litellm_assets && \
chown -R nobody:nogroup /app /tmp/litellm_ui /tmp/litellm_assets /nonexistent /.npm && \
PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
chown -R nobody:nogroup $PRISMA_PATH && \
LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
[ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH
# OpenShift compatibility
RUN PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g=u $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+w $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true
# Switch to non-root user
USER nobody USER nobody
# Set HOME for prisma generate to have a writable directory # Prisma runtime knobs for offline containers
ENV HOME=/app ENV PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
PRISMA_HIDE_UPDATE_MESSAGE=1 \
# Set LITELLM_NON_ROOT flag for runtime PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \
ENV LITELLM_NON_ROOT=true NPM_CONFIG_CACHE=/app/.cache/npm \
NPM_CONFIG_PREFER_OFFLINE=true \
RUN prisma generate PRISMA_OFFLINE_MODE=true
EXPOSE 4000/tcp EXPOSE 4000/tcp
ENTRYPOINT ["/app/docker/prod_entrypoint.sh"] ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]
CMD ["--port", "4000"]

24
docker/README.md
View file

@ -59,6 +59,30 @@ To stop the running containers, use the following command:
docker compose down docker compose down
``` ```
## Hardened / Offline Testing
To ensure changes are safe for non-root, read-only root filesystems and restricted egress, always validate with the hardened compose file:
```bash
docker compose -f docker-compose.yml -f docker-compose.hardened.yml build --no-cache
docker compose -f docker-compose.yml -f docker-compose.hardened.yml up -d
```
This setup:
- Builds from `docker/Dockerfile.non_root` with Prisma engines and Node toolchain baked into the image.
- Runs the proxy as a non-root user with a read-only rootfs and only two writable tmpfs mounts:
- `/app/cache` (Prisma/NPM cache; backing `PRISMA_BINARY_CACHE_DIR`, `NPM_CONFIG_CACHE`, `XDG_CACHE_HOME`)
- `/app/migrations` (Prisma migration workspace; backing `LITELLM_MIGRATION_DIR`)
- Routes all outbound traffic through a local Squid proxy that denies egress, so Prisma migrations must use the cached CLI and engines.
You should also verify offline Prisma behaviour with:
```bash
docker run --rm --network none --entrypoint prisma ghcr.io/berriai/litellm:main-stable --version
```
This command should succeed (showing engine versions) even with `--network none`, confirming that Prisma binaries are available without network access.
## Troubleshooting ## Troubleshooting
- **`build_admin_ui.sh: not found`**: This error can occur if the Docker build context is not set correctly. Ensure that you are running the `docker-compose` command from the root of the project. - **`build_admin_ui.sh: not found`**: This error can occur if the Docker build context is not set correctly. Ensure that you are running the `docker-compose` command from the root of the project.

80
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -18,6 +18,45 @@ def str_to_bool(value: Optional[str]) -> bool:
return value.lower() in ("true", "1", "t", "y", "yes") return value.lower() in ("true", "1", "t", "y", "yes")
def _get_prisma_env() -> dict:
"""Get environment variables for Prisma, handling offline mode if configured."""
prisma_env = os.environ.copy()
if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")):
# These env vars prevent Prisma from attempting downloads
prisma_env["NPM_CONFIG_PREFER_OFFLINE"] = "true"
prisma_env["NPM_CONFIG_CACHE"] = os.getenv("NPM_CONFIG_CACHE", "/app/.cache/npm")
return prisma_env
def _get_prisma_command() -> str:
"""Get the Prisma command to use, bypassing Python wrapper in offline mode."""
if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")):
# Primary location where Prisma Python package installs the CLI
default_cli_path = "/app/.cache/prisma-python/binaries/node_modules/.bin/prisma"
# Check if custom path is provided (for flexibility)
custom_cli_path = os.getenv("PRISMA_CLI_PATH")
if custom_cli_path and os.path.exists(custom_cli_path):
logger.info(f"Using custom Prisma CLI at {custom_cli_path}")
return custom_cli_path
# Check the default location
if os.path.exists(default_cli_path):
logger.info(f"Using cached Prisma CLI at {default_cli_path}")
return default_cli_path
# If not found, log warning and fall back
logger.warning(
f"Prisma CLI not found at {default_cli_path}. "
"Falling back to Python wrapper (may attempt downloads)"
)
# Fall back to the Python wrapper (will work in online mode)
return "prisma"
class ProxyExtrasDBManager: class ProxyExtrasDBManager:
@staticmethod @staticmethod
def _get_prisma_dir() -> str: def _get_prisma_dir() -> str:
@ -57,6 +96,11 @@ class ProxyExtrasDBManager:
init_dir.mkdir(parents=True, exist_ok=True) init_dir.mkdir(parents=True, exist_ok=True)
database_url = os.getenv("DATABASE_URL") database_url = os.getenv("DATABASE_URL")
if not database_url:
logger.error("DATABASE_URL not set")
return False
# Set up environment for offline mode if configured
prisma_env = _get_prisma_env()
try: try:
# 1. Generate migration SQL file by comparing empty state to current db state # 1. Generate migration SQL file by comparing empty state to current db state
@ -64,7 +108,7 @@ class ProxyExtrasDBManager:
migration_file = init_dir / "migration.sql" migration_file = init_dir / "migration.sql"
subprocess.run( subprocess.run(
[ [
"prisma", _get_prisma_command(),
"migrate", "migrate",
"diff", "diff",
"--from-empty", "--from-empty",
@ -75,13 +119,14 @@ class ProxyExtrasDBManager:
stdout=open(migration_file, "w"), stdout=open(migration_file, "w"),
check=True, check=True,
timeout=30, timeout=30,
env=prisma_env
) )
# 3. Mark the migration as applied since it represents current state # 3. Mark the migration as applied since it represents current state
logger.info("Marking baseline migration as applied...") logger.info("Marking baseline migration as applied...")
subprocess.run( subprocess.run(
[ [
"prisma", _get_prisma_command(),
"migrate", "migrate",
"resolve", "resolve",
"--applied", "--applied",
@ -89,6 +134,7 @@ class ProxyExtrasDBManager:
], ],
check=True, check=True,
timeout=30, timeout=30,
env=prisma_env
) )
return True return True
@ -113,21 +159,26 @@ class ProxyExtrasDBManager:
@staticmethod @staticmethod
def _roll_back_migration(migration_name: str): def _roll_back_migration(migration_name: str):
"""Mark a specific migration as rolled back""" """Mark a specific migration as rolled back"""
# Set up environment for offline mode if configured
prisma_env = _get_prisma_env()
subprocess.run( subprocess.run(
["prisma", "migrate", "resolve", "--rolled-back", migration_name], [_get_prisma_command(), "migrate", "resolve", "--rolled-back", migration_name],
timeout=60, timeout=60,
check=True, check=True,
capture_output=True, capture_output=True,
env=prisma_env
) )
@staticmethod @staticmethod
def _resolve_specific_migration(migration_name: str): def _resolve_specific_migration(migration_name: str):
"""Mark a specific migration as applied""" """Mark a specific migration as applied"""
prisma_env = _get_prisma_env()
subprocess.run( subprocess.run(
["prisma", "migrate", "resolve", "--applied", migration_name], [_get_prisma_command(), "migrate", "resolve", "--applied", migration_name],
timeout=60, timeout=60,
check=True, check=True,
capture_output=True, capture_output=True,
env=prisma_env
) )
@staticmethod @staticmethod
@ -194,6 +245,10 @@ class ProxyExtrasDBManager:
3. Mark all existing migrations as applied. 3. Mark all existing migrations as applied.
""" """
database_url = os.getenv("DATABASE_URL") database_url = os.getenv("DATABASE_URL")
if not database_url:
logger.error("DATABASE_URL not set")
return
diff_dir = ( diff_dir = (
Path(migrations_dir) Path(migrations_dir)
/ "migrations" / "migrations"
@ -216,7 +271,7 @@ class ProxyExtrasDBManager:
with open(diff_sql_path, "w") as f: with open(diff_sql_path, "w") as f:
subprocess.run( subprocess.run(
[ [
"prisma", _get_prisma_command(),
"migrate", "migrate",
"diff", "diff",
"--from-url", "--from-url",
@ -228,6 +283,7 @@ class ProxyExtrasDBManager:
check=True, check=True,
timeout=60, timeout=60,
stdout=f, stdout=f,
env=_get_prisma_env()
) )
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
logger.warning(f"Failed to generate migration diff: {e.stderr}") logger.warning(f"Failed to generate migration diff: {e.stderr}")
@ -245,7 +301,7 @@ class ProxyExtrasDBManager:
logger.info("Running prisma db execute to apply the migration diff...") logger.info("Running prisma db execute to apply the migration diff...")
result = subprocess.run( result = subprocess.run(
[ [
"prisma", _get_prisma_command(),
"db", "db",
"execute", "execute",
"--file", "--file",
@ -257,6 +313,7 @@ class ProxyExtrasDBManager:
check=True, check=True,
capture_output=True, capture_output=True,
text=True, text=True,
env=_get_prisma_env()
) )
logger.info(f"prisma db execute stdout: {result.stdout}") logger.info(f"prisma db execute stdout: {result.stdout}")
logger.info("✅ Migration diff applied successfully") logger.info("✅ Migration diff applied successfully")
@ -274,11 +331,12 @@ class ProxyExtrasDBManager:
try: try:
logger.info(f"Resolving migration: {migration_name}") logger.info(f"Resolving migration: {migration_name}")
subprocess.run( subprocess.run(
["prisma", "migrate", "resolve", "--applied", migration_name], [_get_prisma_command(), "migrate", "resolve", "--applied", migration_name],
timeout=60, timeout=60,
check=True, check=True,
capture_output=True, capture_output=True,
text=True, text=True,
env=_get_prisma_env()
) )
logger.debug(f"Resolved migration: {migration_name}") logger.debug(f"Resolved migration: {migration_name}")
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
@ -312,11 +370,12 @@ class ProxyExtrasDBManager:
try: try:
# Set migrations directory for Prisma # Set migrations directory for Prisma
result = subprocess.run( result = subprocess.run(
["prisma", "migrate", "deploy"], [_get_prisma_command(), "migrate", "deploy"],
timeout=60, timeout=60,
check=True, check=True,
capture_output=True, capture_output=True,
text=True, text=True,
env=_get_prisma_env()
) )
logger.info(f"prisma migrate deploy stdout: {result.stdout}") logger.info(f"prisma migrate deploy stdout: {result.stdout}")
@ -344,7 +403,7 @@ class ProxyExtrasDBManager:
# Mark the failed migration as rolled back # Mark the failed migration as rolled back
subprocess.run( subprocess.run(
[ [
"prisma", _get_prisma_command(),
"migrate", "migrate",
"resolve", "resolve",
"--rolled-back", "--rolled-back",
@ -354,6 +413,7 @@ class ProxyExtrasDBManager:
check=True, check=True,
capture_output=True, capture_output=True,
text=True, text=True,
env=_get_prisma_env()
) )
logger.info( logger.info(
f"✅ Migration {failed_migration} marked as rolled back... retrying" f"✅ Migration {failed_migration} marked as rolled back... retrying"
@ -450,7 +510,7 @@ class ProxyExtrasDBManager:
else: else:
# Use prisma db push with increased timeout # Use prisma db push with increased timeout
subprocess.run( subprocess.run(
["prisma", "db", "push", "--accept-data-loss"], [_get_prisma_command(), "db", "push", "--accept-data-loss"],
timeout=60, timeout=60,
check=True, check=True,
) )

1
proxy_server_config.yaml
View file

@ -152,6 +152,7 @@ model_list:
litellm_settings: litellm_settings:
# set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production # set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production
drop_params: True drop_params: True
success_callback: ["prometheus"]
# max_budget: 100 # max_budget: 100
# budget_duration: 30d # budget_duration: 30d
num_retries: 5 num_retries: 5

1
requirements.txt
View file

@ -13,6 +13,7 @@ uvloop==0.21.0 # uvicorn dep, gives us much better performance under load
boto3==1.36.0 # aws bedrock/sagemaker calls boto3==1.36.0 # aws bedrock/sagemaker calls
redis==5.2.1 # redis caching redis==5.2.1 # redis caching
prisma==0.11.0 # for db prisma==0.11.0 # for db
nodejs-bin==18.4.0a4 ## required by prisma for migrations, prevents runtime download
mangum==0.17.0 # for aws lambda functions mangum==0.17.0 # for aws lambda functions
pynacl==1.5.0 # for encrypting keys pynacl==1.5.0 # for encrypting keys
google-cloud-aiplatform==1.47.0 # for vertex ai calls google-cloud-aiplatform==1.47.0 # for vertex ai calls