From 351049441c5f8f9048d6ce77aea4eab5de60ef89 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 18 Jul 2026 15:38:54 -0700 Subject: [PATCH] chore(release): backport #33853 and #33864 onto patch-1.93.0rc2 to complete the 1.93.0 stable cut (#33869) * fix(docker): bake prisma CLI and engines at a fixed path so fresh-DB migrations work for any uid offline (#33853) * fix(docker): bake prisma CLI and engines at a fixed path so fresh-DB migrations work for any uid offline The runtime image shipped the prisma CLI and engines under /root/.cache, the default HOME-derived prisma-python cache location. Any deployment whose runtime HOME is not /root (kubernetes runAsUser, docker --user, HOME overrides) missed that cache on a fresh database, fell back to a nodeenv Node download that crashes on Wolfi (libatomic.so.1), and started the proxy with zero tables while every DB-backed endpoint returned 500 The bake now lives at /opt/prisma, a path no HOME resolution or cache volume mount can shadow. The builder records the engine paths there at generate time, and the runtime stage pins PRISMA_BINARY_CACHE_DIR, PRISMA_CLI_PATH, PRISMA_CLI_QUERY_ENGINE_TYPE=binary and PRISMA_OFFLINE_MODE so both litellm-proxy-extras and prisma-python resolve the baked CLI and engines directly. prisma migrate deploy on a fresh database now needs no npm and no network access for any runtime uid, including readOnlyRootFilesystem deployments Verified against live containers: fresh and existing databases as root, uid 12345, HOME overridden, on an internal-only docker network, and with a read-only root filesystem all migrate and serve /team/new successfully Fixes #33650, #24554 * chore(docker): fail the image build if the baked prisma CLI layout drifts Asserts the baked CLI shim is executable and its entrypoint exists in the runtime stage after the COPY and chmod, so a layout change in a future prisma-python release breaks the image build loudly instead of silently degrading the migration path at container startup (cherry picked from commit 567ebcb3e9b0d7f817ee920007662444dc9046ad) * fix(router): treat malformed configured token limits as absent on /v1/models (#33864) A deployment whose model_info carried a non-numeric max_input_tokens or max_output_tokens (for example "128,000" or an empty string) made the bare int() in get_configured_token_limits raise inside the per-model /v1/models loop, so one misconfigured deployment turned the entire listing into a 500. Coerce each configured limit safely and treat malformed values as absent, matching the graceful degradation the listing had before the cost-map switch (cherry picked from commit ef7007c3dd9c6925c53c4430f4d944f2b646aecc) --- Dockerfile | 28 +++++++++++------- docker/Dockerfile.database | 30 ++++++++++++------- litellm/router.py | 17 +++++++---- tests/test_litellm/proxy/test_proxy_utils.py | 25 ++++++++++++++++ tests/test_litellm/test_router.py | 31 ++++++++++++++++++++ 5 files changed, 106 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index 581d1808f0a..9977ebb82d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -86,7 +86,9 @@ RUN uv sync --frozen --no-default-groups --no-editable \ --extra semantic-router \ --python python3 -RUN prisma generate --schema=./schema.prisma +RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ + npm_config_cache=/root/.npm \ + prisma generate --schema=./schema.prisma RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh @@ -100,7 +102,11 @@ USER root RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile WORKDIR /app -ENV PATH="/app/.venv/bin:${PATH}" +ENV PATH="/app/.venv/bin:${PATH}" \ + PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ + PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \ + PRISMA_CLI_QUERY_ENGINE_TYPE=binary \ + PRISMA_OFFLINE_MODE=true # Copy only what runtime needs. The application is installed inside the venv; # the rest of the builder's /app is source and build metadata that must not @@ -115,16 +121,18 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr # enterprise.enterprise_hooks from it) COPY --from=builder /app/enterprise /app/enterprise COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras -# Prisma binaries live in $HOME/.cache (default prisma-python location), -# which is /root/.cache here. Copy only the Prisma subdirs — copying the -# whole /root/.cache drags in the uv build cache (~660 MB, includes a -# setuptools wheel that surfaces as a CVE finding even though it's not -# on the runtime sys.path). -COPY --from=builder /root/.cache/prisma /root/.cache/prisma -COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python +# Prisma CLI + engines are baked under /opt/prisma, a fixed path every +# runtime uid can read and that no cache volume mount shadows. The paths are +# pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and recorded into the +# generated client at build time, so `prisma migrate deploy` on a fresh +# database needs no npm and no network access (#33650, #24554). +COPY --from=builder /opt/prisma /opt/prisma RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ - find /app/.venv -type d -path "*/tornado/test" -delete + find /app/.venv -type d -path "*/tornado/test" -delete && \ + chmod -R a+rX /opt/prisma && \ + test -x /opt/prisma/binaries/node_modules/.bin/prisma && \ + test -f /opt/prisma/binaries/node_modules/prisma/build/index.js EXPOSE 4000/tcp diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 868b6682276..34c9c606991 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -84,7 +84,9 @@ RUN uv sync --frozen --no-default-groups --no-editable \ --extra semantic-router \ --python python3 -RUN prisma generate --schema=./schema.prisma +RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ + npm_config_cache=/root/.npm \ + prisma generate --schema=./schema.prisma RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh @@ -97,7 +99,11 @@ USER root RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile WORKDIR /app -ENV PATH="/app/.venv/bin:${PATH}" +ENV PATH="/app/.venv/bin:${PATH}" \ + PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ + PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \ + PRISMA_CLI_QUERY_ENGINE_TYPE=binary \ + PRISMA_OFFLINE_MODE=true # Copy only what runtime needs. The application is installed inside the venv; # the rest of the builder's /app is source and build metadata that must not @@ -112,16 +118,20 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr # enterprise.enterprise_hooks from it) COPY --from=builder /app/enterprise /app/enterprise COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras -# Prisma binaries live in $HOME/.cache (default prisma-python location), -# which is /root/.cache here. Copy them from the builder so they survive -# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem -# + emptyDir) — otherwise the mount would shadow the baked-in query engine. -# Only the Prisma subdirs: the whole /root/.cache drags in the uv build cache. -COPY --from=builder /root/.cache/prisma /root/.cache/prisma -COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python +# Prisma CLI + engines are baked under /opt/prisma, a fixed path every +# runtime uid can read and that no cache volume mount shadows (unlike +# /app/.cache or $HOME/.cache in readOnlyRootFilesystem + emptyDir setups). +# The paths are pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and +# recorded into the generated client at build time, so `prisma migrate +# deploy` on a fresh database needs no npm and no network access +# (#33650, #24554). +COPY --from=builder /opt/prisma /opt/prisma RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ - find /app/.venv -type d -path "*/tornado/test" -delete + find /app/.venv -type d -path "*/tornado/test" -delete && \ + chmod -R a+rX /opt/prisma && \ + test -x /opt/prisma/binaries/node_modules/.bin/prisma && \ + test -f /opt/prisma/binaries/node_modules/prisma/build/index.js EXPOSE 4000/tcp diff --git a/litellm/router.py b/litellm/router.py index ced6090f82a..138ba7ebdbc 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8362,7 +8362,8 @@ class Router: Return (max_input_tokens, max_output_tokens) explicitly configured in a concrete deployment's model_info for model_name, via O(1) index lookup. - Returns (None, None) for wildcard-expanded or unknown names. Unlike + Returns (None, None) for wildcard-expanded or unknown names, and treats a + malformed configured value as absent rather than failing the listing. Unlike get_model_group_info, this never triggers pattern matching or deep copies, so it is safe to call per listed model on the /v1/models hot path. """ @@ -8370,12 +8371,18 @@ class Router: if deployment is None: return (None, None) + def _as_int(value: object) -> "int | None": + if value is None or isinstance(value, bool): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + model_info = deployment.model_info - max_input = model_info.get("max_input_tokens") - max_output = model_info.get("max_output_tokens") return ( - int(max_input) if max_input is not None else None, - int(max_output) if max_output is not None else None, + _as_int(model_info.get("max_input_tokens")), + _as_int(model_info.get("max_output_tokens")), ) def get_deployment_credentials_with_provider(self, model_id: str) -> Optional[Dict[str, Any]]: diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index d2bdb1764a4..9486646ea4a 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -556,6 +556,31 @@ def test_create_model_info_response_deployment_limits_override_cost_map(): assert response["max_output_tokens"] == 16384 +def test_create_model_info_response_survives_malformed_configured_limits(): + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "bad-limit-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"max_input_tokens": "128,000"}, + } + ] + ) + + response = create_model_info_response( + model_id="bad-limit-model", + provider="openai", + llm_router=router, + get_model_info=_raise_unmapped, + ) + + assert response["id"] == "bad-limit-model" + assert "max_input_tokens" not in response + assert "max_output_tokens" not in response + + def test_create_model_info_response_emits_integer_token_counts(): response = create_model_info_response( model_id="some-model", diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 3fca5aa6901..3d14d7bf927 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -5351,3 +5351,34 @@ def test_get_configured_token_limits_skips_wildcard_pattern_matching(): assert router.get_configured_token_limits( "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" ) == (None, None) + + +def test_get_configured_token_limits_treats_malformed_values_as_absent(): + malformed = ["", "unlimited", "128,000", [128000], {"max": 128000}, True] + router = litellm.Router( + model_list=[ + { + "model_name": f"bad-limit-{i}", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"max_input_tokens": bad, "max_output_tokens": bad}, + } + for i, bad in enumerate(malformed) + ] + ) + + for i in range(len(malformed)): + assert router.get_configured_token_limits(f"bad-limit-{i}") == (None, None) + + +def test_get_configured_token_limits_coerces_numeric_strings(): + router = litellm.Router( + model_list=[ + { + "model_name": "quoted-limits-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"max_input_tokens": "32000", "max_output_tokens": "8000"}, + } + ] + ) + + assert router.get_configured_token_limits("quoted-limits-model") == (32000, 8000)