diff --git a/Dockerfile b/Dockerfile index 581d1808f0a..9977ebb82d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -86,7 +86,9 @@ RUN uv sync --frozen --no-default-groups --no-editable \ --extra semantic-router \ --python python3 -RUN prisma generate --schema=./schema.prisma +RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ + npm_config_cache=/root/.npm \ + prisma generate --schema=./schema.prisma RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh @@ -100,7 +102,11 @@ USER root RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile WORKDIR /app -ENV PATH="/app/.venv/bin:${PATH}" +ENV PATH="/app/.venv/bin:${PATH}" \ + PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ + PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \ + PRISMA_CLI_QUERY_ENGINE_TYPE=binary \ + PRISMA_OFFLINE_MODE=true # Copy only what runtime needs. The application is installed inside the venv; # the rest of the builder's /app is source and build metadata that must not @@ -115,16 +121,18 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr # enterprise.enterprise_hooks from it) COPY --from=builder /app/enterprise /app/enterprise COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras -# Prisma binaries live in $HOME/.cache (default prisma-python location), -# which is /root/.cache here. Copy only the Prisma subdirs — copying the -# whole /root/.cache drags in the uv build cache (~660 MB, includes a -# setuptools wheel that surfaces as a CVE finding even though it's not -# on the runtime sys.path). -COPY --from=builder /root/.cache/prisma /root/.cache/prisma -COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python +# Prisma CLI + engines are baked under /opt/prisma, a fixed path every +# runtime uid can read and that no cache volume mount shadows. The paths are +# pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and recorded into the +# generated client at build time, so `prisma migrate deploy` on a fresh +# database needs no npm and no network access (#33650, #24554). +COPY --from=builder /opt/prisma /opt/prisma RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ - find /app/.venv -type d -path "*/tornado/test" -delete + find /app/.venv -type d -path "*/tornado/test" -delete && \ + chmod -R a+rX /opt/prisma && \ + test -x /opt/prisma/binaries/node_modules/.bin/prisma && \ + test -f /opt/prisma/binaries/node_modules/prisma/build/index.js EXPOSE 4000/tcp diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 868b6682276..34c9c606991 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -84,7 +84,9 @@ RUN uv sync --frozen --no-default-groups --no-editable \ --extra semantic-router \ --python python3 -RUN prisma generate --schema=./schema.prisma +RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ + npm_config_cache=/root/.npm \ + prisma generate --schema=./schema.prisma RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh @@ -97,7 +99,11 @@ USER root RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile WORKDIR /app -ENV PATH="/app/.venv/bin:${PATH}" +ENV PATH="/app/.venv/bin:${PATH}" \ + PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ + PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \ + PRISMA_CLI_QUERY_ENGINE_TYPE=binary \ + PRISMA_OFFLINE_MODE=true # Copy only what runtime needs. The application is installed inside the venv; # the rest of the builder's /app is source and build metadata that must not @@ -112,16 +118,20 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr # enterprise.enterprise_hooks from it) COPY --from=builder /app/enterprise /app/enterprise COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras -# Prisma binaries live in $HOME/.cache (default prisma-python location), -# which is /root/.cache here. Copy them from the builder so they survive -# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem -# + emptyDir) — otherwise the mount would shadow the baked-in query engine. -# Only the Prisma subdirs: the whole /root/.cache drags in the uv build cache. -COPY --from=builder /root/.cache/prisma /root/.cache/prisma -COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python +# Prisma CLI + engines are baked under /opt/prisma, a fixed path every +# runtime uid can read and that no cache volume mount shadows (unlike +# /app/.cache or $HOME/.cache in readOnlyRootFilesystem + emptyDir setups). +# The paths are pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and +# recorded into the generated client at build time, so `prisma migrate +# deploy` on a fresh database needs no npm and no network access +# (#33650, #24554). +COPY --from=builder /opt/prisma /opt/prisma RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ - find /app/.venv -type d -path "*/tornado/test" -delete + find /app/.venv -type d -path "*/tornado/test" -delete && \ + chmod -R a+rX /opt/prisma && \ + test -x /opt/prisma/binaries/node_modules/.bin/prisma && \ + test -f /opt/prisma/binaries/node_modules/prisma/build/index.js EXPOSE 4000/tcp diff --git a/litellm/router.py b/litellm/router.py index ced6090f82a..138ba7ebdbc 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8362,7 +8362,8 @@ class Router: Return (max_input_tokens, max_output_tokens) explicitly configured in a concrete deployment's model_info for model_name, via O(1) index lookup. - Returns (None, None) for wildcard-expanded or unknown names. Unlike + Returns (None, None) for wildcard-expanded or unknown names, and treats a + malformed configured value as absent rather than failing the listing. Unlike get_model_group_info, this never triggers pattern matching or deep copies, so it is safe to call per listed model on the /v1/models hot path. """ @@ -8370,12 +8371,18 @@ class Router: if deployment is None: return (None, None) + def _as_int(value: object) -> "int | None": + if value is None or isinstance(value, bool): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + model_info = deployment.model_info - max_input = model_info.get("max_input_tokens") - max_output = model_info.get("max_output_tokens") return ( - int(max_input) if max_input is not None else None, - int(max_output) if max_output is not None else None, + _as_int(model_info.get("max_input_tokens")), + _as_int(model_info.get("max_output_tokens")), ) def get_deployment_credentials_with_provider(self, model_id: str) -> Optional[Dict[str, Any]]: diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index d2bdb1764a4..9486646ea4a 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -556,6 +556,31 @@ def test_create_model_info_response_deployment_limits_override_cost_map(): assert response["max_output_tokens"] == 16384 +def test_create_model_info_response_survives_malformed_configured_limits(): + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "bad-limit-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"max_input_tokens": "128,000"}, + } + ] + ) + + response = create_model_info_response( + model_id="bad-limit-model", + provider="openai", + llm_router=router, + get_model_info=_raise_unmapped, + ) + + assert response["id"] == "bad-limit-model" + assert "max_input_tokens" not in response + assert "max_output_tokens" not in response + + def test_create_model_info_response_emits_integer_token_counts(): response = create_model_info_response( model_id="some-model", diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 3fca5aa6901..3d14d7bf927 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -5351,3 +5351,34 @@ def test_get_configured_token_limits_skips_wildcard_pattern_matching(): assert router.get_configured_token_limits( "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" ) == (None, None) + + +def test_get_configured_token_limits_treats_malformed_values_as_absent(): + malformed = ["", "unlimited", "128,000", [128000], {"max": 128000}, True] + router = litellm.Router( + model_list=[ + { + "model_name": f"bad-limit-{i}", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"max_input_tokens": bad, "max_output_tokens": bad}, + } + for i, bad in enumerate(malformed) + ] + ) + + for i in range(len(malformed)): + assert router.get_configured_token_limits(f"bad-limit-{i}") == (None, None) + + +def test_get_configured_token_limits_coerces_numeric_strings(): + router = litellm.Router( + model_list=[ + { + "model_name": "quoted-limits-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"max_input_tokens": "32000", "max_output_tokens": "8000"}, + } + ] + ) + + assert router.get_configured_token_limits("quoted-limits-model") == (32000, 8000)