mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
fix(docker): match USE_DDTRACE case-insensitively and route build_from_pip through prod_entrypoint.sh (#39344)
* fix(docker): match USE_DDTRACE case-insensitively and route build_from_pip through prod_entrypoint.sh Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(docker): run build_from_pip ENTRYPOINT and CMD through the shipped prod_entrypoint.sh Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin <yassin@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
35d20468cd
commit
24666d8dbf
6 changed files with 95 additions and 26 deletions
|
|
@ -55,7 +55,10 @@ RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/
|
|||
|
||||
ENV PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
|
||||
|
||||
COPY docker/prod_entrypoint.sh /app/docker/prod_entrypoint.sh
|
||||
RUN sed -i 's/\r$//' /app/docker/prod_entrypoint.sh && chmod +x /app/docker/prod_entrypoint.sh
|
||||
|
||||
EXPOSE 4000/tcp
|
||||
|
||||
ENTRYPOINT ["litellm"]
|
||||
ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]
|
||||
CMD ["--port", "4000"]
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
#!/bin/sh
|
||||
|
||||
if [ "$USE_DDTRACE" = "true" ]; then
|
||||
export DD_TRACE_OPENAI_ENABLED="False"
|
||||
exec ddtrace-run "$@"
|
||||
fi
|
||||
case "$USE_DDTRACE" in
|
||||
[Tt][Rr][Uu][Ee])
|
||||
export DD_TRACE_OPENAI_ENABLED="False"
|
||||
exec ddtrace-run "$@"
|
||||
;;
|
||||
esac
|
||||
|
||||
exec "$@"
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
#!/bin/sh
|
||||
|
||||
if [ "$USE_DDTRACE" = "true" ]; then
|
||||
export DD_TRACE_OPENAI_ENABLED="False"
|
||||
exec ddtrace-run litellm "$@"
|
||||
else
|
||||
exec litellm "$@"
|
||||
fi
|
||||
case "$USE_DDTRACE" in
|
||||
[Tt][Rr][Uu][Ee])
|
||||
export DD_TRACE_OPENAI_ENABLED="False"
|
||||
exec ddtrace-run litellm "$@"
|
||||
;;
|
||||
esac
|
||||
|
||||
exec litellm "$@"
|
||||
|
|
|
|||
|
|
@ -214,8 +214,8 @@ locals {
|
|||
gateway_uvicorn_args = "--host 0.0.0.0 --port 4000 --workers ${var.gateway_num_workers}"
|
||||
backend_uvicorn_args = "--host 0.0.0.0 --port 4001"
|
||||
|
||||
gateway_launch_cmd = "if [ \"$USE_DDTRACE\" = \"true\" ]; then export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn gateway.main:app ${local.gateway_uvicorn_args}; else exec uvicorn gateway.main:app ${local.gateway_uvicorn_args}; fi"
|
||||
backend_launch_cmd = "if [ \"$USE_DDTRACE\" = \"true\" ]; then export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn backend.main:app ${local.backend_uvicorn_args}; else exec uvicorn backend.main:app ${local.backend_uvicorn_args}; fi"
|
||||
gateway_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn gateway.main:app ${local.gateway_uvicorn_args};; *) exec uvicorn gateway.main:app ${local.gateway_uvicorn_args};; esac"
|
||||
backend_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn backend.main:app ${local.backend_uvicorn_args};; *) exec uvicorn backend.main:app ${local.backend_uvicorn_args};; esac"
|
||||
|
||||
gateway_proxy_overrides = local.proxy_config_enabled ? {
|
||||
entryPoint = ["sh", "-c"]
|
||||
|
|
|
|||
|
|
@ -138,8 +138,8 @@ locals {
|
|||
gateway_uvicorn_args = "--host 0.0.0.0 --port 4000 --workers ${var.gateway_num_workers}"
|
||||
backend_uvicorn_args = "--host 0.0.0.0 --port 4001"
|
||||
|
||||
gateway_launch_cmd = "if [ \"$USE_DDTRACE\" = \"true\" ]; then export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn gateway.main:app ${local.gateway_uvicorn_args}; else exec uvicorn gateway.main:app ${local.gateway_uvicorn_args}; fi"
|
||||
backend_launch_cmd = "if [ \"$USE_DDTRACE\" = \"true\" ]; then export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn backend.main:app ${local.backend_uvicorn_args}; else exec uvicorn backend.main:app ${local.backend_uvicorn_args}; fi"
|
||||
gateway_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn gateway.main:app ${local.gateway_uvicorn_args};; *) exec uvicorn gateway.main:app ${local.gateway_uvicorn_args};; esac"
|
||||
backend_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn backend.main:app ${local.backend_uvicorn_args};; *) exec uvicorn backend.main:app ${local.backend_uvicorn_args};; esac"
|
||||
|
||||
gateway_args = join(" && ", concat(
|
||||
local.redis_ca_fragment,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import re
|
|||
import stat
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -16,11 +15,15 @@ COMPONENT_ENTRYPOINT = REPO_ROOT / "docker" / "component_entrypoint.sh"
|
|||
PROD_ENTRYPOINT = REPO_ROOT / "docker" / "prod_entrypoint.sh"
|
||||
GATEWAY_DOCKERFILE = REPO_ROOT / "gateway" / "Dockerfile"
|
||||
BACKEND_DOCKERFILE = REPO_ROOT / "backend" / "Dockerfile"
|
||||
BUILD_FROM_PIP_DOCKERFILE = REPO_ROOT / "docker" / "build_from_pip" / "Dockerfile.build_from_pip"
|
||||
TERRAFORM_ECS = REPO_ROOT / "terraform" / "litellm" / "aws" / "ecs.tf"
|
||||
TERRAFORM_CLOUDRUN = REPO_ROOT / "terraform" / "litellm" / "gcp" / "cloudrun.tf"
|
||||
|
||||
IMAGE_ENTRYPOINT_PATH = "/app/docker/component_entrypoint.sh"
|
||||
|
||||
TRUTHY_USE_DDTRACE = ("true", "True", "TRUE", "tRuE")
|
||||
FALSY_USE_DDTRACE = (None, "", "false", "False", "1", "yes", "on", "truex")
|
||||
|
||||
PYTHONPATH_SENTINEL = "/lit-entrypoint-sentinel:/app"
|
||||
|
||||
_STUB_TEMPLATE = """#!/bin/sh
|
||||
|
|
@ -34,6 +37,7 @@ _STUB_TEMPLATE = """#!/bin/sh
|
|||
|
||||
_ENTRYPOINT_RE = re.compile(r"^ENTRYPOINT\s+(\[.*\])\s*$", re.MULTILINE)
|
||||
_CMD_RE = re.compile(r"^CMD\s+(\[.*\])\s*$", re.MULTILINE)
|
||||
_COPY_RE = re.compile(r"^COPY\s+(?!--from)(\S+)\s+(\S+)\s*$", re.MULTILINE)
|
||||
_APP_TARGET_RE = re.compile(r"(?:gateway|backend)\.main:app")
|
||||
_TF_STRING_LOCAL_RE = re.compile(r'^\s*(\w+)\s*=\s*"((?:[^"\\]|\\.)*)"\s*$', re.MULTILINE)
|
||||
_TF_INTERPOLATION_RE = re.compile(r"\$\{(local|var)\.(\w+)\}")
|
||||
|
|
@ -53,7 +57,7 @@ def _write_stubs(bin_dir: Path, names: tuple[str, ...]) -> None:
|
|||
def _run_entrypoint(
|
||||
script: Path,
|
||||
argv: tuple[str, ...],
|
||||
use_ddtrace: Optional[str],
|
||||
use_ddtrace: str | None,
|
||||
tmp_path: Path,
|
||||
) -> tuple[str, ...]:
|
||||
"""Run `script` with stubbed executables on PATH and return the recorded lines."""
|
||||
|
|
@ -84,7 +88,7 @@ def _run_entrypoint(
|
|||
return tuple(record.read_text().splitlines()) if record.exists() else ()
|
||||
|
||||
|
||||
def _run_shell_command(command: str, bin_dir: Path, record: Path, use_ddtrace: Optional[str]) -> tuple[str, ...]:
|
||||
def _run_shell_command(command: str, bin_dir: Path, record: Path, use_ddtrace: str | None) -> tuple[str, ...]:
|
||||
"""Run a resolved Terraform launch command through `sh -c` and return the recorded lines."""
|
||||
env = {
|
||||
**os.environ,
|
||||
|
|
@ -184,9 +188,19 @@ def test_ddtrace_disabled_execs_the_command_directly(tmp_path: Path) -> None:
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_ddtrace", [None, "", "false", "True", "TRUE", "1", "yes"])
|
||||
def test_gating_matches_the_monolithic_entrypoint(use_ddtrace: Optional[str], tmp_path: Path) -> None:
|
||||
"""The componentized images must honor `USE_DDTRACE` exactly as the monolith does."""
|
||||
@pytest.mark.parametrize(
|
||||
"use_ddtrace, traced",
|
||||
[*((v, True) for v in TRUTHY_USE_DDTRACE), *((v, False) for v in FALSY_USE_DDTRACE)],
|
||||
)
|
||||
def test_gating_matches_the_monolithic_entrypoint_and_get_secret_bool(
|
||||
use_ddtrace: str | None, traced: bool, tmp_path: Path
|
||||
) -> None:
|
||||
"""Both entrypoints must accept exactly the spellings `get_secret_bool` accepts.
|
||||
|
||||
`ProxyStartupEvent._init_dd_tracer` reads `USE_DDTRACE` through `get_secret_bool`, which
|
||||
matches `true` case-insensitively. If the shell gate were stricter, `USE_DDTRACE=True` would
|
||||
give in-process LLM spans without `ddtrace-run` HTTP spans, a half-enabled state.
|
||||
"""
|
||||
component = _run_entrypoint(
|
||||
COMPONENT_ENTRYPOINT,
|
||||
("uvicorn", "gateway.main:app"),
|
||||
|
|
@ -200,8 +214,56 @@ def test_gating_matches_the_monolithic_entrypoint(use_ddtrace: Optional[str], tm
|
|||
tmp_path=tmp_path / "monolith",
|
||||
)
|
||||
|
||||
assert component[0].startswith("exec=") and monolith[0].startswith("exec=")
|
||||
assert (component[0] == "exec=ddtrace-run") == (monolith[0] == "exec=ddtrace-run")
|
||||
expected_exec = "exec=ddtrace-run" if traced else "exec=uvicorn"
|
||||
expected_openai = "DD_TRACE_OPENAI_ENABLED=False" if traced else "DD_TRACE_OPENAI_ENABLED=<unset>"
|
||||
assert component[0] == expected_exec
|
||||
assert component[2] == expected_openai
|
||||
assert monolith[0] == ("exec=ddtrace-run" if traced else "exec=litellm")
|
||||
assert monolith[2] == expected_openai
|
||||
assert monolith[1] == ("args=litellm --port 4000" if traced else "args=--port 4000")
|
||||
|
||||
|
||||
def _copied_script(dockerfile: Path, image_path: str) -> Path:
|
||||
"""Resolve the repo file a Dockerfile `COPY`s to `image_path`, so tests run what the image ships."""
|
||||
matches = _COPY_RE.findall(dockerfile.read_text())
|
||||
sources = tuple(src for src, dst in matches if dst == image_path)
|
||||
assert sources, f"{dockerfile} never COPYs anything to {image_path}"
|
||||
source = REPO_ROOT / sources[-1]
|
||||
assert source.is_file(), f"{dockerfile} COPYs {sources[-1]}, which does not exist in the build context"
|
||||
return source
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"use_ddtrace, traced",
|
||||
[*((v, True) for v in TRUTHY_USE_DDTRACE), *((v, False) for v in FALSY_USE_DDTRACE)],
|
||||
)
|
||||
def test_build_from_pip_image_launches_litellm_through_the_prod_entrypoint(
|
||||
use_ddtrace: str | None, traced: bool, tmp_path: Path
|
||||
) -> None:
|
||||
"""Run the build_from_pip image's ENTRYPOINT + CMD through the script it actually COPYs.
|
||||
|
||||
The image used to `ENTRYPOINT ["litellm"]`, so `USE_DDTRACE` was inert there at any spelling.
|
||||
Resolving the ENTRYPOINT path back to its COPY source and executing it with the Dockerfile's
|
||||
CMD checks the launch the container performs, not just that the Dockerfile mentions the script.
|
||||
"""
|
||||
entrypoint = _entrypoint_argv(BUILD_FROM_PIP_DOCKERFILE)
|
||||
assert len(entrypoint) == 1, (
|
||||
f"{BUILD_FROM_PIP_DOCKERFILE} ENTRYPOINT must be the bare script so CMD reaches litellm"
|
||||
)
|
||||
script = _copied_script(BUILD_FROM_PIP_DOCKERFILE, entrypoint[0])
|
||||
assert script == PROD_ENTRYPOINT, f"{BUILD_FROM_PIP_DOCKERFILE} bypasses the ddtrace-aware entrypoint"
|
||||
assert f"chmod +x {entrypoint[0]}" in BUILD_FROM_PIP_DOCKERFILE.read_text()
|
||||
|
||||
cmd = _cmd_argv(BUILD_FROM_PIP_DOCKERFILE)
|
||||
recorded = _run_entrypoint(script, cmd, use_ddtrace=use_ddtrace, tmp_path=tmp_path)
|
||||
|
||||
cmd_str = " ".join(cmd)
|
||||
assert recorded == (
|
||||
"exec=ddtrace-run" if traced else "exec=litellm",
|
||||
f"args=litellm {cmd_str}" if traced else f"args={cmd_str}",
|
||||
"DD_TRACE_OPENAI_ENABLED=False" if traced else "DD_TRACE_OPENAI_ENABLED=<unset>",
|
||||
f"PYTHONPATH={PYTHONPATH_SENTINEL}",
|
||||
)
|
||||
|
||||
|
||||
def test_entrypoint_script_is_executable() -> None:
|
||||
|
|
@ -239,9 +301,9 @@ def test_component_images_make_the_entrypoint_executable(dockerfile: Path) -> No
|
|||
|
||||
@pytest.mark.parametrize("terraform_file", TERRAFORM_LAUNCH_SITES, ids=lambda p: p.parent.name)
|
||||
@pytest.mark.parametrize("component", ["gateway", "backend"])
|
||||
@pytest.mark.parametrize("use_ddtrace", [None, "true", "false", "True"])
|
||||
@pytest.mark.parametrize("use_ddtrace", [*TRUTHY_USE_DDTRACE, *FALSY_USE_DDTRACE])
|
||||
def test_terraform_launch_command_matches_the_script_contract(
|
||||
terraform_file: Path, component: str, use_ddtrace: Optional[str], tmp_path: Path
|
||||
terraform_file: Path, component: str, use_ddtrace: str | None, tmp_path: Path
|
||||
) -> None:
|
||||
"""The Terraform command and `docker/component_entrypoint.sh` must decide identically.
|
||||
|
||||
|
|
@ -273,7 +335,7 @@ def test_terraform_launch_command_matches_the_script_contract(
|
|||
assert from_terraform[2] == from_script[2], f"{terraform_file} disagrees with the script on the openai integration"
|
||||
assert app_target in from_terraform[1]
|
||||
|
||||
if use_ddtrace == "true":
|
||||
if use_ddtrace in TRUTHY_USE_DDTRACE:
|
||||
assert from_terraform[0] == "exec=ddtrace-run"
|
||||
assert from_terraform[2] == "DD_TRACE_OPENAI_ENABLED=False"
|
||||
else:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue