diff --git a/docker-compose.yml b/docker-compose.yml index 988860a7877..644efa8291b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,21 +4,21 @@ services: context: . args: target: runtime - image: docker.litellm.ai/berriai/litellm:main-stable - ######################################### - ## Uncomment these lines to start proxy with a config.yaml file ## - # volumes: - # - ./config.yaml:/app/config.yaml - # command: - # - "--config=/app/config.yaml" - ############################################## +# image: docker.litellm.ai/berriai/litellm:main-stable + volumes: + - ./config.yaml:/app/config.yaml # Mount your config file + command: + - "--config=/app/config.yaml" ports: - - "4000:4000" # Map the container port to the host, change the host port if necessary + - "4000:4000" # Main proxy port + - "4001:4001" # Separate health app port environment: DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm" STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI - env_file: - - .env # Load local .env file + SEPARATE_HEALTH_APP: "1" # Enable separate health check app + SEPARATE_HEALTH_PORT: "4001" # Port for health app (default is 4001) + # env_file: + # - .env # Optional: Load local .env file if it exists depends_on: - db # Indicates that this service depends on the 'db' service, ensuring 'db' starts first healthcheck: # Defines the health check configuration for the container diff --git a/steps_to_repro.md b/steps_to_repro.md new file mode 100644 index 00000000000..b75ad3c0e7e --- /dev/null +++ b/steps_to_repro.md @@ -0,0 +1,17 @@ +1. docker compose up +2. Start the fake LLM provider and make sure the delay is set to 10sec +3. Make the request, and wait 3 seconds: +``` bash +# Create the JSON file first +@' +{"model":"db-openai-endpoint","messages":[{"role":"user","content":"Say hello"}],"max_tokens":2000} +'@ | Out-File -FilePath request.json -Encoding utf8 + +# Then use it with curl +curl.exe -X POST http://localhost:4000/v1/chat/completions -H "Content-Type: application/json" -d "@request.json" +``` +4. On a different terminal run: +```bash +docker kill --signal=SIGTERM (docker ps --filter "name=litellm" --format "{{.Names}}" | Select-Object -First 1) +``` +5. Check if the call gets terminated or if LiteLLM waits the 10 seconds. \ No newline at end of file