feat(proxy): start mock chat API once in main process on port 8090

- Add ProxyInitializationHelpers._start_mock_chat_api_once() in proxy_cli
- Start mock (chat_completions_api.app) in a daemon thread on 8090 before forking workers
- Remove mock startup from proxy_server lifespan so workers never start it
- Single mock instance regardless of --num_workers; no port conflict
This commit is contained in:
Alexsander Hamir 2026-02-02 12:51:31 -08:00
parent dae2ffb070
commit dc2db9ca21
2 changed files with 35 additions and 0 deletions

View file

@ -5,6 +5,7 @@ import os
import random
import subprocess
import sys
import threading
import urllib.parse as urlparse
from typing import TYPE_CHECKING, Any, Optional, Union
@ -116,6 +117,35 @@ class ProxyInitializationHelpers:
)
print(completion_response) # noqa
@staticmethod
def _start_mock_chat_api_once() -> None:
"""Start the mock chat completions API on port 8090 once in the main process (for testing)."""
_proxy_cli_dir = os.path.dirname(os.path.abspath(__file__))
_litellm_root = os.path.dirname(_proxy_cli_dir)
_repo_root = os.path.dirname(_litellm_root)
if _repo_root not in sys.path:
sys.path.insert(0, _repo_root)
try:
from chat_completions_api import app as mock_chat_app
except ImportError:
return
def _run_mock():
import uvicorn
uvicorn.run(
mock_chat_app,
host="0.0.0.0",
port=8090,
log_level="warning",
)
t = threading.Thread(target=_run_mock, daemon=True)
t.start()
print( # noqa: T201
"\033[1;32mLiteLLM: Mock chat completions API started on http://0.0.0.0:8090 (for testing)\033[0m\n"
)
@staticmethod
def _get_default_unvicorn_init_args(
host: str,
@ -826,6 +856,9 @@ def run_server( # noqa: PLR0915
)
return
# Start mock chat API once in main process (port 8090) before forking workers
ProxyInitializationHelpers._start_mock_chat_api_once()
uvicorn_args = ProxyInitializationHelpers._get_default_unvicorn_init_args(
host=host,
port=port,

View file

@ -11598,3 +11598,5 @@ async def dynamic_mcp_route(mcp_server_name: str, request: Request):
app.mount(path=BASE_MCP_ROUTE, app=mcp_app)
app.include_router(mcp_rest_endpoints_router)
app.include_router(mcp_discoverable_endpoints_router)
# Mock chat completions API (for testing) is started once on port 8090 from proxy_cli._start_mock_chat_api_once()