mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-12 03:48:56 +00:00
feat(proxy): start mock chat API once in main process on port 8090
- Add ProxyInitializationHelpers._start_mock_chat_api_once() in proxy_cli - Start mock (chat_completions_api.app) in a daemon thread on 8090 before forking workers - Remove mock startup from proxy_server lifespan so workers never start it - Single mock instance regardless of --num_workers; no port conflict
This commit is contained in:
parent
dae2ffb070
commit
dc2db9ca21
2 changed files with 35 additions and 0 deletions
|
|
@ -5,6 +5,7 @@ import os
|
|||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import urllib.parse as urlparse
|
||||
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
|
||||
|
|
@ -116,6 +117,35 @@ class ProxyInitializationHelpers:
|
|||
)
|
||||
print(completion_response) # noqa
|
||||
|
||||
@staticmethod
|
||||
def _start_mock_chat_api_once() -> None:
|
||||
"""Start the mock chat completions API on port 8090 once in the main process (for testing)."""
|
||||
_proxy_cli_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
_litellm_root = os.path.dirname(_proxy_cli_dir)
|
||||
_repo_root = os.path.dirname(_litellm_root)
|
||||
if _repo_root not in sys.path:
|
||||
sys.path.insert(0, _repo_root)
|
||||
try:
|
||||
from chat_completions_api import app as mock_chat_app
|
||||
except ImportError:
|
||||
return
|
||||
|
||||
def _run_mock():
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(
|
||||
mock_chat_app,
|
||||
host="0.0.0.0",
|
||||
port=8090,
|
||||
log_level="warning",
|
||||
)
|
||||
|
||||
t = threading.Thread(target=_run_mock, daemon=True)
|
||||
t.start()
|
||||
print( # noqa: T201
|
||||
"\033[1;32mLiteLLM: Mock chat completions API started on http://0.0.0.0:8090 (for testing)\033[0m\n"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_default_unvicorn_init_args(
|
||||
host: str,
|
||||
|
|
@ -826,6 +856,9 @@ def run_server( # noqa: PLR0915
|
|||
)
|
||||
return
|
||||
|
||||
# Start mock chat API once in main process (port 8090) before forking workers
|
||||
ProxyInitializationHelpers._start_mock_chat_api_once()
|
||||
|
||||
uvicorn_args = ProxyInitializationHelpers._get_default_unvicorn_init_args(
|
||||
host=host,
|
||||
port=port,
|
||||
|
|
|
|||
|
|
@ -11598,3 +11598,5 @@ async def dynamic_mcp_route(mcp_server_name: str, request: Request):
|
|||
app.mount(path=BASE_MCP_ROUTE, app=mcp_app)
|
||||
app.include_router(mcp_rest_endpoints_router)
|
||||
app.include_router(mcp_discoverable_endpoints_router)
|
||||
|
||||
# Mock chat completions API (for testing) is started once on port 8090 from proxy_cli._start_mock_chat_api_once()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue