mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-15 23:31:29 +00:00
Once the active Prisma client is in the disconnected state, every DB call raises ClientNotConnectedError. The reconnect machinery was supposed to recover from this, but _get_engine_pid() inspected the broken client via prisma's _engine property, which re-raises that same error, so recreate_prisma_client failed before it could build a replacement client and the proxy looped on failed reconnects forever (issue #28322 showed 1486+ consecutive failures over 30 days with zero recoveries) Guard both _get_engine_pid implementations with is_connected() so a disconnected client reads as "no engine" (pid 0) and the recreate path proceeds to construct and connect a fresh client
195 lines
6.5 KiB
Python
195 lines
6.5 KiB
Python
import json
|
|
import os
|
|
import signal
|
|
import sys
|
|
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
sys.path.insert(
|
|
0, os.path.abspath("../../../..")
|
|
) # Adds the parent directory to the system path
|
|
|
|
|
|
from litellm.proxy.db.prisma_client import PrismaWrapper, should_update_prisma_schema
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def mock_prisma_binary():
|
|
"""Mock prisma.Prisma to avoid requiring generated Prisma binaries for unit tests."""
|
|
mock_module = MagicMock()
|
|
with patch.dict(sys.modules, {"prisma": mock_module}):
|
|
yield mock_module
|
|
|
|
|
|
def test_should_update_prisma_schema(monkeypatch):
|
|
# CASE 1: Environment variable behavior
|
|
# When DISABLE_SCHEMA_UPDATE is not set -> should update
|
|
monkeypatch.setenv("DISABLE_SCHEMA_UPDATE", None)
|
|
assert should_update_prisma_schema() == True
|
|
|
|
# When DISABLE_SCHEMA_UPDATE="true" -> should not update
|
|
monkeypatch.setenv("DISABLE_SCHEMA_UPDATE", "true")
|
|
assert should_update_prisma_schema() == False
|
|
|
|
# When DISABLE_SCHEMA_UPDATE="false" -> should update
|
|
monkeypatch.setenv("DISABLE_SCHEMA_UPDATE", "false")
|
|
assert should_update_prisma_schema() == True
|
|
|
|
# CASE 2: Explicit parameter behavior (overrides env var)
|
|
monkeypatch.setenv("DISABLE_SCHEMA_UPDATE", None)
|
|
assert should_update_prisma_schema(True) == False # Param True -> should not update
|
|
|
|
monkeypatch.setenv("DISABLE_SCHEMA_UPDATE", None) # Set env var opposite to param
|
|
assert should_update_prisma_schema(False) == True # Param False -> should update
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_recreate_prisma_client_successful_disconnect():
|
|
"""
|
|
Test that recreate_prisma_client works normally when disconnect succeeds.
|
|
"""
|
|
# Mock the original prisma client
|
|
mock_prisma = AsyncMock()
|
|
|
|
# Create a mock PrismaWrapper instance
|
|
wrapper = Mock()
|
|
wrapper._original_prisma = mock_prisma
|
|
|
|
# Configure disconnect to succeed
|
|
mock_prisma.disconnect.return_value = None
|
|
|
|
# Mock the entire recreate_prisma_client method to avoid import issues
|
|
async def mock_recreate_prisma_client(new_db_url: str, http_client=None):
|
|
try:
|
|
await mock_prisma.disconnect()
|
|
except Exception:
|
|
pass
|
|
|
|
mock_new_prisma = AsyncMock()
|
|
wrapper._original_prisma = mock_new_prisma
|
|
await mock_new_prisma.connect()
|
|
|
|
# Assign the mock method to the wrapper
|
|
wrapper.recreate_prisma_client = mock_recreate_prisma_client
|
|
|
|
# Call the method
|
|
await wrapper.recreate_prisma_client("postgresql://new:new@localhost:5432/new")
|
|
|
|
# Verify that disconnect was called
|
|
mock_prisma.disconnect.assert_called_once()
|
|
|
|
# Verify that the new client replaced the original
|
|
assert wrapper._original_prisma != mock_prisma
|
|
assert hasattr(wrapper._original_prisma, "connect")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_recreate_prisma_client_kills_old_engine_on_disconnect_failure(
|
|
mock_prisma_binary,
|
|
):
|
|
"""When disconnect() fails, recreate_prisma_client must SIGTERM/SIGKILL the old engine PID."""
|
|
mock_prisma = AsyncMock()
|
|
mock_prisma.disconnect.side_effect = Exception("engine hung")
|
|
mock_prisma.is_connected = MagicMock(return_value=True)
|
|
|
|
# Simulate engine subprocess with a known PID
|
|
mock_engine = MagicMock()
|
|
mock_engine.process.pid = 12345
|
|
mock_prisma._engine = mock_engine
|
|
|
|
wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=False)
|
|
|
|
# Configure the mock Prisma constructor
|
|
mock_new_prisma = AsyncMock()
|
|
mock_prisma_binary.Prisma.return_value = mock_new_prisma
|
|
|
|
with (
|
|
patch("os.kill") as mock_kill,
|
|
patch("asyncio.sleep", new_callable=AsyncMock),
|
|
):
|
|
await wrapper.recreate_prisma_client("postgresql://new")
|
|
|
|
# Verify old engine was killed
|
|
mock_kill.assert_any_call(12345, signal.SIGTERM)
|
|
# Verify new client was created and connected
|
|
mock_new_prisma.connect.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_recreate_prisma_client_skips_kill_on_successful_disconnect(
|
|
mock_prisma_binary,
|
|
):
|
|
"""When disconnect() succeeds, no kill should be attempted."""
|
|
mock_prisma = AsyncMock()
|
|
mock_prisma.is_connected = MagicMock(return_value=True)
|
|
mock_prisma.disconnect.return_value = None
|
|
|
|
wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=False)
|
|
|
|
mock_new_prisma = AsyncMock()
|
|
mock_prisma_binary.Prisma.return_value = mock_new_prisma
|
|
|
|
with patch("os.kill") as mock_kill:
|
|
await wrapper.recreate_prisma_client("postgresql://new")
|
|
|
|
mock_kill.assert_not_called()
|
|
mock_new_prisma.connect.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_recreate_prisma_client_handles_missing_engine_pid(
|
|
mock_prisma_binary,
|
|
):
|
|
"""When engine PID is unavailable (no _engine attr), kill is skipped gracefully."""
|
|
mock_prisma = AsyncMock()
|
|
mock_prisma.is_connected = MagicMock(return_value=True)
|
|
mock_prisma.disconnect.side_effect = Exception("engine hung")
|
|
mock_prisma._engine = None # No engine subprocess
|
|
|
|
wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=False)
|
|
|
|
mock_new_prisma = AsyncMock()
|
|
mock_prisma_binary.Prisma.return_value = mock_new_prisma
|
|
|
|
with (
|
|
patch("os.kill") as mock_kill,
|
|
patch("asyncio.sleep", new_callable=AsyncMock),
|
|
):
|
|
await wrapper.recreate_prisma_client("postgresql://new")
|
|
|
|
mock_kill.assert_not_called() # PID was 0, kill skipped
|
|
mock_new_prisma.connect.assert_awaited_once()
|
|
|
|
|
|
def test_get_engine_pid_returns_zero_for_disconnected_client(disconnected_prisma):
|
|
"""A disconnected client must read as "no engine" instead of raising,
|
|
otherwise the reconnect path can never recover."""
|
|
wrapper = PrismaWrapper(
|
|
original_prisma=disconnected_prisma, iam_token_db_auth=False
|
|
)
|
|
|
|
assert wrapper._get_engine_pid() == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_recreate_prisma_client_recovers_from_disconnected_client(
|
|
mock_prisma_binary, disconnected_prisma
|
|
):
|
|
"""recreate_prisma_client must still build a replacement client when the
|
|
current one is disconnected."""
|
|
wrapper = PrismaWrapper(
|
|
original_prisma=disconnected_prisma, iam_token_db_auth=False
|
|
)
|
|
|
|
mock_new_prisma = AsyncMock()
|
|
mock_prisma_binary.Prisma.return_value = mock_new_prisma
|
|
|
|
with patch("os.kill") as mock_kill:
|
|
result = await wrapper.recreate_prisma_client("postgresql://new")
|
|
|
|
assert result is True
|
|
mock_kill.assert_not_called()
|
|
assert wrapper._original_prisma is mock_new_prisma
|
|
mock_new_prisma.connect.assert_awaited_once()
|