diff --git a/tests/test_litellm/proxy/prompts/test_prompt_registry.py b/tests/test_litellm/proxy/prompts/test_prompt_registry.py index 20715f8e0fc..8899f183133 100644 --- a/tests/test_litellm/proxy/prompts/test_prompt_registry.py +++ b/tests/test_litellm/proxy/prompts/test_prompt_registry.py @@ -1,3 +1,5 @@ +import asyncio +from pathlib import Path from typing import Final import pytest @@ -367,3 +369,386 @@ def test_restore_fallback_prompt_state_allowlist() -> None: assert kwargs["prompt_environment"] == "staging" assert "malicious_key" not in kwargs assert "_in_prompt_factory" not in kwargs + + +def test_prompt_manager_version_indexing_with_prompt_file(tmp_path: Path) -> None: + from litellm.integrations.dotprompt.prompt_manager import PromptManager + + prompt_path: Final = tmp_path / "greeting_test.prompt" + prompt_path.write_text("---\nmodel: gpt-4o\n---\nHello {{name}}") + + manager: Final = PromptManager( + prompt_id="greeting_test", + prompt_file=str(prompt_path), + prompt_version=2, + ) + assert manager.get_prompt("greeting_test", version=2) is not None + assert manager.get_prompt("greeting_test.v2") is not None + assert manager.get_prompt("greeting_test_v2") is not None + + +def test_dotprompt_manager_async_get_chat_completion_prompt() -> None: + from datetime import datetime + + from litellm.integrations.dotprompt.dotprompt_manager import DotpromptManager + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.types.utils import StandardCallbackDynamicParams + + dot_mgr: Final = DotpromptManager( + prompt_data={"greeting": {"content": "Hello {{name}}"}}, + prompt_id="greeting", + prompt_version=2, + ) + dynamic_params: Final = StandardCallbackDynamicParams() + logging_obj: Final = Logging( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="acompletion", + litellm_call_id="call-async", + start_time=datetime.now(), + function_id="fn-async", + ) + result: Final = asyncio.run( + dot_mgr.async_get_chat_completion_prompt( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + non_default_params={}, + prompt_id="greeting", + prompt_variables={"name": "Bob"}, + dynamic_callback_params=dynamic_params, + litellm_logging_obj=logging_obj, + ) + ) + rendered_messages: Final = result[1] + assert any("Bob" in str(msg.get("content", "")) for msg in rendered_messages) + + +def test_logging_prompt_management_error_and_mismatch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from datetime import datetime + + from litellm.integrations.dotprompt.dotprompt_manager import DotpromptManager + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY + + def mock_broken_resolve(*args: object, **kwargs: object) -> None: + raise RuntimeError("simulated registry failure") + + monkeypatch.setattr(IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", mock_broken_resolve) + + logging_obj: Final = Logging( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="acompletion", + litellm_call_id="call-err", + start_time=datetime.now(), + function_id="fn-err", + ) + + detected: Final = logging_obj._auto_detect_prompt_management_logger( + prompt_id="unknown_prompt", + prompt_spec=None, + dynamic_callback_params=logging_obj.standard_callback_dynamic_params, + prompt_version=1, + ) + assert detected is None + + mismatched_logger: Final = DotpromptManager( + prompt_data={"test": {"content": "Test"}}, + prompt_id="test", + prompt_version=1, + ) + monkeypatch.setattr( + litellm.logging_callback_manager, + "get_custom_loggers_for_type", + lambda callback_type: [mismatched_logger], + ) + + res: Final = logging_obj.get_custom_logger_for_prompt_management( + model="gpt-4o", + non_default_params={}, + prompt_id="test", + dynamic_callback_params=logging_obj.standard_callback_dynamic_params, + prompt_version=2, + ) + assert res is None + + +def test_router_prompt_version_acompletion( + isolated_callbacks: list, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY + from litellm.router import Router + + async def mock_acompletion(*args: object, **kwargs: object) -> litellm.ModelResponse: + return litellm.ModelResponse( + choices=[litellm.utils.Choices(message=litellm.utils.Message(content="router reply", role="assistant"))] + ) + + monkeypatch.setattr(litellm, "acompletion", mock_acompletion) + + IN_MEMORY_PROMPT_REGISTRY.initialize_prompt( + prompt=_db_prompt_spec(content="System: Hello\n\nUser: {{input}}", version=2) + ) + + router: Final = Router( + model_list=[ + { + "model_name": "p-model", + "litellm_params": { + "model": "dotprompt/gpt-4o", + "prompt_id": "greeting", + "prompt_version": 2, + }, + } + ] + ) + + response: Final = asyncio.run( + router.acompletion( + model="p-model", + messages=[{"role": "user", "content": "hi"}], + prompt_variables={"input": "world"}, + ) + ) + assert isinstance(response, litellm.ModelResponse) + assert response.choices[0].message.content == "router reply" + + +def test_router_async_function_with_fallbacks_prompt_management( + isolated_callbacks: list, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY + from litellm.router import Router + + async def mock_acompletion(*args: object, **kwargs: object) -> litellm.ModelResponse: + return litellm.ModelResponse( + choices=[litellm.utils.Choices(message=litellm.utils.Message(content="fallback reply", role="assistant"))] + ) + + monkeypatch.setattr(litellm, "acompletion", mock_acompletion) + + IN_MEMORY_PROMPT_REGISTRY.initialize_prompt( + prompt=_db_prompt_spec(content="System: Hello\n\nUser: {{input}}", version=1) + ) + + router: Final = Router( + model_list=[ + { + "model_name": "pm-model", + "litellm_params": { + "model": "dotprompt/gpt-4o", + "prompt_id": "greeting", + "prompt_version": 1, + }, + } + ] + ) + + res_kw: Final = asyncio.run( + router.async_function_with_fallbacks( + model="pm-model", + messages=[{"role": "user", "content": "hi"}], + original_function=router._acompletion, + ) + ) + assert isinstance(res_kw, litellm.ModelResponse) + + res_pos: Final = asyncio.run( + router.async_function_with_fallbacks( + [{"role": "user", "content": "hi"}], + model="pm-model", + original_function=router._acompletion, + ) + ) + assert isinstance(res_pos, litellm.ModelResponse) + + +def test_router_dispatch_prompt_completion_direct_and_fallbacks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: Final[list[str]] = [] + + async def mock_failover_acompletion(*args: object, **kwargs: object) -> litellm.ModelResponse: + model: Final = str(kwargs.get("model")) + calls.append(model) + if model == "unlisted-direct": + raise RuntimeError("primary direct failure") + return litellm.ModelResponse( + choices=[litellm.utils.Choices(message=litellm.utils.Message(content="failover ok", role="assistant"))] + ) + + monkeypatch.setattr(litellm, "acompletion", mock_failover_acompletion) + + from litellm.router import Router + + router: Final = Router( + model_list=[ + { + "model_name": "fallback-target", + "litellm_params": {"model": "openai/gpt-4o-fallback"}, + } + ], + fallbacks=[{"unlisted-direct": ["fallback-target"]}], + num_retries=0, + ) + + response: Final = asyncio.run( + router._dispatch_prompt_completion( + model="unlisted-direct", + original_model_name="unlisted-direct", + unrendered_messages=[{"role": "user", "content": "hi"}], + original_prompt_params={"prompt_id": "test"}, + original_function=router._acompletion, + prompt_management_params={"prompt_id", "prompt_version"}, + kwargs={"model": "unlisted-direct", "messages": [{"role": "user", "content": "hi"}], "metadata": {}}, + ) + ) + assert isinstance(response, litellm.ModelResponse) + assert response.choices[0].message.content == "failover ok" + assert len(calls) == 2 + + async def mock_success_acompletion(*args: object, **kwargs: object) -> litellm.ModelResponse: + return litellm.ModelResponse( + choices=[litellm.utils.Choices(message=litellm.utils.Message(content="direct success", role="assistant"))] + ) + + monkeypatch.setattr(litellm, "acompletion", mock_success_acompletion) + + direct_res: Final = asyncio.run( + router._dispatch_prompt_completion( + model="unlisted-direct-2", + original_model_name="unlisted-direct-2", + unrendered_messages=[{"role": "user", "content": "hi"}], + original_prompt_params={"prompt_id": "test"}, + original_function=router._acompletion, + prompt_management_params={"prompt_id", "prompt_version"}, + kwargs={"model": "unlisted-direct-2", "messages": [{"role": "user", "content": "hi"}]}, + ) + ) + assert isinstance(direct_res, litellm.ModelResponse) + assert direct_res.choices[0].message.content == "direct success" + + async def mock_error_acompletion(*args: object, **kwargs: object) -> litellm.ModelResponse: + raise RuntimeError("unhandled direct error") + + monkeypatch.setattr(litellm, "acompletion", mock_error_acompletion) + + empty_router: Final = Router(model_list=[]) + with pytest.raises(RuntimeError, match="unhandled direct error"): + asyncio.run( + empty_router._dispatch_prompt_completion( + model="unlisted-err", + original_model_name="unlisted-err", + unrendered_messages=[{"role": "user", "content": "hi"}], + original_prompt_params={"prompt_id": "test"}, + original_function=empty_router._acompletion, + prompt_management_params={"prompt_id", "prompt_version"}, + kwargs={"model": "unlisted-err", "messages": [{"role": "user", "content": "hi"}]}, + ) + ) + + +def test_router_prompt_management_factory_registry_lookup_exception( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY + from litellm.router import Router + + def mock_failing_resolve(*args: object, **kwargs: object) -> None: + raise RuntimeError("registry exception") + + monkeypatch.setattr(IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", mock_failing_resolve) + + async def mock_acompletion(*args: object, **kwargs: object) -> litellm.ModelResponse: + return litellm.ModelResponse( + choices=[litellm.utils.Choices(message=litellm.utils.Message(content="fallback reply", role="assistant"))] + ) + + monkeypatch.setattr(litellm, "acompletion", mock_acompletion) + + router: Final = Router( + model_list=[ + { + "model_name": "direct-model", + "litellm_params": { + "model": "gpt-4o", + "prompt_id": "test-prompt", + "prompt_version": 1, + }, + } + ] + ) + + response: Final = asyncio.run( + router._prompt_management_factory( + model="direct-model", + messages=[{"role": "user", "content": "hi"}], + kwargs={"model": "direct-model", "original_function": router._acompletion}, + ) + ) + assert isinstance(response, litellm.ModelResponse) + assert response.choices[0].message.content == "fallback reply" + + +def test_router_fallback_hop_restores_prompt_parameters_and_messages( + isolated_callbacks: list, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY + from litellm.router import Router + + IN_MEMORY_PROMPT_REGISTRY.initialize_prompt( + prompt=_db_prompt_spec(content="System: Primary V1\n\nUser: {{input}}", version=1) + ) + IN_MEMORY_PROMPT_REGISTRY.initialize_prompt( + prompt=_db_prompt_spec(content="System: Fallback V2\n\nUser: {{input}}", version=2) + ) + + calls: Final[list[dict]] = [] + + async def mock_acompletion(*args: object, **kwargs: object) -> litellm.ModelResponse: + calls.append(dict(kwargs)) + if len(calls) == 1: + raise RuntimeError("primary prompt failure") + return litellm.ModelResponse( + choices=[litellm.utils.Choices(message=litellm.utils.Message(content="hop ok", role="assistant"))] + ) + + monkeypatch.setattr(litellm, "acompletion", mock_acompletion) + + router: Final = Router( + model_list=[ + { + "model_name": "primary-pm", + "litellm_params": { + "model": "dotprompt/gpt-4o", + "prompt_id": "greeting", + "prompt_version": 1, + }, + }, + { + "model_name": "fallback-pm", + "litellm_params": { + "model": "dotprompt/gpt-4o", + "prompt_id": "greeting", + "prompt_version": 2, + }, + }, + ], + fallbacks=[{"primary-pm": ["fallback-pm"]}], + num_retries=0, + ) + + response: Final = asyncio.run( + router.acompletion( + model="primary-pm", + messages=[{"role": "user", "content": "hi"}], + prompt_variables={"input": "test-val"}, + ) + ) + assert isinstance(response, litellm.ModelResponse) + assert response.choices[0].message.content == "hop ok" + assert len(calls) == 2