mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
test(prompts): add prompt_file version indexing and router registry tests
This commit is contained in:
parent
dccd239d6a
commit
b3159b2c53
1 changed files with 385 additions and 0 deletions
|
|
@ -1,3 +1,5 @@
|
|||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
|
@ -367,3 +369,386 @@ def test_restore_fallback_prompt_state_allowlist() -> None:
|
|||
assert kwargs["prompt_environment"] == "staging"
|
||||
assert "malicious_key" not in kwargs
|
||||
assert "_in_prompt_factory" not in kwargs
|
||||
|
||||
|
||||
def test_prompt_manager_version_indexing_with_prompt_file(tmp_path: Path) -> None:
|
||||
from litellm.integrations.dotprompt.prompt_manager import PromptManager
|
||||
|
||||
prompt_path: Final = tmp_path / "greeting_test.prompt"
|
||||
prompt_path.write_text("---\nmodel: gpt-4o\n---\nHello {{name}}")
|
||||
|
||||
manager: Final = PromptManager(
|
||||
prompt_id="greeting_test",
|
||||
prompt_file=str(prompt_path),
|
||||
prompt_version=2,
|
||||
)
|
||||
assert manager.get_prompt("greeting_test", version=2) is not None
|
||||
assert manager.get_prompt("greeting_test.v2") is not None
|
||||
assert manager.get_prompt("greeting_test_v2") is not None
|
||||
|
||||
|
||||
def test_dotprompt_manager_async_get_chat_completion_prompt() -> None:
|
||||
from datetime import datetime
|
||||
|
||||
from litellm.integrations.dotprompt.dotprompt_manager import DotpromptManager
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
dot_mgr: Final = DotpromptManager(
|
||||
prompt_data={"greeting": {"content": "Hello {{name}}"}},
|
||||
prompt_id="greeting",
|
||||
prompt_version=2,
|
||||
)
|
||||
dynamic_params: Final = StandardCallbackDynamicParams()
|
||||
logging_obj: Final = Logging(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type="acompletion",
|
||||
litellm_call_id="call-async",
|
||||
start_time=datetime.now(),
|
||||
function_id="fn-async",
|
||||
)
|
||||
result: Final = asyncio.run(
|
||||
dot_mgr.async_get_chat_completion_prompt(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
non_default_params={},
|
||||
prompt_id="greeting",
|
||||
prompt_variables={"name": "Bob"},
|
||||
dynamic_callback_params=dynamic_params,
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
)
|
||||
rendered_messages: Final = result[1]
|
||||
assert any("Bob" in str(msg.get("content", "")) for msg in rendered_messages)
|
||||
|
||||
|
||||
def test_logging_prompt_management_error_and_mismatch(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from datetime import datetime
|
||||
|
||||
from litellm.integrations.dotprompt.dotprompt_manager import DotpromptManager
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY
|
||||
|
||||
def mock_broken_resolve(*args: object, **kwargs: object) -> None:
|
||||
raise RuntimeError("simulated registry failure")
|
||||
|
||||
monkeypatch.setattr(IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", mock_broken_resolve)
|
||||
|
||||
logging_obj: Final = Logging(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type="acompletion",
|
||||
litellm_call_id="call-err",
|
||||
start_time=datetime.now(),
|
||||
function_id="fn-err",
|
||||
)
|
||||
|
||||
detected: Final = logging_obj._auto_detect_prompt_management_logger(
|
||||
prompt_id="unknown_prompt",
|
||||
prompt_spec=None,
|
||||
dynamic_callback_params=logging_obj.standard_callback_dynamic_params,
|
||||
prompt_version=1,
|
||||
)
|
||||
assert detected is None
|
||||
|
||||
mismatched_logger: Final = DotpromptManager(
|
||||
prompt_data={"test": {"content": "Test"}},
|
||||
prompt_id="test",
|
||||
prompt_version=1,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
litellm.logging_callback_manager,
|
||||
"get_custom_loggers_for_type",
|
||||
lambda callback_type: [mismatched_logger],
|
||||
)
|
||||
|
||||
res: Final = logging_obj.get_custom_logger_for_prompt_management(
|
||||
model="gpt-4o",
|
||||
non_default_params={},
|
||||
prompt_id="test",
|
||||
dynamic_callback_params=logging_obj.standard_callback_dynamic_params,
|
||||
prompt_version=2,
|
||||
)
|
||||
assert res is None
|
||||
|
||||
|
||||
def test_router_prompt_version_acompletion(
|
||||
isolated_callbacks: list, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY
|
||||
from litellm.router import Router
|
||||
|
||||
async def mock_acompletion(*args: object, **kwargs: object) -> litellm.ModelResponse:
|
||||
return litellm.ModelResponse(
|
||||
choices=[litellm.utils.Choices(message=litellm.utils.Message(content="router reply", role="assistant"))]
|
||||
)
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", mock_acompletion)
|
||||
|
||||
IN_MEMORY_PROMPT_REGISTRY.initialize_prompt(
|
||||
prompt=_db_prompt_spec(content="System: Hello\n\nUser: {{input}}", version=2)
|
||||
)
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "p-model",
|
||||
"litellm_params": {
|
||||
"model": "dotprompt/gpt-4o",
|
||||
"prompt_id": "greeting",
|
||||
"prompt_version": 2,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
response: Final = asyncio.run(
|
||||
router.acompletion(
|
||||
model="p-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
prompt_variables={"input": "world"},
|
||||
)
|
||||
)
|
||||
assert isinstance(response, litellm.ModelResponse)
|
||||
assert response.choices[0].message.content == "router reply"
|
||||
|
||||
|
||||
def test_router_async_function_with_fallbacks_prompt_management(
|
||||
isolated_callbacks: list, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY
|
||||
from litellm.router import Router
|
||||
|
||||
async def mock_acompletion(*args: object, **kwargs: object) -> litellm.ModelResponse:
|
||||
return litellm.ModelResponse(
|
||||
choices=[litellm.utils.Choices(message=litellm.utils.Message(content="fallback reply", role="assistant"))]
|
||||
)
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", mock_acompletion)
|
||||
|
||||
IN_MEMORY_PROMPT_REGISTRY.initialize_prompt(
|
||||
prompt=_db_prompt_spec(content="System: Hello\n\nUser: {{input}}", version=1)
|
||||
)
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "pm-model",
|
||||
"litellm_params": {
|
||||
"model": "dotprompt/gpt-4o",
|
||||
"prompt_id": "greeting",
|
||||
"prompt_version": 1,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
res_kw: Final = asyncio.run(
|
||||
router.async_function_with_fallbacks(
|
||||
model="pm-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
original_function=router._acompletion,
|
||||
)
|
||||
)
|
||||
assert isinstance(res_kw, litellm.ModelResponse)
|
||||
|
||||
res_pos: Final = asyncio.run(
|
||||
router.async_function_with_fallbacks(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
model="pm-model",
|
||||
original_function=router._acompletion,
|
||||
)
|
||||
)
|
||||
assert isinstance(res_pos, litellm.ModelResponse)
|
||||
|
||||
|
||||
def test_router_dispatch_prompt_completion_direct_and_fallbacks(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
calls: Final[list[str]] = []
|
||||
|
||||
async def mock_failover_acompletion(*args: object, **kwargs: object) -> litellm.ModelResponse:
|
||||
model: Final = str(kwargs.get("model"))
|
||||
calls.append(model)
|
||||
if model == "unlisted-direct":
|
||||
raise RuntimeError("primary direct failure")
|
||||
return litellm.ModelResponse(
|
||||
choices=[litellm.utils.Choices(message=litellm.utils.Message(content="failover ok", role="assistant"))]
|
||||
)
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", mock_failover_acompletion)
|
||||
|
||||
from litellm.router import Router
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "fallback-target",
|
||||
"litellm_params": {"model": "openai/gpt-4o-fallback"},
|
||||
}
|
||||
],
|
||||
fallbacks=[{"unlisted-direct": ["fallback-target"]}],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
response: Final = asyncio.run(
|
||||
router._dispatch_prompt_completion(
|
||||
model="unlisted-direct",
|
||||
original_model_name="unlisted-direct",
|
||||
unrendered_messages=[{"role": "user", "content": "hi"}],
|
||||
original_prompt_params={"prompt_id": "test"},
|
||||
original_function=router._acompletion,
|
||||
prompt_management_params={"prompt_id", "prompt_version"},
|
||||
kwargs={"model": "unlisted-direct", "messages": [{"role": "user", "content": "hi"}], "metadata": {}},
|
||||
)
|
||||
)
|
||||
assert isinstance(response, litellm.ModelResponse)
|
||||
assert response.choices[0].message.content == "failover ok"
|
||||
assert len(calls) == 2
|
||||
|
||||
async def mock_success_acompletion(*args: object, **kwargs: object) -> litellm.ModelResponse:
|
||||
return litellm.ModelResponse(
|
||||
choices=[litellm.utils.Choices(message=litellm.utils.Message(content="direct success", role="assistant"))]
|
||||
)
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", mock_success_acompletion)
|
||||
|
||||
direct_res: Final = asyncio.run(
|
||||
router._dispatch_prompt_completion(
|
||||
model="unlisted-direct-2",
|
||||
original_model_name="unlisted-direct-2",
|
||||
unrendered_messages=[{"role": "user", "content": "hi"}],
|
||||
original_prompt_params={"prompt_id": "test"},
|
||||
original_function=router._acompletion,
|
||||
prompt_management_params={"prompt_id", "prompt_version"},
|
||||
kwargs={"model": "unlisted-direct-2", "messages": [{"role": "user", "content": "hi"}]},
|
||||
)
|
||||
)
|
||||
assert isinstance(direct_res, litellm.ModelResponse)
|
||||
assert direct_res.choices[0].message.content == "direct success"
|
||||
|
||||
async def mock_error_acompletion(*args: object, **kwargs: object) -> litellm.ModelResponse:
|
||||
raise RuntimeError("unhandled direct error")
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", mock_error_acompletion)
|
||||
|
||||
empty_router: Final = Router(model_list=[])
|
||||
with pytest.raises(RuntimeError, match="unhandled direct error"):
|
||||
asyncio.run(
|
||||
empty_router._dispatch_prompt_completion(
|
||||
model="unlisted-err",
|
||||
original_model_name="unlisted-err",
|
||||
unrendered_messages=[{"role": "user", "content": "hi"}],
|
||||
original_prompt_params={"prompt_id": "test"},
|
||||
original_function=empty_router._acompletion,
|
||||
prompt_management_params={"prompt_id", "prompt_version"},
|
||||
kwargs={"model": "unlisted-err", "messages": [{"role": "user", "content": "hi"}]},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_router_prompt_management_factory_registry_lookup_exception(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY
|
||||
from litellm.router import Router
|
||||
|
||||
def mock_failing_resolve(*args: object, **kwargs: object) -> None:
|
||||
raise RuntimeError("registry exception")
|
||||
|
||||
monkeypatch.setattr(IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", mock_failing_resolve)
|
||||
|
||||
async def mock_acompletion(*args: object, **kwargs: object) -> litellm.ModelResponse:
|
||||
return litellm.ModelResponse(
|
||||
choices=[litellm.utils.Choices(message=litellm.utils.Message(content="fallback reply", role="assistant"))]
|
||||
)
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", mock_acompletion)
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "direct-model",
|
||||
"litellm_params": {
|
||||
"model": "gpt-4o",
|
||||
"prompt_id": "test-prompt",
|
||||
"prompt_version": 1,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
response: Final = asyncio.run(
|
||||
router._prompt_management_factory(
|
||||
model="direct-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
kwargs={"model": "direct-model", "original_function": router._acompletion},
|
||||
)
|
||||
)
|
||||
assert isinstance(response, litellm.ModelResponse)
|
||||
assert response.choices[0].message.content == "fallback reply"
|
||||
|
||||
|
||||
def test_router_fallback_hop_restores_prompt_parameters_and_messages(
|
||||
isolated_callbacks: list, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY
|
||||
from litellm.router import Router
|
||||
|
||||
IN_MEMORY_PROMPT_REGISTRY.initialize_prompt(
|
||||
prompt=_db_prompt_spec(content="System: Primary V1\n\nUser: {{input}}", version=1)
|
||||
)
|
||||
IN_MEMORY_PROMPT_REGISTRY.initialize_prompt(
|
||||
prompt=_db_prompt_spec(content="System: Fallback V2\n\nUser: {{input}}", version=2)
|
||||
)
|
||||
|
||||
calls: Final[list[dict]] = []
|
||||
|
||||
async def mock_acompletion(*args: object, **kwargs: object) -> litellm.ModelResponse:
|
||||
calls.append(dict(kwargs))
|
||||
if len(calls) == 1:
|
||||
raise RuntimeError("primary prompt failure")
|
||||
return litellm.ModelResponse(
|
||||
choices=[litellm.utils.Choices(message=litellm.utils.Message(content="hop ok", role="assistant"))]
|
||||
)
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", mock_acompletion)
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "primary-pm",
|
||||
"litellm_params": {
|
||||
"model": "dotprompt/gpt-4o",
|
||||
"prompt_id": "greeting",
|
||||
"prompt_version": 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "fallback-pm",
|
||||
"litellm_params": {
|
||||
"model": "dotprompt/gpt-4o",
|
||||
"prompt_id": "greeting",
|
||||
"prompt_version": 2,
|
||||
},
|
||||
},
|
||||
],
|
||||
fallbacks=[{"primary-pm": ["fallback-pm"]}],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
response: Final = asyncio.run(
|
||||
router.acompletion(
|
||||
model="primary-pm",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
prompt_variables={"input": "test-val"},
|
||||
)
|
||||
)
|
||||
assert isinstance(response, litellm.ModelResponse)
|
||||
assert response.choices[0].message.content == "hop ok"
|
||||
assert len(calls) == 2
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue