diff --git a/litellm/__init__.py b/litellm/__init__.py index 117ba9604e9..c2c13a12eea 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1729,6 +1729,9 @@ if TYPE_CHECKING: from .llms.hosted_vllm.decisions.transformation import ( HostedVLLMDecisionsConfig as HostedVLLMDecisionsConfig, ) + from .llms.lm_studio.decisions.transformation import ( + LMStudioDecisionsConfig as LMStudioDecisionsConfig, + ) from .llms.laya.decisions.transformation import ( LayaDecisionsConfig as LayaDecisionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index a56004d1274..6aecfd2ff8c 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -170,6 +170,7 @@ LLM_CONFIG_NAMES: Final = ( "StrandsDeciderDecisionsConfig", "DatabricksDecisionsConfig", "HostedVLLMDecisionsConfig", + "LMStudioDecisionsConfig", "LayaDecisionsConfig", "BespokeDecisionsConfig", "OpenAIDecisionsConfig", @@ -746,6 +747,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.hosted_vllm.decisions.transformation", "HostedVLLMDecisionsConfig", ), + "LMStudioDecisionsConfig": (".llms.lm_studio.decisions.transformation", "LMStudioDecisionsConfig"), "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"), "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"), "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), diff --git a/litellm/llms/lm_studio/decisions/transformation.py b/litellm/llms/lm_studio/decisions/transformation.py new file mode 100644 index 00000000000..84e5ff14443 --- /dev/null +++ b/litellm/llms/lm_studio/decisions/transformation.py @@ -0,0 +1,16 @@ +from typing import Final + +from litellm.constants import PLACEHOLDER_API_KEY +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig + + +class LMStudioDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("LM_STUDIO_API_KEY",) + api_base_env = ("LM_STUDIO_API_BASE",) + api_key_required = False + supports_images = True + inlines_remote_images = True + + def resolve_api_key(self, api_key: str | None) -> str | None: + resolved: Final = super().resolve_api_key(api_key) + return None if resolved == PLACEHOLDER_API_KEY else resolved diff --git a/litellm/utils.py b/litellm/utils.py index 1aed15ba83e..fdfad7fc665 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9610,6 +9610,8 @@ class ProviderConfigManager: return litellm.DatabricksDecisionsConfig() if provider == LlmProviders.HOSTED_VLLM: return litellm.HostedVLLMDecisionsConfig() + if provider == LlmProviders.LM_STUDIO: + return litellm.LMStudioDecisionsConfig() if provider == LlmProviders.LAYA: return litellm.LayaDecisionsConfig() if provider == LlmProviders.BESPOKE: diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 5b32008b279..d13d212b4de 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -38,6 +38,7 @@ from litellm.types.decisions import ( from litellm.types.router import ModelInfo from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager +from tests.unit.conftest import AsyncOnlyImageFetch _QUESTIONS: Final[Mapping[str, object]] = MappingProxyType( { @@ -682,7 +683,9 @@ _PPLX_USAGE: Final = {"model": "pplx-decider-v1-27b", "usage": {"input_tokens": ("reply", "detail"), ( pytest.param(_PPLX_USAGE, "DecisionsResponse answers: Field required", id="usage"), - pytest.param({"error": "Rate limit reached"}, "DecisionsResponse answers: Field required", id="rate_limit_text"), + pytest.param( + {"error": "Rate limit reached"}, "DecisionsResponse answers: Field required", id="rate_limit_text" + ), pytest.param({"error": "Request timed out"}, "DecisionsResponse answers: Field required", id="timeout_text"), pytest.param( {**_PPLX_USAGE, "answers": {"rate_limit": None}}, @@ -694,7 +697,9 @@ _PPLX_USAGE: Final = {"model": "pplx-decider-v1-27b", "usage": {"input_tokens": "DecisionsResponse answers.is_defect", id="answer_type_timeout_text", ), - pytest.param(["Rate limit reached"], "DecisionsResponse Input should be a valid dictionary", id="not_an_object"), + pytest.param( + ["Rate limit reached"], "DecisionsResponse Input should be a valid dictionary", id="not_an_object" + ), ), ) async def test_a_malformed_upstream_reply_is_a_server_error_whatever_its_text( @@ -1525,7 +1530,12 @@ async def test_databricks_router_deployment_sends_its_own_key_to_its_own_base( _PROVIDERS_WHOSE_CHAT_MODELS_ALSO_DECIDE: Final = frozenset( - {LlmProviders.OPENAI.value, LlmProviders.OPENROUTER.value, LlmProviders.HOSTED_VLLM.value} + { + LlmProviders.OPENAI.value, + LlmProviders.OPENROUTER.value, + LlmProviders.HOSTED_VLLM.value, + LlmProviders.LM_STUDIO.value, + } ) _DEDICATED_DECIDER_PROVIDERS: Final[tuple[str, ...]] = tuple( sorted( @@ -3791,7 +3801,12 @@ async def test_a_returned_model_registered_without_prices_is_billed_at_the_reque monkeypatch.setitem( litellm.model_cost, "typesafe/jev-test-alias", - {"litellm_provider": "typesafe", "mode": "evaluation", "input_cost_per_token": 0.001, "output_cost_per_token": 0}, + { + "litellm_provider": "typesafe", + "mode": "evaluation", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0, + }, ) monkeypatch.setitem(litellm.model_cost, "typesafe/jev-test-unpriced", unpriced_row) respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json={**_RESPONSE, "model": "jev-test-unpriced"}) @@ -3893,3 +3908,120 @@ async def test_native_decisions_distinguishes_missing_usage_from_reported_zero( expected = litellm.completion_cost(completion_response=response) assert response._hidden_params["response_cost"] == pytest.approx(expected) assert (expected == 0) == (usage["input_tokens"] == usage["output_tokens"] == 0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", (True, False), ids=("asystemone", "systemone")) +@pytest.mark.parametrize("api_base", ("http://lmstudio.local:1234", "http://lmstudio.local:1234/v1")) +async def test_lm_studio_systemone_preserves_questions_and_omits_placeholder_auth( + use_async: bool, api_base: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.delenv("LM_STUDIO_API_KEY", raising=False) + monkeypatch.setenv("OPENAI_API_KEY", "unrelated-openai-key") + route: Final = respx_mock.post("http://lmstudio.local:1234/v1/systemone").respond(json=_RESPONSE) + kwargs: Final = { + "model": "lm_studio/local-decision", + "state": "review", + "questions": _QUESTIONS, + "api_base": api_base, + } + + response: Final = await litellm.asystemone(**kwargs) if use_async else litellm.systemone(**kwargs) + + assert json.loads(route.calls.last.request.content) == { + "model": "local-decision", + "state": "review", + "questions": dict(_QUESTIONS), + } + assert "authorization" not in route.calls.last.request.headers + assert response.answers["is_defect"] == NoulAnswer(type="noul", noul=0.9) + assert response.answers["sentiment"].choice == "positive" + assert response.answers["severity"].score == 1 + assert response.usage == DecisionsUsage(input_tokens=_INPUT_TOKENS, output_tokens=_OUTPUT_TOKENS) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", (True, False), ids=("adecisions", "decisions")) +async def test_lm_studio_decisions_translates_openai_questions_and_answers( + use_async: bool, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.delenv("LM_STUDIO_API_KEY", raising=False) + route: Final = respx_mock.post("http://lmstudio.local:1234/v1/systemone").respond(json=_RESPONSE) + kwargs: Final = { + "model": "lm_studio/local-decision", + "input": "review", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + "api_base": "http://lmstudio.local:1234/v1", + } + + response: Final = await litellm.adecisions(**kwargs) if use_async else litellm.decisions(**kwargs) + + assert json.loads(route.calls.last.request.content) == { + "model": "local-decision", + "state": "review", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert response.answers[0].type == "predicate" + assert response.answers[0].name == "is_defect" + assert response.answers[0].probability == 0.9 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("explicit_key", (None, "deployment-key")) +async def test_lm_studio_decisions_resolves_environment_connection_and_explicit_key_override( + explicit_key: str | None, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setenv("LM_STUDIO_API_BASE", "http://lmstudio.local:1234/v1") + monkeypatch.setenv("LM_STUDIO_API_KEY", "environment-key") + route: Final = respx_mock.post("http://lmstudio.local:1234/v1/systemone").respond(json=_RESPONSE) + + response: Final = await litellm.asystemone( + model="lm_studio/local-decision", state="review", questions=_QUESTIONS, api_key=explicit_key + ) + + assert route.calls.last.request.headers["authorization"] == f"Bearer {explicit_key or 'environment-key'}" + assert response.answers["is_defect"] == NoulAnswer(type="noul", noul=0.9) + + +@pytest.mark.asyncio +async def test_lm_studio_decisions_requires_a_server_before_http( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.delenv("LM_STUDIO_API_BASE", raising=False) + with pytest.raises(litellm.BadRequestError, match="api_base is required"): + await litellm.asystemone(model="lm_studio/local-decision", state="review", questions=_QUESTIONS) + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("remote_image", (False, True), ids=("inline", "remote")) +async def test_lm_studio_decisions_sends_inline_images_to_systemone( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + remote_image: bool, + async_only_image_fetch: AsyncOnlyImageFetch, +) -> None: + monkeypatch.delenv("LM_STUDIO_API_KEY", raising=False) + image: Final = "https://images.example/input.png" if remote_image else "data:image/png;base64,AA==" + expected_image: Final = async_only_image_fetch.data_url if remote_image else image + route: Final = respx_mock.post("http://lmstudio.local:1234/v1/systemone").respond(json=_RESPONSE) + + response: Final = await litellm.adecisions( + model="lm_studio/local-decision", + input=[ + { + "role": "user", + "content": [{"type": "input_text", "text": "review"}, {"type": "input_image", "image_url": image}], + } + ], + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + api_base="http://lmstudio.local:1234/v1", + ) + + assert json.loads(route.calls.last.request.content) == { + "model": "local-decision", + "state": "review", + "images": [expected_image], + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert response.answers[0].probability == 0.9 diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index 4384507e841..a0a79f9b7cf 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -1784,3 +1784,53 @@ def test_a_whole_number_request_size_limit_is_named_as_configured_in_the_413( assert response.status_code == 413, response.text assert response.json() == {"error": "Request size is too large. Max size is 1 MB"} assert not upstream.called + + +@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/v1/decisions")) +@pytest.mark.parametrize("api_key", (None, "deployment-key")) +def test_proxy_routes_lm_studio_decisions_with_optional_authentication( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + endpoint: str, + api_key: str | None, +) -> None: + monkeypatch.delenv("LM_STUDIO_API_KEY", raising=False) + deployment_router: Final = litellm.Router( + model_list=[ + { + "model_name": "local-decision", + "litellm_params": { + "model": "lm_studio/served-decision", + "api_base": "http://lmstudio.local:1234/v1", + "api_key": api_key, + }, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", deployment_router) + upstream: Final = respx_mock.post("http://lmstudio.local:1234/v1/systemone").respond(json=_RESPONSE) + fields: Final = ( + {"state": "review", "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}} + if endpoint.endswith("systemone") + else { + "input": "review", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + } + ) + + response: Final = client.post(endpoint, json={"model": "local-decision", **fields}) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls.last.request.content) == { + "model": "served-decision", + "state": "review", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert upstream.calls.last.request.headers.get("authorization") == (f"Bearer {api_key}" if api_key else None) + expected: Final = ( + _RESPONSE["answers"] + if endpoint.endswith("systemone") + else [{"type": "predicate", "name": "is_defect", "probability": 0.9}] + ) + assert response.json()["answers"] == expected