mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-12 03:48:56 +00:00
feat(decisions): backport Microsoft-Decision-1 and the decision models Admin UI to rc/1.106.0 (#45639, #45673, #45654, #45717) (#45735)
* feat(azure): add azure_ai/Microsoft-Decision-1 (#45639) Backport of #45639 to rc/1.106.0. Cherry-picked fromde6a0717b7(main). * feat(azure_ai): support Microsoft-Decision-1 on the decisions API (#45673) Backport of #45673 to rc/1.106.0. Cherry-picked from2fcc780498(main). * feat(ui): let admins find, call, and test decision models from the Admin UI (#45654) Backport of #45654 to rc/1.106.0. Cherry-picked frome2276ac2b9(main). * fix(model_prices): drop unverified output price for azure_ai/Microsoft-Decision-1 (#45717) Backport of #45717 to rc/1.106.0. Cherry-picked from674208bb7d(main). Co-authored-by: kerry <kerry@berri.ai> --------- Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: moyai-devin-berriai[bot] <336287033+moyai-devin-berriai[bot]@users.noreply.github.com> Co-authored-by: kerry <kerry@berri.ai>
This commit is contained in:
Mateo Wang
GitHub
parent
bbcc985d4a
commit
98f717afce
42 changed files with 1477 additions and 168 deletions
|
|
@ -14,8 +14,9 @@ import argparse
|
|||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors
|
||||
|
|
@ -25,6 +26,7 @@ BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
|
|||
SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
|
||||
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH)
|
||||
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
|
||||
EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
|
||||
CostMap = dict[str, object]
|
||||
|
||||
|
|
@ -51,7 +53,31 @@ def _rendered_schema(cost_map: CostMap) -> str:
|
|||
return str(error)
|
||||
|
||||
|
||||
def _file_failures(head: Snapshot, head_map: CostMap) -> tuple[str, ...]:
|
||||
def _evaluation_output_price_failure(
|
||||
key: str, entry: object, evaluation_output_price_sources: Mapping[str, str]
|
||||
) -> str | None:
|
||||
if not isinstance(entry, dict) or entry.get("mode") != "evaluation":
|
||||
return None
|
||||
output_cost: Final = entry.get("output_cost_per_token")
|
||||
if (
|
||||
not isinstance(output_cost, (int, float))
|
||||
or isinstance(output_cost, bool)
|
||||
or output_cost <= 0
|
||||
or key in evaluation_output_price_sources
|
||||
):
|
||||
return None
|
||||
return (
|
||||
f"{key}: mode evaluation rows bill input only; output_cost_per_token must be 0 unless the vendor "
|
||||
"prices output tokens for this model (add the source to EVALUATION_OUTPUT_PRICE_SOURCES)"
|
||||
)
|
||||
|
||||
|
||||
def _file_failures(
|
||||
head: Snapshot,
|
||||
head_map: CostMap,
|
||||
*,
|
||||
evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES,
|
||||
) -> tuple[str, ...]:
|
||||
schema_text: Final = _rendered_schema(head_map)
|
||||
if not schema_text.startswith("{"):
|
||||
return (schema_text,)
|
||||
|
|
@ -75,6 +101,16 @@ def _file_failures(head: Snapshot, head_map: CostMap) -> tuple[str, ...]:
|
|||
f"{COST_MAP_PATH} does not validate against its schema: {error}"
|
||||
for error in validation_errors(head_map, json.loads(schema_text))[:20]
|
||||
),
|
||||
*(
|
||||
failure
|
||||
for key, entry in head_map.items()
|
||||
if (
|
||||
failure := _evaluation_output_price_failure(
|
||||
key, entry, evaluation_output_price_sources
|
||||
)
|
||||
)
|
||||
is not None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -121,13 +157,25 @@ def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
|
|||
return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
|
||||
|
||||
|
||||
def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]:
|
||||
def guard_failures(
|
||||
base: Snapshot,
|
||||
head: Snapshot,
|
||||
changed_files: Sequence[str],
|
||||
bot: bool,
|
||||
*,
|
||||
evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES,
|
||||
) -> tuple[str, ...]:
|
||||
if not bot and not touches_cost_map(changed_files):
|
||||
return ()
|
||||
head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH)
|
||||
if isinstance(head_map, str):
|
||||
return (head_map,)
|
||||
return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ()))
|
||||
return (
|
||||
*_file_failures(
|
||||
head, head_map, evaluation_output_price_sources=evaluation_output_price_sources
|
||||
),
|
||||
*(_bot_failures(base, head_map, changed_files) if bot else ()),
|
||||
)
|
||||
|
||||
|
||||
def _git(*args: str) -> str | None:
|
||||
|
|
|
|||
|
|
@ -1684,6 +1684,9 @@ if TYPE_CHECKING:
|
|||
from .llms.openai.decisions.transformation import (
|
||||
OpenAIDecisionsConfig as OpenAIDecisionsConfig,
|
||||
)
|
||||
from .llms.azure_ai.decisions.transformation import (
|
||||
AzureAIDecisionsConfig as AzureAIDecisionsConfig,
|
||||
)
|
||||
from .llms.nvidia_nim.rerank.transformation import (
|
||||
NvidiaNimRerankConfig as NvidiaNimRerankConfig,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -163,6 +163,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"CloudflareDecisionsConfig",
|
||||
"StrandsDeciderDecisionsConfig",
|
||||
"OpenAIDecisionsConfig",
|
||||
"AzureAIDecisionsConfig",
|
||||
"NvidiaNimRerankConfig",
|
||||
"NvidiaNimRankingConfig",
|
||||
"VertexAIRerankConfig",
|
||||
|
|
@ -724,6 +725,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
"StrandsDeciderDecisionsConfig",
|
||||
),
|
||||
"OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
|
||||
"AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"),
|
||||
"NvidiaNimRerankConfig": (
|
||||
".llms.nvidia_nim.rerank.transformation",
|
||||
"NvidiaNimRerankConfig",
|
||||
|
|
|
|||
24
litellm/llms/azure_ai/decisions/transformation.py
Normal file
24
litellm/llms/azure_ai/decisions/transformation.py
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
import re
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
|
||||
_FOUNDRY_ROUTE_SUFFIX: Final = re.compile(r"(/api/projects/[^/]+)?(/openai/v1|/openai|/models|/v1)?$")
|
||||
|
||||
|
||||
class AzureAIDecisionsConfig(BaseDecisionsConfig):
|
||||
"""Microsoft Foundry decision models (Microsoft-Decision-1), where model is the deployment name"""
|
||||
|
||||
path = "/providers/microsoft/v1/systemone"
|
||||
api_key_env = ("AZURE_AI_API_KEY",)
|
||||
api_base_env = ("AZURE_AI_API_BASE",)
|
||||
|
||||
def missing_api_base_message(self, custom_llm_provider: str) -> str:
|
||||
return "Missing AZURE_AI_API_BASE - set AZURE_AI_API_BASE or pass api_base, e.g. https://<resource>.services.ai.azure.com"
|
||||
|
||||
def get_complete_url(self, api_base: str, model: str) -> str:
|
||||
url: Final = httpx.URL(api_base)
|
||||
resource_path: Final = _FOUNDRY_ROUTE_SUFFIX.sub("", url.path.rstrip("/"))
|
||||
return str(url.copy_with(path=f"{resource_path}{self.path}", query=None, fragment=None))
|
||||
|
|
@ -81933,5 +81933,12 @@
|
|||
"output_cost_per_token": 7.5e-06,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
|
||||
"supports_video_input": true
|
||||
},
|
||||
"azure_ai/Microsoft-Decision-1": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8991,6 +8991,8 @@ class ProviderConfigManager:
|
|||
return litellm.StrandsDeciderDecisionsConfig()
|
||||
if provider == LlmProviders.OPENAI:
|
||||
return litellm.OpenAIDecisionsConfig()
|
||||
if provider == LlmProviders.AZURE_AI:
|
||||
return litellm.AzureAIDecisionsConfig()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -81933,5 +81933,12 @@
|
|||
"output_cost_per_token": 7.5e-06,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
|
||||
"supports_video_input": true
|
||||
},
|
||||
"azure_ai/Microsoft-Decision-1": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -103,6 +103,9 @@ _PROVIDERS: Final = (
|
|||
True,
|
||||
"cloudflare/@cf/cloudflare/clef",
|
||||
),
|
||||
_Provider(
|
||||
"azure_ai", "azure_ai/decision-1", "/providers/microsoft/v1/systemone", "decision-1", _API_KEY, False, None
|
||||
),
|
||||
)
|
||||
_PERPLEXITY: Final = _PROVIDERS[0]
|
||||
_OPENROUTER: Final = _PROVIDERS[2]
|
||||
|
|
|
|||
|
|
@ -346,6 +346,13 @@ model_list:
|
|||
litellm_params:
|
||||
model: strands_decider/strands-decider-2B-hobson-v19
|
||||
api_base: http://127.0.0.1:8191
|
||||
- model_name: azure_ai/decision-1
|
||||
litellm_params:
|
||||
model: azure_ai/decision-1
|
||||
api_base: http://127.0.0.1:8191
|
||||
api_key: synthetic-azure-ai-key
|
||||
model_info:
|
||||
base_model: azure_ai/Microsoft-Decision-1
|
||||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
database_url: os.environ/DATABASE_URL
|
||||
|
|
|
|||
79
tests/integration/translation/decisions/bases/azure_ai.py
Normal file
79
tests/integration/translation/decisions/bases/azure_ai.py
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
from typing import Final
|
||||
|
||||
from integration.translation.case import TranslationTestCase
|
||||
|
||||
"""Provider request and reply shape from the Microsoft Foundry Decision playground (POST /providers/microsoft/v1/systemone, model is the deployment name). Mock reply captured live from a Microsoft-Decision-1 deployment on 2026-10-09.
|
||||
"""
|
||||
MICROSOFT_DECISION_1_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/systemone",
|
||||
litellm_request={
|
||||
"model": "azure_ai/decision-1",
|
||||
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
|
||||
"questions": {
|
||||
"defect": {"type": "noul", "instructions": "Is this a defect?"},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"instructions": "How severe is it?",
|
||||
"criteria": {"low": "cosmetic", "high": "blocks users"},
|
||||
},
|
||||
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
|
||||
},
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
expected_provider_endpoint="/providers/microsoft/v1/systemone",
|
||||
expected_provider_headers={"authorization": "Bearer synthetic-azure-ai-key", "content-type": "application/json"},
|
||||
expected_provider_request={
|
||||
"model": "decision-1",
|
||||
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
|
||||
"questions": {
|
||||
"defect": {"type": "noul", "instructions": "Is this a defect?"},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"instructions": "How severe is it?",
|
||||
"criteria": {"low": "cosmetic", "high": "blocks users"},
|
||||
},
|
||||
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
|
||||
},
|
||||
},
|
||||
mock_provider_response={
|
||||
"model": "microsoft-decision-1",
|
||||
"answers": {
|
||||
"confidence": {
|
||||
"confidence": 0.6351489346076444,
|
||||
"legend": {"0": "unsure", "1": "sure"},
|
||||
"probabilities": {"0": 0.1824255326961778, "1": 0.8175744673038222},
|
||||
"score": 0.8175744673038222,
|
||||
"type": "score",
|
||||
},
|
||||
"defect": {"noul": 0.982013764002278, "type": "noul"},
|
||||
"severity": {
|
||||
"choice": "high",
|
||||
"confidence": 0.9866142978108987,
|
||||
"probabilities": {"high": 0.9933071489054494, "low": 0.006692851094550702},
|
||||
"type": "choice",
|
||||
},
|
||||
},
|
||||
"usage": {"input_tokens": 87, "output_tokens": 3},
|
||||
},
|
||||
expected_litellm_response={
|
||||
"model": "microsoft-decision-1",
|
||||
"answers": {
|
||||
"defect": {"type": "noul", "noul": 0.982013764002278},
|
||||
"severity": {
|
||||
"type": "choice",
|
||||
"choice": "high",
|
||||
"confidence": 0.9866142978108987,
|
||||
"probabilities": {"high": 0.9933071489054494, "low": 0.006692851094550702},
|
||||
},
|
||||
"confidence": {
|
||||
"type": "score",
|
||||
"score": 0.8175744673038222,
|
||||
"confidence": 0.6351489346076444,
|
||||
"legend": {"0": "unsure", "1": "sure"},
|
||||
"probabilities": {"0": 0.1824255326961778, "1": 0.8175744673038222},
|
||||
},
|
||||
},
|
||||
"usage": {"input_tokens": 87, "output_tokens": 3},
|
||||
},
|
||||
)
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.decisions.bases.azure_ai import MICROSOFT_DECISION_1_TEST_CASE
|
||||
from integration.translation.runner import assert_translation
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", [MICROSOFT_DECISION_1_TEST_CASE], ids=lambda case: case.id)
|
||||
def test_decisions_basic_azure_ai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
|
||||
assert_translation(case, gateway, provider)
|
||||
|
|
@ -922,3 +922,88 @@ async def test_requests_a_provider_cannot_serve_are_rejected_before_http(
|
|||
await litellm.adecisions(model=model, api_key="caller-key", **request_kwargs)
|
||||
|
||||
assert len(respx_mock.calls) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"api_base",
|
||||
(
|
||||
"https://res.services.ai.azure.com",
|
||||
"https://res.services.ai.azure.com/",
|
||||
"https://res.services.ai.azure.com/models",
|
||||
"https://res.services.ai.azure.com/openai/v1",
|
||||
"https://res.services.ai.azure.com/api/projects/proj",
|
||||
"https://res.services.ai.azure.com/api/projects/proj/openai/v1",
|
||||
"https://res.services.ai.azure.com/api/projects/proj/models",
|
||||
"https://res.services.ai.azure.com/models?api-version=2024-05-01-preview",
|
||||
),
|
||||
)
|
||||
async def test_azure_ai_decision_posts_deployment_to_foundry_systemone_route(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
api_base: str,
|
||||
) -> None:
|
||||
monkeypatch.delenv("AZURE_AI_API_BASE", raising=False)
|
||||
monkeypatch.delenv("AZURE_AI_API_KEY", raising=False)
|
||||
route: Final = respx_mock.post("https://res.services.ai.azure.com/providers/microsoft/v1/systemone").respond(
|
||||
json={**_RESPONSE, "model": "microsoft-decision-1"}
|
||||
)
|
||||
|
||||
response: Final = await litellm.adecisions(
|
||||
model="azure_ai/decision-1",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
api_base=api_base,
|
||||
api_key="foundry-key",
|
||||
)
|
||||
|
||||
assert route.called
|
||||
request: Final = respx_mock.calls[0].request
|
||||
assert request.headers["authorization"] == "Bearer foundry-key"
|
||||
assert "api-key" not in request.headers
|
||||
assert json.loads(request.content) == {
|
||||
"model": "decision-1",
|
||||
"state": "review",
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
}
|
||||
assert response.answers == {"is_defect": NoulAnswer(type="noul", noul=0.9)}
|
||||
assert response._hidden_params["model"] == "azure_ai/decision-1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_azure_ai_decision_reads_foundry_env_and_keeps_gateway_path_prefix(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setenv("AZURE_AI_API_BASE", "https://gateway.example.com/foundry/models")
|
||||
monkeypatch.setenv("AZURE_AI_API_KEY", "env-foundry-key")
|
||||
route: Final = respx_mock.post("https://gateway.example.com/foundry/providers/microsoft/v1/systemone").respond(
|
||||
json=_RESPONSE
|
||||
)
|
||||
|
||||
await litellm.adecisions(
|
||||
model="azure_ai/decision-1",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
)
|
||||
|
||||
assert route.called
|
||||
assert respx_mock.calls[0].request.headers["authorization"] == "Bearer env-foundry-key"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_azure_ai_decision_requires_api_base_before_http(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.delenv("AZURE_AI_API_BASE", raising=False)
|
||||
monkeypatch.setenv("AZURE_AI_API_KEY", "foundry-key")
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="Missing AZURE_AI_API_BASE"):
|
||||
await litellm.adecisions(
|
||||
model="azure_ai/decision-1",
|
||||
state="review",
|
||||
questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
)
|
||||
|
||||
assert len(respx_mock.calls) == 0
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from collections.abc import Mapping
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
|
|
@ -62,8 +63,19 @@ def _snapshot(cost_map: dict[str, object], backup: str | None = None, schema: st
|
|||
BASE: Final = _snapshot(BASE_MAP)
|
||||
|
||||
|
||||
def _failures(head: object, changed_files: tuple[str, ...] = MAP_FILES, bot: bool = True) -> tuple[str, ...]:
|
||||
return guard.guard_failures(BASE, head, changed_files, bot)
|
||||
def _failures(
|
||||
head: object,
|
||||
changed_files: tuple[str, ...] = MAP_FILES,
|
||||
bot: bool = True,
|
||||
evaluation_output_price_sources: Mapping[str, str] = guard.EVALUATION_OUTPUT_PRICE_SOURCES,
|
||||
) -> tuple[str, ...]:
|
||||
return guard.guard_failures(
|
||||
BASE,
|
||||
head,
|
||||
changed_files,
|
||||
bot,
|
||||
evaluation_output_price_sources=evaluation_output_price_sources,
|
||||
)
|
||||
|
||||
|
||||
def test_in_sync_files_pass_for_humans_and_bots() -> None:
|
||||
|
|
@ -71,6 +83,52 @@ def test_in_sync_files_pass_for_humans_and_bots() -> None:
|
|||
assert _failures(BASE, bot=True) == ()
|
||||
|
||||
|
||||
EVALUATION_MODEL: Final = "azure_ai/Microsoft-Decision-1"
|
||||
EVALUATION_OUTPUT_PRICE_FAILURE: Final = (
|
||||
f"{EVALUATION_MODEL}: mode evaluation rows bill input only; output_cost_per_token must be 0 unless the vendor "
|
||||
"prices output tokens for this model (add the source to EVALUATION_OUTPUT_PRICE_SOURCES)"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bot", (True, False))
|
||||
def test_evaluation_rows_with_output_price_fail_for_humans_and_bots(bot: bool) -> None:
|
||||
head: Final = _snapshot(
|
||||
{**BASE_MAP, EVALUATION_MODEL: _entry(mode="evaluation", output_cost_per_token=4.2e-08)}
|
||||
)
|
||||
assert _failures(head, bot=bot) == (EVALUATION_OUTPUT_PRICE_FAILURE,)
|
||||
|
||||
|
||||
def test_evaluation_rows_with_zero_or_missing_output_price_pass() -> None:
|
||||
zero_output: Final = _snapshot(
|
||||
{**BASE_MAP, EVALUATION_MODEL: _entry(mode="evaluation", output_cost_per_token=0.0)}
|
||||
)
|
||||
missing_output_entry: Final = {
|
||||
key: value
|
||||
for key, value in _entry(mode="evaluation").items()
|
||||
if key != "output_cost_per_token"
|
||||
}
|
||||
missing_output: Final = _snapshot({**BASE_MAP, EVALUATION_MODEL: missing_output_entry})
|
||||
|
||||
assert _failures(zero_output) == ()
|
||||
assert _failures(missing_output) == ()
|
||||
|
||||
|
||||
def test_chat_rows_with_output_price_pass() -> None:
|
||||
head: Final = _snapshot(
|
||||
{**BASE_MAP, "openrouter/chat-with-output": _entry(output_cost_per_token=4.2e-08)}
|
||||
)
|
||||
assert _failures(head) == ()
|
||||
|
||||
|
||||
def test_evaluation_output_price_source_allowlist_passes() -> None:
|
||||
head: Final = _snapshot(
|
||||
{**BASE_MAP, EVALUATION_MODEL: _entry(mode="evaluation", output_cost_per_token=4.2e-08)}
|
||||
)
|
||||
source: Final = 'https://vendor.example/pricing "Output tokens are charged per token."'
|
||||
|
||||
assert _failures(head, evaluation_output_price_sources={EVALUATION_MODEL: source}) == ()
|
||||
|
||||
|
||||
def test_bot_may_add_and_reprice_models() -> None:
|
||||
head = _snapshot({**BASE_MAP, "openrouter/a": _entry(9e-06, supports_vision=True), "openrouter/c": _entry()})
|
||||
assert _failures(head) == ()
|
||||
|
|
|
|||
|
|
@ -6,9 +6,17 @@ import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
|||
import { isAdminRole } from "@/utils/roles";
|
||||
|
||||
export default function ModelHubTablePage() {
|
||||
const { accessToken, userRole, premiumUser } = useAuthorized();
|
||||
const { accessToken, userRole, premiumUser, isViewOnly } = useAuthorized();
|
||||
if (!isAdminRole(userRole)) {
|
||||
return <PublicModelHub accessToken={accessToken} isEmbedded={true} />;
|
||||
}
|
||||
return <ModelHubTable accessToken={accessToken} publicPage={false} premiumUser={premiumUser} userRole={userRole} />;
|
||||
return (
|
||||
<ModelHubTable
|
||||
accessToken={accessToken}
|
||||
publicPage={false}
|
||||
premiumUser={premiumUser}
|
||||
userRole={userRole}
|
||||
canOpenPlayground={!isViewOnly}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ vi.mock("./panels/ModelRetrySettingsPanel", () => ({ default: () => <div data-te
|
|||
vi.mock("./panels/ModelGroupAliasPanel", () => ({ default: () => <div data-testid="panel-alias" /> }));
|
||||
vi.mock("./panels/PriceDataPanel", () => ({ default: () => <div data-testid="panel-price" /> }));
|
||||
vi.mock("./panels/AccessGroupBudgetsPanel", () => ({ default: () => <div data-testid="panel-budgets" /> }));
|
||||
vi.mock("@/components/molecules/cost_optimization_feedback_banner", () => ({ default: () => null }));
|
||||
vi.mock("@/components/molecules/DecisionModelsBanner", () => ({ default: () => null }));
|
||||
vi.mock("@/components/model_info_view", () => ({
|
||||
default: ({ modelId }: { modelId: string }) => <div data-testid="model-info">model:{modelId}</div>,
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -21,7 +21,17 @@ vi.mock("./detailNavigation", () => ({
|
|||
useModelDetailRouting: () => ({ ...detailState, close: vi.fn(), openModel: vi.fn(), openTeam: vi.fn() }),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/molecules/cost_optimization_feedback_banner", () => ({ default: () => null }));
|
||||
vi.mock("@/components/molecules/DecisionModelsBanner", () => ({
|
||||
default: ({ onAddModel }: { onAddModel?: () => void }) => (
|
||||
<section aria-label="Decision models">
|
||||
{onAddModel && (
|
||||
<button type="button" onClick={onAddModel}>
|
||||
Add a decision model
|
||||
</button>
|
||||
)}
|
||||
</section>
|
||||
),
|
||||
}));
|
||||
vi.mock("@/components/model_info_view", () => ({
|
||||
default: ({ modelId }: { modelId: string }) => <div data-testid="model-info">model:{modelId}</div>,
|
||||
}));
|
||||
|
|
@ -97,6 +107,27 @@ describe("ModelsAndEndpointsPage", () => {
|
|||
expect(screen.queryByTestId("panel-all-models")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("opens the Add Model tab from the decision models banner", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderPage();
|
||||
await user.click(screen.getByRole("button", { name: "Add a decision model" }));
|
||||
expect(screen.getByTestId("panel-add")).toBeInTheDocument();
|
||||
expect(screen.queryByTestId("panel-all-models")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("does not offer the banner's Add a decision model to a session that cannot add models", () => {
|
||||
mockUseAuthorized.mockReturnValue(NON_ADMIN);
|
||||
renderPage();
|
||||
expect(screen.getByRole("region", { name: "Decision models" })).toBeInTheDocument();
|
||||
expect(screen.queryByRole("button", { name: "Add a decision model" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("does not show a view-only session the decision models banner, whose Playground it cannot open", () => {
|
||||
mockUseAuthorized.mockReturnValue(VIEW_ONLY_ADMIN);
|
||||
renderPage();
|
||||
expect(screen.queryByRole("region", { name: "Decision models" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("renders the model detail overlay from the ?model drill-in and hides the tabs", () => {
|
||||
detailState.modelId = "abc-123";
|
||||
renderPage();
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"
|
|||
import { all_admin_roles, internalUserRoles } from "@/utils/roles";
|
||||
import { autoRouterCreationScope, canCreateModels } from "@/utils/modelPermissions";
|
||||
import BetaBadge from "@/components/BetaBadge";
|
||||
import CostOptimizationFeedbackBanner from "@/components/molecules/cost_optimization_feedback_banner";
|
||||
import DecisionModelsBanner from "@/components/molecules/DecisionModelsBanner";
|
||||
import ModelInfoView from "@/components/model_info_view";
|
||||
import TeamInfoView from "@/components/team/TeamInfo";
|
||||
import { useModelDetailRouting } from "@/app/(dashboard)/models-and-endpoints/detailNavigation";
|
||||
|
|
@ -177,7 +177,9 @@ export default function ModelsAndEndpointsPage() {
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<CostOptimizationFeedbackBanner />
|
||||
{!isViewOnly && (
|
||||
<DecisionModelsBanner onAddModel={canCreate && !modelId ? () => setActiveKey("add") : undefined} />
|
||||
)}
|
||||
|
||||
{modelId ? (
|
||||
<ModelInfoView
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { useSyncExternalStore } from "react";
|
||||
import { act, renderWithProviders, screen, waitFor } from "../../../../../tests/test-utils";
|
||||
import { act, renderWithProviders, screen, waitFor, within } from "../../../../../tests/test-utils";
|
||||
import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import AddModelPanel from "./AddModelPanel";
|
||||
|
|
@ -72,6 +72,13 @@ vi.mock("@/app/(dashboard)/hooks/providers/useProviderFields", () => ({
|
|||
default_model_placeholder: "claude-3-opus",
|
||||
credential_fields: [{ key: "api_key", label: "API Key", field_type: "password", required: false }],
|
||||
},
|
||||
{
|
||||
provider: "TypeSafe",
|
||||
provider_display_name: "TypeSafe",
|
||||
litellm_provider: "typesafe",
|
||||
default_model_placeholder: "jev-latest",
|
||||
credential_fields: [{ key: "api_key", label: "API Key", field_type: "password", required: false }],
|
||||
},
|
||||
],
|
||||
isLoading: false,
|
||||
error: null,
|
||||
|
|
@ -538,3 +545,61 @@ describe("AddModelPanel behaviours the removed Advanced Settings form instance n
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("AddModelPanel decision models", () => {
|
||||
const DECISION_COST_MAP = {
|
||||
"typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" },
|
||||
"gpt-4o-2024-08-06": { litellm_provider: "openai", mode: "chat" },
|
||||
"gpt-6-luna": { litellm_provider: "openai", mode: "chat", supported_endpoints: ["/v1/decisions"] },
|
||||
"claude-sonnet-4-5": { litellm_provider: "anthropic", mode: "chat" },
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockUseModelCostMap.mockReturnValue({ data: DECISION_COST_MAP });
|
||||
mockPtuEnabled.mockReturnValue(false);
|
||||
mockAuthorized.mockReturnValue(PROXY_ADMIN);
|
||||
});
|
||||
|
||||
const decisionNotice = () => screen.queryByRole("note", { name: "Decision model notice" });
|
||||
|
||||
it("finds a provider by the name of one of its decision models", async () => {
|
||||
const { user } = await setup();
|
||||
await user.type(screen.getByRole("combobox", { name: /provider/i }), "jev");
|
||||
|
||||
expect(await screen.findByRole("option", { name: /TypeSafe/ })).toBeInTheDocument();
|
||||
expect(screen.queryByRole("option", { name: /Anthropic/ })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("points a decision-only provider at the decision routes, the docs, and the playground", async () => {
|
||||
const { user } = await setup();
|
||||
await user.type(screen.getByRole("combobox", { name: /provider/i }), "TypeSafe");
|
||||
await user.click(await screen.findByRole("option", { name: /TypeSafe/ }));
|
||||
|
||||
const note = await screen.findByRole("note", { name: "Decision model notice" });
|
||||
expect(note).toHaveTextContent("/v1/decisions");
|
||||
expect(within(note).getByRole("link", { name: "How to call decision models" })).toHaveAttribute(
|
||||
"href",
|
||||
"https://docs.litellm.ai/docs/decisions",
|
||||
);
|
||||
expect(within(note).getByRole("link", { name: "test it in the System One playground" })).toHaveAttribute(
|
||||
"href",
|
||||
"/ui/playground?tab=system-one",
|
||||
);
|
||||
});
|
||||
|
||||
it("shows the notice on a chat provider only once one of its decision models is picked", async () => {
|
||||
const { user } = await setup();
|
||||
await user.click(screen.getByRole("combobox", { name: /provider/i }));
|
||||
await user.click(await screen.findByRole("option", { name: /OpenAI/ }));
|
||||
await user.click(await screen.findByPlaceholderText("Select models"));
|
||||
await user.click(await screen.findByText("gpt-4o-2024-08-06"));
|
||||
expect(decisionNotice()).not.toBeInTheDocument();
|
||||
|
||||
await user.click(await screen.findByText("gpt-6-luna"));
|
||||
|
||||
const note = await screen.findByRole("note", { name: "Decision model notice" });
|
||||
expect(note).toHaveTextContent("/v1/decisions");
|
||||
expect(note).not.toHaveTextContent("/chat/completions");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"
|
|||
import { useCredentials } from "@/app/(dashboard)/hooks/credentials/useCredentials";
|
||||
import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import { buildDecisionCatalog } from "@/lib/decisionModels";
|
||||
|
||||
const INITIAL_VALUES: MountedFormValues = { litellm_credential_name: null };
|
||||
|
||||
|
|
@ -32,6 +33,7 @@ export default function AddModelPanel() {
|
|||
() => (typeof pickedProvider === "string" ? getProviderModels(pickedProvider, modelCostMapData) : []),
|
||||
[pickedProvider, modelCostMapData],
|
||||
);
|
||||
const decisionCatalog = useMemo(() => buildDecisionCatalog(modelCostMapData), [modelCostMapData]);
|
||||
const [showAdvancedSettings, setShowAdvancedSettings] = useState(false);
|
||||
|
||||
const refresh = () => queryClient.invalidateQueries({ queryKey: ["models", "list"] });
|
||||
|
|
@ -66,6 +68,7 @@ export default function AddModelPanel() {
|
|||
setShowAdvancedSettings={setShowAdvancedSettings}
|
||||
teams={teams ?? null}
|
||||
credentials={credentialsResponse?.credentials || []}
|
||||
decisionCatalog={decisionCatalog}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,17 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|||
import SystemOneUI from "./SystemOneUI";
|
||||
import type { SystemOneResponse } from "./lib/schemas";
|
||||
|
||||
interface ModelGroupInfo {
|
||||
model_group: string;
|
||||
mode: string;
|
||||
}
|
||||
|
||||
const DECISION_AND_CHAT_MODELS: ModelGroupInfo[] = [
|
||||
{ model_group: "gpt-5.5", mode: "chat" },
|
||||
{ model_group: "jev-latest", mode: "evaluation" },
|
||||
{ model_group: "pplx-decider", mode: "evaluation" },
|
||||
];
|
||||
|
||||
const responseBody: SystemOneResponse = {
|
||||
model: "jev-1.13.0",
|
||||
answers: {
|
||||
|
|
@ -32,6 +43,21 @@ const render = (ui: ReactElement) =>
|
|||
</QueryClientProvider>,
|
||||
);
|
||||
|
||||
const modelGroupInfoResponse = (groups: ModelGroupInfo[]) =>
|
||||
({ ok: true, status: 200, text: async () => JSON.stringify({ data: groups }) }) as Response;
|
||||
|
||||
const EXPIRED_KEY_RESPONSE = {
|
||||
ok: false,
|
||||
status: 401,
|
||||
text: async () =>
|
||||
JSON.stringify({ error: { message: "Authentication Error - Expired Key.", type: "expired_key", code: "401" } }),
|
||||
} as Response;
|
||||
|
||||
const isModelLookup = (input: RequestInfo | URL) => String(input).endsWith("/model_group/info");
|
||||
|
||||
const lookupAuthHeaders = (calls: Parameters<typeof fetch>[]) =>
|
||||
calls.map(([, init]) => Object.values((init?.headers ?? {}) as Record<string, string>));
|
||||
|
||||
const createResponse = (body: SystemOneResponse, status = 200, errorText = "") =>
|
||||
({
|
||||
ok: status >= 200 && status < 300,
|
||||
|
|
@ -41,17 +67,23 @@ const createResponse = (body: SystemOneResponse, status = 200, errorText = "") =
|
|||
|
||||
describe("SystemOneUI integration", () => {
|
||||
const mockFetch = vi.fn<typeof fetch>();
|
||||
const mockModelLookup = vi.fn<typeof fetch>();
|
||||
|
||||
beforeEach(() => {
|
||||
sessionStorage.clear();
|
||||
mockFetch.mockReset();
|
||||
vi.stubGlobal("fetch", mockFetch);
|
||||
mockModelLookup.mockReset();
|
||||
vi.stubGlobal("fetch", (input: RequestInfo | URL, init?: RequestInit) =>
|
||||
isModelLookup(input) ? mockModelLookup(input, init) : mockFetch(input, init),
|
||||
);
|
||||
mockFetch.mockResolvedValue(createResponse(responseBody));
|
||||
mockModelLookup.mockResolvedValue(modelGroupInfoResponse([]));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
sessionStorage.clear();
|
||||
document.cookie = "token=; path=/; max-age=0";
|
||||
});
|
||||
|
||||
it("restores the example after the request is edited", async () => {
|
||||
|
|
@ -80,6 +112,10 @@ describe("SystemOneUI integration", () => {
|
|||
expect(screen.getByRole("note", { name: "Decision endpoint notice" })).toHaveTextContent(
|
||||
"omit model to use the proxy's configured default.",
|
||||
);
|
||||
expect(screen.getByRole("link", { name: "How to call /v1/decisions and /v1/systemone" })).toHaveAttribute(
|
||||
"href",
|
||||
"https://docs.litellm.ai/docs/decisions",
|
||||
);
|
||||
expect(screen.getByRole("link", { name: "Give us feedback on what you want for decision models" })).toHaveAttribute(
|
||||
"href",
|
||||
"https://github.com/BerriAI/litellm/discussions/44231",
|
||||
|
|
@ -87,6 +123,106 @@ describe("SystemOneUI integration", () => {
|
|||
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("opens on /v1/systemone with the first decision model the key can call", async () => {
|
||||
const user = userEvent.setup();
|
||||
mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
|
||||
render(<SystemOneUI accessToken="session-key" />);
|
||||
|
||||
expect(await screen.findByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest");
|
||||
expect(lookupAuthHeaders(mockModelLookup.mock.calls)).toEqual([expect.arrayContaining(["Bearer session-key"])]);
|
||||
await user.click(screen.getByRole("button", { name: "Send" }));
|
||||
|
||||
expect(await screen.findByText("Selected choice")).toBeInTheDocument();
|
||||
expect(mockFetch.mock.calls[0]?.[0]).toMatch(/\/v1\/systemone$/);
|
||||
expect(mockFetch.mock.calls[0]?.[0]).not.toMatch(/typesafe/);
|
||||
expect(JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string)).toMatchObject({
|
||||
model: "jev-latest",
|
||||
questions: { has_repro_steps: { type: "noul" } },
|
||||
});
|
||||
});
|
||||
|
||||
it("offers only decision models and writes the picked one into the request", async () => {
|
||||
const user = userEvent.setup();
|
||||
mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
|
||||
render(<SystemOneUI accessToken="session-key" />);
|
||||
const picker = await screen.findByRole("combobox", { name: "Decision model" });
|
||||
|
||||
await user.clear(picker);
|
||||
await user.type(picker, "p");
|
||||
expect(await screen.findByRole("option", { name: "pplx-decider" })).toBeInTheDocument();
|
||||
expect(screen.queryByRole("option", { name: "gpt-5.5" })).not.toBeInTheDocument();
|
||||
await user.click(screen.getByRole("option", { name: "pplx-decider" }));
|
||||
|
||||
const editor = screen.getByRole("textbox", { name: "System One JSON payload" });
|
||||
expect(JSON.parse((editor as HTMLTextAreaElement).value)).toMatchObject({
|
||||
model: "pplx-decider",
|
||||
questions: { has_repro_steps: { type: "noul" } },
|
||||
});
|
||||
await user.click(screen.getByRole("button", { name: "Send" }));
|
||||
expect(await screen.findByText("Selected choice")).toBeInTheDocument();
|
||||
expect(JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string).model).toBe("pplx-decider");
|
||||
});
|
||||
|
||||
it("links to Add Model when the key cannot call any decision model", async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<SystemOneUI accessToken="session-key" />);
|
||||
screen.getByRole("combobox", { name: "Decision endpoint" }).focus();
|
||||
await user.keyboard("{ArrowDown}");
|
||||
await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" }));
|
||||
|
||||
expect(await screen.findByRole("link", { name: "Add a decision model" })).toHaveAttribute(
|
||||
"href",
|
||||
"/ui/models-and-endpoints",
|
||||
);
|
||||
expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveAttribute(
|
||||
"placeholder",
|
||||
"No decision models yet",
|
||||
);
|
||||
});
|
||||
|
||||
it("looks up a typed virtual key's decision models once, after typing settles", async () => {
|
||||
const user = userEvent.setup();
|
||||
mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
|
||||
render(<SystemOneUI accessToken={null} disabledPersonalKeyCreation />);
|
||||
|
||||
await user.type(screen.getByLabelText("Virtual Key", { exact: true }), "sk-typed-key");
|
||||
|
||||
expect(await screen.findByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest");
|
||||
expect(lookupAuthHeaders(mockModelLookup.mock.calls)).toEqual([expect.arrayContaining(["Bearer sk-typed-key"])]);
|
||||
});
|
||||
|
||||
it("keeps the admin signed in when a typed virtual key has expired", async () => {
|
||||
document.cookie = "token=ui-session; path=/";
|
||||
mockModelLookup.mockResolvedValue(EXPIRED_KEY_RESPONSE);
|
||||
render(<SystemOneUI accessToken={null} disabledPersonalKeyCreation />);
|
||||
|
||||
fireEvent.change(screen.getByLabelText("Virtual Key", { exact: true }), { target: { value: "sk-expired" } });
|
||||
|
||||
await waitFor(() => expect(mockModelLookup).toHaveBeenCalled());
|
||||
await act(async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
});
|
||||
expect(document.cookie).toContain("token=ui-session");
|
||||
expect(screen.getByRole("combobox", { name: "Decision endpoint" })).toHaveTextContent("/typesafe/v1/systemone");
|
||||
});
|
||||
|
||||
it("keeps an edited TypeSafe request in place when decision models load afterwards", async () => {
|
||||
const models = Promise.withResolvers<Response>();
|
||||
mockModelLookup.mockReturnValue(models.promise);
|
||||
render(<SystemOneUI accessToken="session-key" />);
|
||||
const editor = screen.getByRole("textbox", { name: "System One JSON payload" });
|
||||
const draft = JSON.stringify({ state: "edited", questions: { q: { type: "noul", instructions: "Yes?" } } });
|
||||
fireEvent.change(editor, { target: { value: draft } });
|
||||
|
||||
await act(async () => {
|
||||
models.resolve(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS));
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
});
|
||||
|
||||
expect(editor).toHaveValue(draft);
|
||||
expect(screen.queryByRole("combobox", { name: "Decision model" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows invalid JSON and disables Send", async () => {
|
||||
render(<SystemOneUI accessToken="session-key" />);
|
||||
|
||||
|
|
|
|||
|
|
@ -4,24 +4,27 @@ import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
|
|||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { SearchSelect } from "@/components/shared/SearchSelect";
|
||||
import { DECISIONS_DOCS_URL } from "@/lib/decisionModels";
|
||||
import { uiHref } from "@/utils/uiHref";
|
||||
import { useMutation } from "@tanstack/react-query";
|
||||
import { Code, Info, LoaderCircle, RotateCcw, Send } from "lucide-react";
|
||||
import { useEffect, useMemo, useRef, useState } from "react";
|
||||
import { makeSystemOneRequest } from "../../llm_calls/system_one";
|
||||
import { DECISIONS_EXAMPLE, SYSTEM_ONE_EXAMPLE } from "./lib/example";
|
||||
import { PLACEHOLDER_DECISION_MODEL, SYSTEM_ONE_EXAMPLE, decisionsExample } from "./lib/example";
|
||||
import { payloadModel, withPayloadModel } from "./lib/payloadModel";
|
||||
import type { DecisionEndpoint, PlaygroundRequest } from "./lib/schemas";
|
||||
import JsonEditor from "./JsonEditor";
|
||||
import QuestionBreakdown from "./QuestionBreakdown";
|
||||
import ResponseView from "./ResponseView";
|
||||
import { validateSystemOnePayload } from "./lib/validatePayload";
|
||||
import { useDecisionModels, type ApiKeySource } from "./useDecisionModels";
|
||||
|
||||
interface SystemOneUIProps {
|
||||
accessToken: string | null;
|
||||
disabledPersonalKeyCreation?: boolean;
|
||||
}
|
||||
|
||||
type ApiKeySource = "session" | "custom";
|
||||
|
||||
interface SystemOneSendVariables {
|
||||
payload: PlaygroundRequest;
|
||||
endpoint: DecisionEndpoint;
|
||||
|
|
@ -30,7 +33,6 @@ interface SystemOneSendVariables {
|
|||
}
|
||||
|
||||
const EXAMPLE_PAYLOAD = JSON.stringify(SYSTEM_ONE_EXAMPLE, null, 2);
|
||||
const DECISIONS_EXAMPLE_PAYLOAD = JSON.stringify(DECISIONS_EXAMPLE, null, 2);
|
||||
const DECISION_MODELS_DISCUSSION_URL = "https://github.com/BerriAI/litellm/discussions/44231";
|
||||
|
||||
function getCustomProxyBaseUrl(): string | undefined {
|
||||
|
|
@ -40,16 +42,23 @@ function getCustomProxyBaseUrl(): string | undefined {
|
|||
export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = false }: SystemOneUIProps) {
|
||||
const [apiKeySource, setApiKeySource] = useState<ApiKeySource>(disabledPersonalKeyCreation ? "custom" : "session");
|
||||
const [customApiKey, setCustomApiKey] = useState("");
|
||||
const [endpoint, setEndpoint] = useState<DecisionEndpoint>("/typesafe/v1/systemone");
|
||||
const [payloads, setPayloads] = useState<Record<DecisionEndpoint, string>>({
|
||||
"/v1/systemone": DECISIONS_EXAMPLE_PAYLOAD,
|
||||
"/typesafe/v1/systemone": EXAMPLE_PAYLOAD,
|
||||
});
|
||||
const rawPayload = payloads[endpoint];
|
||||
const examplePayload = endpoint === "/v1/systemone" ? DECISIONS_EXAMPLE_PAYLOAD : EXAMPLE_PAYLOAD;
|
||||
const effectiveApiKey = apiKeySource === "session" ? accessToken || "" : customApiKey.trim();
|
||||
const { decisionModels, isLoaded: decisionModelsLoaded } = useDecisionModels(
|
||||
apiKeySource,
|
||||
effectiveApiKey,
|
||||
getCustomProxyBaseUrl(),
|
||||
);
|
||||
const [chosenEndpoint, setChosenEndpoint] = useState<DecisionEndpoint | null>(null);
|
||||
const endpoint: DecisionEndpoint =
|
||||
chosenEndpoint ?? (decisionModels.length > 0 ? "/v1/systemone" : "/typesafe/v1/systemone");
|
||||
const [drafts, setDrafts] = useState<Partial<Record<DecisionEndpoint, string>>>({});
|
||||
const examplePayload =
|
||||
endpoint === "/v1/systemone"
|
||||
? JSON.stringify(decisionsExample(decisionModels[0] ?? PLACEHOLDER_DECISION_MODEL), null, 2)
|
||||
: EXAMPLE_PAYLOAD;
|
||||
const rawPayload = drafts[endpoint] ?? examplePayload;
|
||||
const activeController = useRef<AbortController | null>(null);
|
||||
const validation = useMemo(() => validateSystemOnePayload(rawPayload, endpoint), [rawPayload, endpoint]);
|
||||
const effectiveApiKey = apiKeySource === "session" ? accessToken || "" : customApiKey.trim();
|
||||
const hasSyntaxError = validation.issues.some((issue) => issue.path === "syntax");
|
||||
|
||||
const systemOne = useMutation({
|
||||
|
|
@ -76,7 +85,21 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
|
|||
function handlePayloadChange(value: string) {
|
||||
if (value !== rawPayload) {
|
||||
clearRequestState();
|
||||
setPayloads((current) => ({ ...current, [endpoint]: value }));
|
||||
setChosenEndpoint(endpoint);
|
||||
setDrafts((current) => ({ ...current, [endpoint]: value }));
|
||||
}
|
||||
}
|
||||
|
||||
function handleResetExample() {
|
||||
clearRequestState();
|
||||
setChosenEndpoint(endpoint);
|
||||
setDrafts((current) => Object.fromEntries(Object.entries(current).filter(([key]) => key !== endpoint)));
|
||||
}
|
||||
|
||||
function handleModelPick(model: string | null) {
|
||||
const next = model === null ? undefined : withPayloadModel(rawPayload, model);
|
||||
if (next !== undefined) {
|
||||
handlePayloadChange(next);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -91,6 +114,7 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
|
|||
return;
|
||||
}
|
||||
clearRequestState();
|
||||
setChosenEndpoint(endpoint);
|
||||
const controller = new AbortController();
|
||||
activeController.current = controller;
|
||||
const variables = { payload: validation.payload, apiKey: effectiveApiKey, signal: controller.signal, endpoint };
|
||||
|
|
@ -107,7 +131,7 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
|
|||
onValueChange={(value) => {
|
||||
if (value === "/v1/systemone" || value === "/typesafe/v1/systemone") {
|
||||
clearRequestState();
|
||||
setEndpoint(value);
|
||||
setChosenEndpoint(value);
|
||||
}
|
||||
}}
|
||||
>
|
||||
|
|
@ -121,6 +145,28 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
|
|||
<SelectItem value="/typesafe/v1/systemone">TypeSafe · /typesafe/v1/systemone</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
{endpoint === "/v1/systemone" && (
|
||||
<>
|
||||
<span className="ml-2 text-sm font-medium text-muted-foreground">Model</span>
|
||||
<div className="w-80">
|
||||
<SearchSelect
|
||||
aria-label="Decision model"
|
||||
options={decisionModels.map((model) => ({ label: model, value: model }))}
|
||||
value={payloadModel(rawPayload) ?? null}
|
||||
onValueChange={handleModelPick}
|
||||
placeholder={decisionModels.length > 0 ? "Pick a decision model" : "No decision models yet"}
|
||||
emptyText="No decision models found"
|
||||
disabled={hasSyntaxError}
|
||||
allowClear={false}
|
||||
/>
|
||||
</div>
|
||||
{decisionModelsLoaded && decisionModels.length === 0 && (
|
||||
<a href={uiHref("models-and-endpoints")} className="text-sm text-primary underline">
|
||||
Add a decision model
|
||||
</a>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-3">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
|
|
@ -156,11 +202,7 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
|
|||
)}
|
||||
</div>
|
||||
<div className="ml-auto flex items-center gap-2">
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={() => handlePayloadChange(examplePayload)}
|
||||
disabled={rawPayload === examplePayload}
|
||||
>
|
||||
<Button variant="outline" onClick={handleResetExample} disabled={rawPayload === examplePayload}>
|
||||
<RotateCcw />
|
||||
Reset example
|
||||
</Button>
|
||||
|
|
@ -186,8 +228,12 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
|
|||
</AlertTitle>
|
||||
<AlertDescription>
|
||||
{endpoint === "/v1/systemone"
|
||||
? "Sends choice, noul, and score questions through /v1/systemone. Replace the example model with a decision model configured on your proxy, or omit model to use the proxy's configured default."
|
||||
? "Sends choice, noul, and score questions through /v1/systemone to a decision model on your proxy. Pick one under Model, or omit model to use the proxy's configured default."
|
||||
: "Sends requests through /typesafe/v1/systemone and requires TYPESAFE_API_KEY on the proxy."}{" "}
|
||||
<a href={DECISIONS_DOCS_URL} target="_blank" rel="noopener noreferrer" className="underline">
|
||||
How to call /v1/decisions and /v1/systemone
|
||||
</a>
|
||||
{" · "}
|
||||
<a href={DECISION_MODELS_DISCUSSION_URL} target="_blank" rel="noopener noreferrer" className="underline">
|
||||
Give us feedback on what you want for decision models
|
||||
</a>
|
||||
|
|
|
|||
|
|
@ -37,7 +37,6 @@ export const SYSTEM_ONE_EXAMPLE = {
|
|||
},
|
||||
} satisfies SystemOneRequest;
|
||||
|
||||
export const DECISIONS_EXAMPLE: DecisionRequest = {
|
||||
...SYSTEM_ONE_EXAMPLE,
|
||||
model: "your-decision-model",
|
||||
};
|
||||
export const PLACEHOLDER_DECISION_MODEL = "your-decision-model";
|
||||
|
||||
export const decisionsExample = (model: string): DecisionRequest => ({ ...SYSTEM_ONE_EXAMPLE, model });
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { payloadModel, withPayloadModel } from "./payloadModel";
|
||||
|
||||
describe("payloadModel", () => {
|
||||
it("reads the model from a JSON object payload", () => {
|
||||
expect(payloadModel('{"model": "jev-latest", "state": "hi"}')).toBe("jev-latest");
|
||||
});
|
||||
|
||||
it("is undefined when the model is missing, empty, or not a string", () => {
|
||||
expect(payloadModel('{"state": "hi"}')).toBeUndefined();
|
||||
expect(payloadModel('{"model": ""}')).toBeUndefined();
|
||||
expect(payloadModel('{"model": 3}')).toBeUndefined();
|
||||
});
|
||||
|
||||
it("is undefined when the payload is not a JSON object", () => {
|
||||
expect(payloadModel("{not json")).toBeUndefined();
|
||||
expect(payloadModel('["jev-latest"]')).toBeUndefined();
|
||||
expect(payloadModel('"jev-latest"')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("withPayloadModel", () => {
|
||||
it("replaces the model and keeps every other field", () => {
|
||||
const next = withPayloadModel(
|
||||
'{"model": "old", "state": "hi", "questions": {"q": {"type": "noul"}}}',
|
||||
"jev-latest",
|
||||
);
|
||||
expect(next).toBeDefined();
|
||||
expect(JSON.parse(next ?? "")).toEqual({ model: "jev-latest", state: "hi", questions: { q: { type: "noul" } } });
|
||||
});
|
||||
|
||||
it("adds the model as the first field when the payload has none", () => {
|
||||
const next = withPayloadModel('{"state": "hi"}', "jev-latest");
|
||||
expect(Object.keys(JSON.parse(next ?? "{}"))).toEqual(["model", "state"]);
|
||||
});
|
||||
|
||||
it("leaves a payload that is not a JSON object alone", () => {
|
||||
expect(withPayloadModel("{not json", "jev-latest")).toBeUndefined();
|
||||
expect(withPayloadModel("[1, 2]", "jev-latest")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
import { z } from "zod";
|
||||
import { parseJson } from "./validatePayload";
|
||||
|
||||
const jsonObject = z.record(z.string(), z.unknown());
|
||||
|
||||
const parseObject = (raw: string): Record<string, unknown> | undefined => {
|
||||
const json = parseJson(raw);
|
||||
if (!json.ok) {
|
||||
return undefined;
|
||||
}
|
||||
const parsed = jsonObject.safeParse(json.value);
|
||||
return parsed.success ? parsed.data : undefined;
|
||||
};
|
||||
|
||||
export const payloadModel = (raw: string): string | undefined => {
|
||||
const model = parseObject(raw)?.model;
|
||||
return typeof model === "string" && model !== "" ? model : undefined;
|
||||
};
|
||||
|
||||
export const withPayloadModel = (raw: string, model: string): string | undefined => {
|
||||
const payload = parseObject(raw);
|
||||
if (payload === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const rest = Object.fromEntries(Object.entries(payload).filter(([key]) => key !== "model"));
|
||||
return JSON.stringify({ model, ...rest }, null, 2);
|
||||
};
|
||||
|
|
@ -24,7 +24,7 @@ const invalid = (path: string, message: string): SystemOnePayloadValidation => (
|
|||
issues: [{ path, message, severity: "error" }],
|
||||
});
|
||||
|
||||
function parseJson(raw: string): { ok: true; value: unknown } | { ok: false; message: string } {
|
||||
export function parseJson(raw: string): { ok: true; value: unknown } | { ok: false; message: string } {
|
||||
try {
|
||||
return { ok: true, value: JSON.parse(raw) };
|
||||
} catch (error: unknown) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
import { useDebouncedValue } from "@tanstack/react-pacer/debouncer";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { fetchDecisionModels } from "../../llm_calls/system_one";
|
||||
|
||||
export type ApiKeySource = "session" | "custom";
|
||||
|
||||
export interface DecisionModelsLookup {
|
||||
decisionModels: string[];
|
||||
isLoaded: boolean;
|
||||
}
|
||||
|
||||
const TYPED_KEY_DEBOUNCE_WAIT_MS = 400;
|
||||
const NO_DECISION_MODELS: string[] = [];
|
||||
|
||||
export function useDecisionModels(
|
||||
apiKeySource: ApiKeySource,
|
||||
apiKey: string,
|
||||
customBaseUrl: string | undefined,
|
||||
): DecisionModelsLookup {
|
||||
const typedApiKey = apiKeySource === "custom" ? apiKey : "";
|
||||
const [settledTypedApiKey] = useDebouncedValue(typedApiKey, { wait: TYPED_KEY_DEBOUNCE_WAIT_MS });
|
||||
const lookupApiKey = apiKeySource === "session" ? apiKey : settledTypedApiKey;
|
||||
const queryOptions = {
|
||||
queryKey: ["playground", "systemOne", "decisionModels", customBaseUrl ?? "", lookupApiKey],
|
||||
queryFn: ({ signal }: { signal: AbortSignal }) => fetchDecisionModels(lookupApiKey, customBaseUrl, signal),
|
||||
enabled: lookupApiKey !== "",
|
||||
retry: false,
|
||||
staleTime: 60 * 1000,
|
||||
};
|
||||
const query = useQuery(queryOptions);
|
||||
return { decisionModels: query.data ?? NO_DECISION_MODELS, isLoaded: query.isSuccess };
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { makeSystemOneRequest } from "./system_one";
|
||||
import { fetchDecisionModels, makeSystemOneRequest } from "./system_one";
|
||||
import type { SystemOneRequest, SystemOneResponse } from "../components/systemOneUI/lib/schemas";
|
||||
|
||||
vi.mock("@/components/networking", () => ({
|
||||
|
|
@ -106,3 +106,46 @@ describe("makeSystemOneRequest", () => {
|
|||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("fetchDecisionModels", () => {
|
||||
const mockFetch = vi.fn<(input: RequestInfo | URL, init?: RequestInit) => Promise<Response>>();
|
||||
const modelGroupInfo = (data: unknown) => ({ ok: true, text: async () => JSON.stringify({ data }) }) as Response;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.stubGlobal("fetch", mockFetch);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("lists the key's decision models from the proxy that answers the request, sorted by name", async () => {
|
||||
mockFetch.mockResolvedValueOnce(
|
||||
modelGroupInfo([
|
||||
{ model_group: "pplx-decider", mode: "evaluation" },
|
||||
{ model_group: "gpt-5.5", mode: "chat" },
|
||||
{ model_group: "jev-latest", mode: "evaluation" },
|
||||
{ model_group: "unknown-mode", mode: null },
|
||||
]),
|
||||
);
|
||||
|
||||
await expect(fetchDecisionModels("virtual-key", "https://custom.example.com/")).resolves.toEqual([
|
||||
"jev-latest",
|
||||
"pplx-decider",
|
||||
]);
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
"https://custom.example.com/model_group/info",
|
||||
expect.objectContaining({
|
||||
method: "GET",
|
||||
headers: expect.objectContaining({ Authorization: "Bearer virtual-key" }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a response that is not a model group list", async () => {
|
||||
mockFetch.mockResolvedValueOnce(modelGroupInfo([{ mode: "evaluation" }]));
|
||||
|
||||
await expect(fetchDecisionModels("virtual-key")).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import { z } from "zod";
|
||||
import { getGlobalLitellmHeaderName, getProxyBaseUrl } from "@/components/networking";
|
||||
import { withRequiredHeaders } from "@/components/llm_calls/request_headers";
|
||||
import { createApiClient } from "@/lib/http/client";
|
||||
import { isDecisionMode } from "@/lib/decisionModels";
|
||||
import {
|
||||
systemOneResponseSchema,
|
||||
type DecisionEndpoint,
|
||||
|
|
@ -13,24 +15,54 @@ export interface SystemOneResult {
|
|||
latencyMs: number;
|
||||
}
|
||||
|
||||
export async function makeSystemOneRequest(
|
||||
payload: PlaygroundRequest,
|
||||
accessToken: string,
|
||||
customBaseUrl?: string,
|
||||
{ signal, endpoint = "/typesafe/v1/systemone" }: { signal?: AbortSignal; endpoint?: DecisionEndpoint } = {},
|
||||
): Promise<SystemOneResult> {
|
||||
const modelGroupInfoSchema = z.object({
|
||||
data: z.array(z.object({ model_group: z.string(), mode: z.string().nullish() })),
|
||||
});
|
||||
|
||||
function proxyClient(customBaseUrl?: string) {
|
||||
const proxyBaseUrl = customBaseUrl || getProxyBaseUrl();
|
||||
const normalizedBaseUrl = proxyBaseUrl.endsWith("/") ? proxyBaseUrl.slice(0, -1) : proxyBaseUrl;
|
||||
const startedAt = performance.now();
|
||||
const headers = withRequiredHeaders(
|
||||
return createApiClient({ getBaseUrl: () => normalizedBaseUrl });
|
||||
}
|
||||
|
||||
function authHeaders(accessToken: string): Record<string, string> {
|
||||
return withRequiredHeaders(
|
||||
{},
|
||||
{
|
||||
"Content-Type": "application/json",
|
||||
[getGlobalLitellmHeaderName()]: `Bearer ${accessToken}`,
|
||||
},
|
||||
);
|
||||
const client = createApiClient({ getBaseUrl: () => normalizedBaseUrl });
|
||||
const body = await client.post<unknown>(endpoint, { body: payload, headers, signal });
|
||||
}
|
||||
|
||||
export async function fetchDecisionModels(
|
||||
accessToken: string,
|
||||
customBaseUrl?: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<string[]> {
|
||||
const body = await proxyClient(customBaseUrl).get<unknown>("/model_group/info", {
|
||||
headers: authHeaders(accessToken),
|
||||
signal,
|
||||
});
|
||||
return modelGroupInfoSchema
|
||||
.parse(body)
|
||||
.data.filter((group) => isDecisionMode(group.mode))
|
||||
.map((group) => group.model_group)
|
||||
.sort((a, b) => a.localeCompare(b));
|
||||
}
|
||||
|
||||
export async function makeSystemOneRequest(
|
||||
payload: PlaygroundRequest,
|
||||
accessToken: string,
|
||||
customBaseUrl?: string,
|
||||
{ signal, endpoint = "/typesafe/v1/systemone" }: { signal?: AbortSignal; endpoint?: DecisionEndpoint } = {},
|
||||
): Promise<SystemOneResult> {
|
||||
const startedAt = performance.now();
|
||||
const body = await proxyClient(customBaseUrl).post<unknown>(endpoint, {
|
||||
body: payload,
|
||||
headers: authHeaders(accessToken),
|
||||
signal,
|
||||
});
|
||||
const parsed = systemOneResponseSchema.safeParse(body);
|
||||
if (!parsed.success) {
|
||||
throw new Error("System One response has an invalid shape.");
|
||||
|
|
|
|||
|
|
@ -15,7 +15,15 @@ function PublicModelHubTableContent() {
|
|||
setAccessToken(key);
|
||||
}, [key]);
|
||||
|
||||
return <ModelHubTable accessToken={accessToken} publicPage={true} premiumUser={false} userRole={null} />;
|
||||
return (
|
||||
<ModelHubTable
|
||||
accessToken={accessToken}
|
||||
publicPage={true}
|
||||
premiumUser={false}
|
||||
userRole={null}
|
||||
canOpenPlayground={false}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
export default function PublicModelHubTable() {
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ vi.mock("@/components/networking", () => ({
|
|||
modelHubPublicModelsCall: vi.fn(),
|
||||
modelHubCall: vi.fn(),
|
||||
getConfigFieldSetting: vi.fn(),
|
||||
getGlobalLitellmHeaderName: vi.fn(() => "X-Litellm-Key"),
|
||||
getProxyBaseUrl: vi.fn(() => "http://localhost:4000"),
|
||||
getAgentsList: vi.fn(),
|
||||
fetchMCPServers: vi.fn(),
|
||||
|
|
@ -115,7 +116,15 @@ describe("ModelHubTable", () => {
|
|||
it(description, async () => {
|
||||
setupAuthRedirectTest(requireAuth, tokenValue, isTokenValid);
|
||||
|
||||
renderWithProviders(<ModelHubTable accessToken={null} publicPage={true} premiumUser={false} userRole={null} />);
|
||||
renderWithProviders(
|
||||
<ModelHubTable
|
||||
accessToken={null}
|
||||
publicPage={true}
|
||||
premiumUser={false}
|
||||
userRole={null}
|
||||
canOpenPlayground={false}
|
||||
/>,
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
if (shouldRedirect) {
|
||||
|
|
@ -148,7 +157,13 @@ describe("ModelHubTable", () => {
|
|||
});
|
||||
|
||||
renderWithProviders(
|
||||
<ModelHubTable accessToken="test-token" publicPage={false} premiumUser={false} userRole={null} />,
|
||||
<ModelHubTable
|
||||
accessToken="test-token"
|
||||
publicPage={false}
|
||||
premiumUser={false}
|
||||
userRole={null}
|
||||
canOpenPlayground
|
||||
/>,
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
|
|
@ -165,7 +180,9 @@ describe("ModelHubTable", () => {
|
|||
isLoading: false,
|
||||
});
|
||||
|
||||
renderWithProviders(<ModelHubTable accessToken={null} publicPage={false} premiumUser={false} userRole={null} />);
|
||||
renderWithProviders(
|
||||
<ModelHubTable accessToken={null} publicPage={false} premiumUser={false} userRole={null} canOpenPlayground />,
|
||||
);
|
||||
|
||||
expect(await screen.findByText("No models yet")).toBeInTheDocument();
|
||||
expect(networking.modelHubCall).not.toHaveBeenCalled();
|
||||
|
|
@ -191,7 +208,15 @@ describe("ModelHubTable", () => {
|
|||
isLoading: false,
|
||||
});
|
||||
|
||||
renderWithProviders(<ModelHubTable accessToken={null} publicPage={true} premiumUser={false} userRole={null} />);
|
||||
renderWithProviders(
|
||||
<ModelHubTable
|
||||
accessToken={null}
|
||||
publicPage={true}
|
||||
premiumUser={false}
|
||||
userRole={null}
|
||||
canOpenPlayground={false}
|
||||
/>,
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(getUiConfigMock).toHaveBeenCalled();
|
||||
|
|
@ -204,6 +229,66 @@ describe("ModelHubTable", () => {
|
|||
expect(getUiConfigCallOrder).toBeLessThan(modelHubPublicModelsCallOrder);
|
||||
});
|
||||
|
||||
describe("model details usage example", () => {
|
||||
const openDetails = async (
|
||||
model: { model_group: string; providers: string[]; mode: string },
|
||||
canOpenPlayground = true,
|
||||
) => {
|
||||
vi.mocked(networking.modelHubCall).mockResolvedValue({ data: [model] });
|
||||
vi.mocked(networking.getConfigFieldSetting).mockResolvedValue({ field_value: false });
|
||||
vi.mocked(networking.getAgentsList).mockResolvedValue({ agents: [] });
|
||||
vi.mocked(networking.fetchMCPServers).mockResolvedValue([]);
|
||||
vi.mocked(networking.getUiSettings).mockResolvedValue({ values: {} });
|
||||
mockUseUISettings.mockReturnValue({ data: { values: {} }, isLoading: false });
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(
|
||||
<ModelHubTable
|
||||
accessToken="test-token"
|
||||
publicPage={false}
|
||||
premiumUser={false}
|
||||
userRole="Admin"
|
||||
canOpenPlayground={canOpenPlayground}
|
||||
/>,
|
||||
);
|
||||
await user.click(await screen.findByRole("button", { name: model.model_group }));
|
||||
return screen.findByRole("dialog");
|
||||
};
|
||||
|
||||
it("shows a decision model how to call it on /v1/systemone, with the docs and the playground", async () => {
|
||||
const dialog = await openDetails({ model_group: "jev-latest", providers: ["typesafe"], mode: "evaluation" });
|
||||
|
||||
expect(dialog).toHaveTextContent('"http://localhost:4000/v1/systemone"');
|
||||
expect(dialog).toHaveTextContent('headers={"X-Litellm-Key": "Bearer your_api_key"}');
|
||||
expect(dialog).not.toHaveTextContent("chat.completions");
|
||||
expect(screen.getByRole("link", { name: "How to call decision models" })).toHaveAttribute(
|
||||
"href",
|
||||
"https://docs.litellm.ai/docs/decisions",
|
||||
);
|
||||
expect(screen.getByRole("link", { name: "Try it in the Playground" })).toHaveAttribute(
|
||||
"href",
|
||||
"/ui/playground?tab=system-one",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not link a view-only session to the Playground it cannot open", async () => {
|
||||
const dialog = await openDetails(
|
||||
{ model_group: "jev-latest", providers: ["typesafe"], mode: "evaluation" },
|
||||
false,
|
||||
);
|
||||
|
||||
expect(dialog).toHaveTextContent('"http://localhost:4000/v1/systemone"');
|
||||
expect(screen.getByRole("link", { name: "How to call decision models" })).toBeInTheDocument();
|
||||
expect(screen.queryByRole("link", { name: "Try it in the Playground" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("keeps the chat completions example for a chat model", async () => {
|
||||
const dialog = await openDetails({ model_group: "claude-opus-4-8", providers: ["anthropic"], mode: "chat" });
|
||||
|
||||
expect(dialog).toHaveTextContent("client.chat.completions.create(");
|
||||
expect(screen.queryByRole("link", { name: "How to call decision models" })).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
describe("hub tabs", () => {
|
||||
const renderHub = async (agents: object[] = [], mcpServers: Promise<MCPServerData[]> = Promise.resolve([])) => {
|
||||
vi.mocked(networking.modelHubCall).mockResolvedValue({
|
||||
|
|
@ -217,7 +302,13 @@ describe("ModelHubTable", () => {
|
|||
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(
|
||||
<ModelHubTable accessToken="test-token" publicPage={false} premiumUser={false} userRole="Admin" />,
|
||||
<ModelHubTable
|
||||
accessToken="test-token"
|
||||
publicPage={false}
|
||||
premiumUser={false}
|
||||
userRole="Admin"
|
||||
canOpenPlayground
|
||||
/>,
|
||||
);
|
||||
return { user, search: await screen.findByPlaceholderText("Search model names...") };
|
||||
};
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import MakeMCPPublicForm from "@/components/AIHub/forms/MakeMCPPublicForm";
|
|||
import MakeModelPublicForm from "@/components/AIHub/forms/MakeModelPublicForm";
|
||||
import { getMCPHubTableColumns, MCPServerData } from "@/components/AIHub/MCPHubTableColumns";
|
||||
import { getModelHubTableColumns, ModelHubData } from "@/components/AIHub/ModelHubTableColumns";
|
||||
import { modelUsageExample } from "@/components/AIHub/modelUsageExample";
|
||||
import UsefulLinksManagement from "@/components/AIHub/UsefulLinksManagement";
|
||||
import { getClaudeCodePluginsList } from "@/components/networking";
|
||||
import { Plugin } from "@/components/claude_code_plugins/types";
|
||||
|
|
@ -15,6 +16,7 @@ import {
|
|||
fetchMCPServers,
|
||||
getAgentsList,
|
||||
getConfigFieldSetting,
|
||||
getGlobalLitellmHeaderName,
|
||||
getProxyBaseUrl,
|
||||
getUiConfig,
|
||||
modelHubCall,
|
||||
|
|
@ -41,12 +43,15 @@ import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"
|
|||
import { checkTokenValidity } from "@/utils/jwtUtils";
|
||||
import { getCookie } from "@/utils/cookieUtils";
|
||||
import { getLoginUrl } from "@/utils/returnUrlUtils";
|
||||
import { uiHref } from "@/utils/uiHref";
|
||||
import { DECISIONS_DOCS_URL, SYSTEM_ONE_PLAYGROUND_ROUTE, isDecisionMode } from "@/lib/decisionModels";
|
||||
|
||||
interface ModelHubTableProps {
|
||||
accessToken: string | null;
|
||||
publicPage: boolean;
|
||||
premiumUser: boolean;
|
||||
userRole: string | null;
|
||||
canOpenPlayground: boolean;
|
||||
}
|
||||
|
||||
function isMCPHubVisibilityDisabled(isLoading: boolean, servers: readonly MCPServerData[] | null): boolean {
|
||||
|
|
@ -65,7 +70,13 @@ function HubEmptyState({ title, body }: { title: string; body: string }) {
|
|||
);
|
||||
}
|
||||
|
||||
const ModelHubTable: React.FC<ModelHubTableProps> = ({ accessToken, publicPage, premiumUser, userRole }) => {
|
||||
const ModelHubTable: React.FC<ModelHubTableProps> = ({
|
||||
accessToken,
|
||||
publicPage,
|
||||
premiumUser,
|
||||
userRole,
|
||||
canOpenPlayground,
|
||||
}) => {
|
||||
const syntaxTheme = useSyntaxTheme(prism);
|
||||
// Admin Viewer follows the read-parity rule: see the AI Hub catalog, but
|
||||
// cannot toggle public visibility (write).
|
||||
|
|
@ -763,28 +774,32 @@ const ModelHubTable: React.FC<ModelHubTableProps> = ({ accessToken, publicPage,
|
|||
</div>
|
||||
)}
|
||||
|
||||
{/* Usage Example */}
|
||||
<div>
|
||||
<p className="text-lg font-semibold mb-4">Usage Example</p>
|
||||
{isDecisionMode(selectedModel.mode) && (
|
||||
<p className="mb-2 text-sm text-muted-foreground">
|
||||
Decision models answer on <code>/v1/decisions</code> and <code>/v1/systemone</code>, not{" "}
|
||||
<code>/chat/completions</code>.{" "}
|
||||
<a href={DECISIONS_DOCS_URL} target="_blank" rel="noopener noreferrer" className="underline">
|
||||
How to call decision models
|
||||
</a>
|
||||
{canOpenPlayground && (
|
||||
<>
|
||||
{" · "}
|
||||
<a href={uiHref(SYSTEM_ONE_PLAYGROUND_ROUTE)} className="underline">
|
||||
Try it in the Playground
|
||||
</a>
|
||||
</>
|
||||
)}
|
||||
</p>
|
||||
)}
|
||||
<SyntaxHighlighter language="python" className="text-sm" style={syntaxTheme}>
|
||||
{`import openai
|
||||
|
||||
client = openai.OpenAI(
|
||||
api_key="your_api_key",
|
||||
base_url="${getProxyBaseUrl()}" # Your LiteLLM Proxy URL
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="${selectedModel.model_group}",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello, how are you?"
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
print(response.choices[0].message.content)`}
|
||||
{modelUsageExample(
|
||||
selectedModel.mode,
|
||||
getProxyBaseUrl(),
|
||||
selectedModel.model_group,
|
||||
getGlobalLitellmHeaderName(),
|
||||
)}
|
||||
</SyntaxHighlighter>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,40 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { modelUsageExample } from "./modelUsageExample";
|
||||
|
||||
const BASE_URL = "https://proxy.example.com";
|
||||
|
||||
describe("modelUsageExample", () => {
|
||||
it("posts a decision model to /v1/systemone on the proxy", () => {
|
||||
const example = modelUsageExample("evaluation", BASE_URL, "jev-latest", "Authorization");
|
||||
expect(example).toContain('"https://proxy.example.com/v1/systemone"');
|
||||
expect(example).toContain('"model": "jev-latest"');
|
||||
expect(example).toContain('"type": "noul"');
|
||||
expect(example).not.toContain("chat.completions");
|
||||
});
|
||||
|
||||
it("sends the key in the header the proxy reads keys from", () => {
|
||||
expect(modelUsageExample("evaluation", BASE_URL, "jev-latest", "Authorization")).toContain(
|
||||
'headers={"Authorization": "Bearer your_api_key"}',
|
||||
);
|
||||
expect(modelUsageExample("evaluation", BASE_URL, "jev-latest", "X-Litellm-Key")).toContain(
|
||||
'headers={"X-Litellm-Key": "Bearer your_api_key"}',
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the chat completions example for every other mode", () => {
|
||||
for (const mode of ["chat", null, undefined]) {
|
||||
const example = modelUsageExample(mode, BASE_URL, "gpt-5.5", "Authorization");
|
||||
expect(example).toContain("client.chat.completions.create(");
|
||||
expect(example).toContain('base_url="https://proxy.example.com"');
|
||||
expect(example).toContain('model="gpt-5.5"');
|
||||
expect(example).not.toContain("/v1/systemone");
|
||||
}
|
||||
});
|
||||
|
||||
it("escapes quotes in a model name so the snippet stays valid Python", () => {
|
||||
expect(modelUsageExample("evaluation", BASE_URL, 'team"model', "Authorization")).toContain(
|
||||
'"model": "team\\"model"',
|
||||
);
|
||||
expect(modelUsageExample("chat", BASE_URL, 'team"model', "Authorization")).toContain('model="team\\"model"');
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
import { isDecisionMode } from "@/lib/decisionModels";
|
||||
|
||||
const chatCompletionsExample = (baseUrl: string, model: string): string => `import openai
|
||||
|
||||
client = openai.OpenAI(
|
||||
api_key="your_api_key",
|
||||
base_url=${JSON.stringify(baseUrl)} # Your LiteLLM Proxy URL
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=${JSON.stringify(model)},
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello, how are you?"
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
print(response.choices[0].message.content)`;
|
||||
|
||||
const systemOneExample = (baseUrl: string, model: string, keyHeaderName: string): string => `import requests
|
||||
|
||||
response = requests.post(
|
||||
${JSON.stringify(`${baseUrl}/v1/systemone`)}, # Your LiteLLM Proxy URL
|
||||
headers={${JSON.stringify(keyHeaderName)}: "Bearer your_api_key"},
|
||||
json={
|
||||
"model": ${JSON.stringify(model)},
|
||||
"state": "I was charged twice for my subscription this month.",
|
||||
"questions": {
|
||||
"wants_refund": {
|
||||
"type": "noul",
|
||||
"instructions": "Is the customer asking for money back?"
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
print(response.json()["answers"])`;
|
||||
|
||||
export const modelUsageExample = (
|
||||
mode: string | null | undefined,
|
||||
baseUrl: string,
|
||||
model: string,
|
||||
keyHeaderName: string,
|
||||
): string =>
|
||||
isDecisionMode(mode) ? systemOneExample(baseUrl, model, keyHeaderName) : chatCompletionsExample(baseUrl, model);
|
||||
|
|
@ -7,6 +7,7 @@ import { Providers } from "../provider_info_helpers";
|
|||
import { projectMountedValues, useMountRegistry, type MountedFormValues } from "../common_components/MountedFormField";
|
||||
import { useForm } from "react-hook-form";
|
||||
import AddModelForm from "./AddModelForm";
|
||||
import { EMPTY_DECISION_CATALOG } from "@/lib/decisionModels";
|
||||
|
||||
vi.mock("../molecules/models/ProviderLogo", () => ({
|
||||
ProviderLogo: ({ provider, className }: { provider: string; className?: string }) => (
|
||||
|
|
@ -172,6 +173,7 @@ const createTestProps = (userRole = "proxy_admin", userId = "user-1", isTeamAdmi
|
|||
showAdvancedSettings: false,
|
||||
teams,
|
||||
credentials,
|
||||
decisionCatalog: EMPTY_DECISION_CATALOG,
|
||||
userRole,
|
||||
userId,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -37,6 +37,8 @@ import LiteLLMModelNameField from "./litellm_model_name";
|
|||
import ConnectionErrorDisplay from "./model_connection_test";
|
||||
import ProviderSpecificFields from "./provider_specific_fields";
|
||||
import { TEST_MODES } from "./add_model_modes";
|
||||
import DecisionModelNote from "./DecisionModelNote";
|
||||
import { decisionModelsSublabel, type DecisionCatalog } from "@/lib/decisionModels";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import { credentialsKeys } from "@/app/(dashboard)/hooks/credentials/useCredentials";
|
||||
import { extractProxyErrorMessage } from "@/lib/http/client";
|
||||
|
|
@ -58,6 +60,7 @@ interface AddModelFormProps {
|
|||
setShowAdvancedSettings: (show: boolean) => void;
|
||||
teams: Team[] | null;
|
||||
credentials: CredentialItem[];
|
||||
decisionCatalog: DecisionCatalog;
|
||||
}
|
||||
|
||||
const connectionTestModelName = (values: MountedFormValues): string | undefined => {
|
||||
|
|
@ -81,6 +84,7 @@ const AddModelForm: React.FC<AddModelFormProps> = ({
|
|||
setShowAdvancedSettings,
|
||||
teams,
|
||||
credentials,
|
||||
decisionCatalog,
|
||||
}) => {
|
||||
const [testMode, setTestMode] = useState<string>("chat");
|
||||
const [isResultModalVisible, setIsResultModalVisible] = useState<boolean>(false);
|
||||
|
|
@ -98,6 +102,7 @@ const AddModelForm: React.FC<AddModelFormProps> = ({
|
|||
const guardrailsList = guardrailsData?.guardrails.map((g) => g.guardrail_name);
|
||||
const { data: tagsList } = useTags();
|
||||
const selectedCredentialName = useWatch({ control: form.control, name: "litellm_credential_name" });
|
||||
const pickedProvider = useWatch({ control: form.control, name: "custom_llm_provider" });
|
||||
const queryClient = useQueryClient();
|
||||
const [isFederatedCredentialModalOpen, setIsFederatedCredentialModalOpen] = useState(false);
|
||||
const canCreateFederatedCredential =
|
||||
|
|
@ -148,11 +153,16 @@ const AddModelForm: React.FC<AddModelFormProps> = ({
|
|||
sortedProviderMetadata.map((providerInfo) => ({
|
||||
label: providerInfo.provider_display_name,
|
||||
value: providerInfo.provider,
|
||||
sublabel: decisionModelsSublabel(decisionCatalog, providerInfo.litellm_provider),
|
||||
icon: <ProviderLogo provider={providerInfo.provider} className="w-5 h-5" />,
|
||||
})),
|
||||
[sortedProviderMetadata],
|
||||
[sortedProviderMetadata, decisionCatalog],
|
||||
);
|
||||
|
||||
const pickedLitellmProvider = sortedProviderMetadata.find(
|
||||
(providerInfo) => providerInfo.provider === pickedProvider,
|
||||
)?.litellm_provider;
|
||||
|
||||
const credentialSelectOptions: SearchSelectOption[] = useMemo(() => credentialOptions(credentials), [credentials]);
|
||||
|
||||
const applyProviderSelection = (provider: string | null) => {
|
||||
|
|
@ -254,6 +264,7 @@ const AddModelForm: React.FC<AddModelFormProps> = ({
|
|||
providerModels={providerModels}
|
||||
getPlaceholder={getPlaceholder}
|
||||
/>
|
||||
<DecisionModelNote catalog={decisionCatalog} litellmProvider={pickedLitellmProvider} />
|
||||
|
||||
{/* Conditionally Render "Public Model Name" */}
|
||||
<ConditionalPublicModelName />
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
import { Info } from "lucide-react";
|
||||
import { useFormContext, useWatch } from "react-hook-form";
|
||||
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
|
||||
import {
|
||||
DECISIONS_DOCS_URL,
|
||||
SYSTEM_ONE_PLAYGROUND_ROUTE,
|
||||
isDecisionSelection,
|
||||
type DecisionCatalog,
|
||||
} from "@/lib/decisionModels";
|
||||
import { uiHref } from "@/utils/uiHref";
|
||||
import type { MountedFormValues } from "../common_components/MountedFormField";
|
||||
|
||||
interface DecisionModelNoteProps {
|
||||
catalog: DecisionCatalog;
|
||||
litellmProvider: string | undefined;
|
||||
}
|
||||
|
||||
const selectedModelNames = (value: unknown): string[] => {
|
||||
if (Array.isArray(value)) {
|
||||
return value.filter((model): model is string => typeof model === "string");
|
||||
}
|
||||
return typeof value === "string" && value !== "" ? [value] : [];
|
||||
};
|
||||
|
||||
export default function DecisionModelNote({ catalog, litellmProvider }: DecisionModelNoteProps) {
|
||||
const { control } = useFormContext<MountedFormValues>();
|
||||
const model = useWatch({ control, name: "model" });
|
||||
|
||||
if (!isDecisionSelection(catalog, litellmProvider, selectedModelNames(model))) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<Alert variant="info" role="note" aria-label="Decision model notice" className="mb-4">
|
||||
<Info />
|
||||
<AlertTitle>Decision model</AlertTitle>
|
||||
<AlertDescription>
|
||||
<p>
|
||||
For decision requests, call <code>/v1/decisions</code> (OpenAI format) or <code>/v1/systemone</code> (System
|
||||
One format).{" "}
|
||||
<a href={DECISIONS_DOCS_URL} target="_blank" rel="noopener noreferrer" className="underline">
|
||||
How to call decision models
|
||||
</a>
|
||||
</p>
|
||||
<p>
|
||||
After you add it,{" "}
|
||||
<a href={uiHref(SYSTEM_ONE_PLAYGROUND_ROUTE)} className="underline">
|
||||
test it in the System One playground
|
||||
</a>
|
||||
</p>
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
import { fireEvent, render, screen } from "@testing-library/react";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import DecisionModelsBanner from "./DecisionModelsBanner";
|
||||
|
||||
const STORAGE_KEY = "hideDecisionModelsBanner";
|
||||
|
||||
describe("DecisionModelsBanner", () => {
|
||||
beforeEach(() => {
|
||||
localStorage.removeItem(STORAGE_KEY);
|
||||
localStorage.removeItem("hideCostOptimizationFeedbackBanner");
|
||||
});
|
||||
|
||||
it("links directly to the System One playground", () => {
|
||||
render(<DecisionModelsBanner />);
|
||||
const link = screen.getByRole("link", { name: "Try decision models" });
|
||||
expect(link).toHaveAttribute("href", "/ui/playground?tab=system-one");
|
||||
});
|
||||
|
||||
it("links to the docs on how to call decision models", () => {
|
||||
render(<DecisionModelsBanner />);
|
||||
expect(screen.getByRole("link", { name: "How to call them" })).toHaveAttribute(
|
||||
"href",
|
||||
"https://docs.litellm.ai/docs/decisions",
|
||||
);
|
||||
});
|
||||
|
||||
it("offers Add a decision model only when the page can add models, and calls back on click", () => {
|
||||
const onAddModel = vi.fn();
|
||||
const { rerender } = render(<DecisionModelsBanner />);
|
||||
expect(screen.queryByRole("button", { name: "Add a decision model" })).not.toBeInTheDocument();
|
||||
|
||||
rerender(<DecisionModelsBanner onAddModel={onAddModel} />);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Add a decision model" }));
|
||||
|
||||
expect(onAddModel).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("points only users who can add models at Add Model", () => {
|
||||
const { rerender } = render(<DecisionModelsBanner />);
|
||||
expect(screen.getByRole("paragraph")).not.toHaveTextContent("Add Model");
|
||||
expect(screen.getByRole("paragraph")).toHaveTextContent("Call them at /v1/decisions or /v1/systemone");
|
||||
|
||||
rerender(<DecisionModelsBanner onAddModel={vi.fn()} />);
|
||||
|
||||
expect(screen.getByRole("paragraph")).toHaveTextContent(
|
||||
"Search decision in Add Model to find them. Call them at /v1/decisions",
|
||||
);
|
||||
});
|
||||
|
||||
it("shows the announcement even if the old feedback banner was dismissed", () => {
|
||||
localStorage.setItem("hideCostOptimizationFeedbackBanner", "true");
|
||||
render(<DecisionModelsBanner />);
|
||||
expect(screen.getByText("Decision models are now supported")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("hides itself and persists the dismissal when the dismiss button is clicked", () => {
|
||||
render(<DecisionModelsBanner />);
|
||||
expect(screen.getByText("Decision models are now supported")).toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByLabelText("Dismiss banner"));
|
||||
|
||||
expect(screen.queryByText("Decision models are now supported")).not.toBeInTheDocument();
|
||||
expect(localStorage.getItem(STORAGE_KEY)).toBe("true");
|
||||
});
|
||||
|
||||
it("stays dismissed on remount once persisted", () => {
|
||||
localStorage.setItem(STORAGE_KEY, "true");
|
||||
render(<DecisionModelsBanner />);
|
||||
expect(screen.queryByText("Decision models are now supported")).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
import React, { useState } from "react";
|
||||
import { ArrowRight, Sparkles, X } from "lucide-react";
|
||||
|
||||
import { DECISIONS_DOCS_URL, SYSTEM_ONE_PLAYGROUND_ROUTE } from "@/lib/decisionModels";
|
||||
import { uiHref } from "@/utils/uiHref";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
||||
const STORAGE_KEY = "hideDecisionModelsBanner";
|
||||
|
||||
interface DecisionModelsBannerProps {
|
||||
onAddModel?: () => void;
|
||||
}
|
||||
|
||||
const DecisionModelsBanner: React.FC<DecisionModelsBannerProps> = ({ onAddModel }) => {
|
||||
const [dismissed, setDismissed] = useState(() => {
|
||||
if (typeof window !== "undefined") {
|
||||
return localStorage.getItem(STORAGE_KEY) === "true";
|
||||
}
|
||||
return false;
|
||||
});
|
||||
|
||||
if (dismissed) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="mb-4 flex items-center gap-4 rounded-lg border bg-muted/40 px-4 py-3">
|
||||
<div className="flex size-10 shrink-0 items-center justify-center rounded-full border bg-background">
|
||||
<Sparkles className="size-4 text-muted-foreground" />
|
||||
</div>
|
||||
<div className="min-w-0 flex-1">
|
||||
<h4 className="m-0 text-sm font-semibold text-foreground">Decision models are now supported</h4>
|
||||
<p className="m-0 mt-0.5 text-xs text-muted-foreground">
|
||||
{onAddModel && (
|
||||
<>
|
||||
Search <code>decision</code> in Add Model to find them.{" "}
|
||||
</>
|
||||
)}
|
||||
Call them at <code>/v1/decisions</code> or <code>/v1/systemone</code>, or try them in the System One
|
||||
playground.{" "}
|
||||
<a href={DECISIONS_DOCS_URL} target="_blank" rel="noopener noreferrer" className="underline">
|
||||
How to call them
|
||||
</a>
|
||||
</p>
|
||||
</div>
|
||||
{onAddModel && (
|
||||
<Button type="button" variant="outline" className="shrink-0" onClick={onAddModel}>
|
||||
Add a decision model
|
||||
</Button>
|
||||
)}
|
||||
<Button
|
||||
className="shrink-0"
|
||||
nativeButton={false}
|
||||
role="link"
|
||||
render={<a href={uiHref(SYSTEM_ONE_PLAYGROUND_ROUTE)} />}
|
||||
>
|
||||
Try decision models
|
||||
<ArrowRight />
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
onClick={() => {
|
||||
setDismissed(true);
|
||||
localStorage.setItem(STORAGE_KEY, "true");
|
||||
}}
|
||||
className="shrink-0"
|
||||
aria-label="Dismiss banner"
|
||||
>
|
||||
<X />
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default DecisionModelsBanner;
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
import { fireEvent, render, screen } from "@testing-library/react";
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import CostOptimizationFeedbackBanner from "./cost_optimization_feedback_banner";
|
||||
|
||||
const STORAGE_KEY = "hideCostOptimizationFeedbackBanner";
|
||||
|
||||
describe("CostOptimizationFeedbackBanner", () => {
|
||||
beforeEach(() => {
|
||||
localStorage.removeItem(STORAGE_KEY);
|
||||
});
|
||||
|
||||
it("renders with a link to the feedback discussion", () => {
|
||||
render(<CostOptimizationFeedbackBanner />);
|
||||
const link = screen.getByText("Share Feedback").closest("a");
|
||||
expect(link).toHaveAttribute("href", "https://github.com/BerriAI/litellm/discussions/32172");
|
||||
});
|
||||
|
||||
it("hides itself and persists the dismissal when the dismiss button is clicked", () => {
|
||||
render(<CostOptimizationFeedbackBanner />);
|
||||
expect(screen.getByText("Help shape cost optimization")).toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByLabelText("Dismiss banner"));
|
||||
|
||||
expect(screen.queryByText("Help shape cost optimization")).not.toBeInTheDocument();
|
||||
expect(localStorage.getItem(STORAGE_KEY)).toBe("true");
|
||||
});
|
||||
|
||||
it("stays dismissed on remount once persisted", () => {
|
||||
localStorage.setItem(STORAGE_KEY, "true");
|
||||
render(<CostOptimizationFeedbackBanner />);
|
||||
expect(screen.queryByText("Help shape cost optimization")).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
import React, { useState } from "react";
|
||||
import { ExternalLink, MessageSquare, X } from "lucide-react";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
||||
const STORAGE_KEY = "hideCostOptimizationFeedbackBanner";
|
||||
const DISCUSSION_URL = "https://github.com/BerriAI/litellm/discussions/32172";
|
||||
|
||||
const CostOptimizationFeedbackBanner: React.FC = () => {
|
||||
const [dismissed, setDismissed] = useState(() => {
|
||||
if (typeof window !== "undefined") {
|
||||
return localStorage.getItem(STORAGE_KEY) === "true";
|
||||
}
|
||||
return false;
|
||||
});
|
||||
|
||||
if (dismissed) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="mb-4 flex items-center gap-4 rounded-lg border bg-muted/40 px-4 py-3">
|
||||
<div className="flex size-10 shrink-0 items-center justify-center rounded-full border bg-background">
|
||||
<MessageSquare className="size-4 text-muted-foreground" />
|
||||
</div>
|
||||
<div className="min-w-0 flex-1">
|
||||
<h4 className="m-0 text-sm font-semibold text-foreground">Help shape cost optimization</h4>
|
||||
<p className="m-0 mt-0.5 text-xs text-muted-foreground">
|
||||
We're collecting suggestions for cost optimization improvements across routing, budgets, and more. Let us
|
||||
know what you'd like to see.
|
||||
</p>
|
||||
</div>
|
||||
<Button
|
||||
className="shrink-0"
|
||||
nativeButton={false}
|
||||
render={<a href={DISCUSSION_URL} target="_blank" rel="noopener noreferrer" />}
|
||||
>
|
||||
Share Feedback
|
||||
<ExternalLink />
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
onClick={() => {
|
||||
setDismissed(true);
|
||||
localStorage.setItem(STORAGE_KEY, "true");
|
||||
}}
|
||||
className="shrink-0"
|
||||
aria-label="Dismiss banner"
|
||||
>
|
||||
<X />
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default CostOptimizationFeedbackBanner;
|
||||
108
ui/litellm-dashboard/src/lib/decisionModels.test.ts
Normal file
108
ui/litellm-dashboard/src/lib/decisionModels.test.ts
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
EMPTY_DECISION_CATALOG,
|
||||
buildDecisionCatalog,
|
||||
decisionModelsSublabel,
|
||||
isDecisionMode,
|
||||
isDecisionSelection,
|
||||
} from "./decisionModels";
|
||||
|
||||
const COST_MAP = {
|
||||
"typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" },
|
||||
"typesafe/jev-preview": { litellm_provider: "typesafe", mode: "evaluation" },
|
||||
"gpt-6-luna": {
|
||||
litellm_provider: "openai",
|
||||
mode: "chat",
|
||||
supported_endpoints: ["/v1/chat/completions", "/v1/decisions"],
|
||||
},
|
||||
"gpt-5.5": { litellm_provider: "openai", mode: "chat", supported_endpoints: ["/v1/chat/completions"] },
|
||||
"perplexity/pplx-decider-v1-27b": { litellm_provider: "perplexity", mode: "evaluation" },
|
||||
"perplexity/sonar-pro": { litellm_provider: "perplexity", mode: "chat" },
|
||||
"systemone-only": { litellm_provider: "somevendor", supported_endpoints: ["/v1/systemone"] },
|
||||
sample_spec: { max_tokens: "set to max_output_tokens if provider specifies it" },
|
||||
};
|
||||
|
||||
describe("isDecisionMode", () => {
|
||||
it("is true only for the evaluation mode", () => {
|
||||
expect(isDecisionMode("evaluation")).toBe(true);
|
||||
expect(isDecisionMode("chat")).toBe(false);
|
||||
expect(isDecisionMode(null)).toBe(false);
|
||||
expect(isDecisionMode(undefined)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildDecisionCatalog", () => {
|
||||
const catalog = buildDecisionCatalog(COST_MAP);
|
||||
|
||||
it("collects evaluation models and models that list a decision endpoint", () => {
|
||||
expect([...catalog.models].sort()).toEqual(
|
||||
[
|
||||
"gpt-6-luna",
|
||||
"perplexity/pplx-decider-v1-27b",
|
||||
"systemone-only",
|
||||
"typesafe/jev-latest",
|
||||
"typesafe/jev-preview",
|
||||
].sort(),
|
||||
);
|
||||
});
|
||||
|
||||
it("strips the provider prefix from the names shown per provider", () => {
|
||||
expect(catalog.providers.get("typesafe")?.names).toEqual(["jev-latest", "jev-preview"]);
|
||||
expect(catalog.providers.get("perplexity")?.names).toEqual(["pplx-decider-v1-27b"]);
|
||||
expect(catalog.providers.get("openai")?.names).toEqual(["gpt-6-luna"]);
|
||||
});
|
||||
|
||||
it("marks a provider decision-only when every one of its models is a decision model", () => {
|
||||
expect(catalog.providers.get("typesafe")?.decisionOnly).toBe(true);
|
||||
expect(catalog.providers.get("somevendor")?.decisionOnly).toBe(true);
|
||||
expect(catalog.providers.get("perplexity")?.decisionOnly).toBe(false);
|
||||
expect(catalog.providers.get("openai")?.decisionOnly).toBe(false);
|
||||
});
|
||||
|
||||
it("leaves providers without decision models out", () => {
|
||||
expect(catalog.providers.has("anthropic")).toBe(false);
|
||||
});
|
||||
|
||||
it("returns an empty catalog for a missing or malformed cost map", () => {
|
||||
expect(buildDecisionCatalog(undefined)).toEqual(EMPTY_DECISION_CATALOG);
|
||||
expect(buildDecisionCatalog("not a map")).toEqual(EMPTY_DECISION_CATALOG);
|
||||
expect(buildDecisionCatalog({ broken: { mode: "evaluation" } })).toEqual(EMPTY_DECISION_CATALOG);
|
||||
});
|
||||
});
|
||||
|
||||
describe("decisionModelsSublabel", () => {
|
||||
const catalog = buildDecisionCatalog(COST_MAP);
|
||||
|
||||
it("lists the provider's decision models", () => {
|
||||
expect(decisionModelsSublabel(catalog, "typesafe")).toBe("Decision models: jev-latest, jev-preview");
|
||||
});
|
||||
|
||||
it("is undefined for a provider with no decision models", () => {
|
||||
expect(decisionModelsSublabel(catalog, "anthropic")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("isDecisionSelection", () => {
|
||||
const catalog = buildDecisionCatalog(COST_MAP);
|
||||
|
||||
it("is true for a decision-only provider before any model is picked", () => {
|
||||
expect(isDecisionSelection(catalog, "typesafe", [])).toBe(true);
|
||||
});
|
||||
|
||||
it("waits for a decision model on a provider that also serves chat models", () => {
|
||||
expect(isDecisionSelection(catalog, "perplexity", [])).toBe(false);
|
||||
expect(isDecisionSelection(catalog, "perplexity", ["perplexity/sonar-pro"])).toBe(false);
|
||||
expect(isDecisionSelection(catalog, "perplexity", ["perplexity/sonar-pro", "perplexity/pplx-decider-v1-27b"])).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("is true for a chat-mode model that lists a decision endpoint", () => {
|
||||
expect(isDecisionSelection(catalog, "openai", ["gpt-6-luna"])).toBe(true);
|
||||
expect(isDecisionSelection(catalog, "openai", ["gpt-5.5"])).toBe(false);
|
||||
});
|
||||
|
||||
it("is false with no provider and no models", () => {
|
||||
expect(isDecisionSelection(catalog, undefined, [])).toBe(false);
|
||||
});
|
||||
});
|
||||
74
ui/litellm-dashboard/src/lib/decisionModels.ts
Normal file
74
ui/litellm-dashboard/src/lib/decisionModels.ts
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
import { z } from "zod";
|
||||
|
||||
export const DECISIONS_DOCS_URL = "https://docs.litellm.ai/docs/decisions";
|
||||
export const SYSTEM_ONE_PLAYGROUND_ROUTE = "playground?tab=system-one";
|
||||
|
||||
const DECISION_ENDPOINTS: ReadonlySet<string> = new Set(["/v1/decisions", "/v1/systemone"]);
|
||||
|
||||
const costMapEntrySchema = z.object({
|
||||
litellm_provider: z.string(),
|
||||
mode: z.string().nullish(),
|
||||
supported_endpoints: z.array(z.string()).nullish(),
|
||||
});
|
||||
|
||||
type CostMapEntry = z.infer<typeof costMapEntrySchema>;
|
||||
|
||||
interface ProviderDecisionModels {
|
||||
readonly names: readonly string[];
|
||||
readonly decisionOnly: boolean;
|
||||
}
|
||||
|
||||
export interface DecisionCatalog {
|
||||
readonly models: ReadonlySet<string>;
|
||||
readonly providers: ReadonlyMap<string, ProviderDecisionModels>;
|
||||
}
|
||||
|
||||
export const EMPTY_DECISION_CATALOG: DecisionCatalog = { models: new Set(), providers: new Map() };
|
||||
|
||||
export const isDecisionMode = (mode: string | null | undefined): boolean => mode === "evaluation";
|
||||
|
||||
const isDecisionEntry = (entry: CostMapEntry): boolean =>
|
||||
isDecisionMode(entry.mode) || (entry.supported_endpoints ?? []).some((endpoint) => DECISION_ENDPOINTS.has(endpoint));
|
||||
|
||||
const withoutProviderPrefix = (key: string, provider: string): string =>
|
||||
key.startsWith(`${provider}/`) ? key.slice(provider.length + 1) : key;
|
||||
|
||||
const parsedEntries = (costMap: unknown): ReadonlyArray<readonly [string, CostMapEntry]> => {
|
||||
const record = z.record(z.string(), z.unknown()).safeParse(costMap);
|
||||
if (!record.success) {
|
||||
return [];
|
||||
}
|
||||
return Object.entries(record.data).flatMap(([key, value]) => {
|
||||
const entry = costMapEntrySchema.safeParse(value);
|
||||
return entry.success ? [[key, entry.data] as const] : [];
|
||||
});
|
||||
};
|
||||
|
||||
export function buildDecisionCatalog(costMap: unknown): DecisionCatalog {
|
||||
const entries = parsedEntries(costMap);
|
||||
const decisionEntries = entries.filter(([, entry]) => isDecisionEntry(entry));
|
||||
const decisionProviders = new Set(decisionEntries.map(([, entry]) => entry.litellm_provider));
|
||||
const providers = new Map(
|
||||
[...decisionProviders].map((provider) => {
|
||||
const providerEntries = entries.filter(([, entry]) => entry.litellm_provider === provider);
|
||||
const names = decisionEntries
|
||||
.filter(([, entry]) => entry.litellm_provider === provider)
|
||||
.map(([key]) => withoutProviderPrefix(key, provider));
|
||||
return [provider, { names, decisionOnly: providerEntries.every(([, entry]) => isDecisionEntry(entry)) }] as const;
|
||||
}),
|
||||
);
|
||||
return { models: new Set(decisionEntries.map(([key]) => key)), providers };
|
||||
}
|
||||
|
||||
export const decisionModelsSublabel = (catalog: DecisionCatalog, litellmProvider: string): string | undefined => {
|
||||
const names = catalog.providers.get(litellmProvider)?.names ?? [];
|
||||
return names.length > 0 ? `Decision models: ${names.join(", ")}` : undefined;
|
||||
};
|
||||
|
||||
export const isDecisionSelection = (
|
||||
catalog: DecisionCatalog,
|
||||
litellmProvider: string | undefined,
|
||||
selectedModels: readonly string[],
|
||||
): boolean =>
|
||||
(litellmProvider !== undefined && catalog.providers.get(litellmProvider)?.decisionOnly === true) ||
|
||||
selectedModels.some((model) => catalog.models.has(model));
|
||||
Loading…
Add table
Reference in a new issue