diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py
index 9f3aa0130d1..dfe0788e50a 100644
--- a/litellm/llms/anthropic/pass_through/messages/transformation.py
+++ b/litellm/llms/anthropic/pass_through/messages/transformation.py
@@ -75,6 +75,12 @@ def messages_carry_output_config(messages: Sequence[object]) -> bool:
_messages_carry_output_config = messages_carry_output_config
+def _ensure_stop_sequence(response: AnthropicMessagesResponse) -> AnthropicMessagesResponse:
+ if "stop_sequence" not in response:
+ response["stop_sequence"] = None # rebind-ok: normalize the request-owned response in place
+ return response
+
+
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
_workload_identity_eligible: ClassVar[bool] = True
@@ -715,14 +721,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> AnthropicMessagesResponse:
- """
- No transformation is needed for Anthropic messages, since we want the response in the Anthropic /v1/messages API spec
- """
try:
raw_response_json: Final = raw_response.json()
except Exception:
raise AnthropicError(message=raw_response.text, status_code=raw_response.status_code)
- return AnthropicMessagesResponse(**raw_response_json)
+ response: Final = AnthropicMessagesResponse(**raw_response_json)
+ return _ensure_stop_sequence(response)
def get_async_streaming_response_iterator(
self,
diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json
index 4572e59944b..e5e0027a808 100644
--- a/litellm/proxy/_lazy_openapi_snapshot.json
+++ b/litellm/proxy/_lazy_openapi_snapshot.json
@@ -11407,7 +11407,7 @@
},
"/systemone": {
"post": {
- "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
+ "description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"operationId": "systemone_systemone_post",
"requestBody": {
"content": {
@@ -11813,7 +11813,6 @@
]
},
"description": "Named noul, choice and score questions. Each key becomes a key in the answers",
- "maxProperties": 128,
"minProperties": 1,
"propertyNames": {
"minLength": 1
@@ -12814,7 +12813,7 @@
},
"/v1/systemone": {
"post": {
- "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
+ "description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"operationId": "systemone_v1_systemone_post",
"requestBody": {
"content": {
@@ -13220,7 +13219,6 @@
]
},
"description": "Named noul, choice and score questions. Each key becomes a key in the answers",
- "maxProperties": 128,
"minProperties": 1,
"propertyNames": {
"minLength": 1
diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py
index 366842d23a6..a8684dd9757 100644
--- a/litellm/proxy/decisions_endpoints/endpoints.py
+++ b/litellm/proxy/decisions_endpoints/endpoints.py
@@ -228,7 +228,7 @@ async def systemone(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
):
"""
- Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.
+ Judge a `state` against one or more named `questions` and get one calibrated answer per question name.
Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with
`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,
diff --git a/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py b/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py
index 7411f1b4947..ef2a5f88e4e 100644
--- a/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py
+++ b/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py
@@ -1,5 +1,6 @@
import asyncio
import re
+import xml.etree.ElementTree as ET
from collections import Counter
from collections.abc import Awaitable, Mapping
from dataclasses import dataclass
@@ -17,7 +18,8 @@ from litellm.types.llms.base import LiteLLMBaseModel
router: Final = APIRouter()
-LATEST_RELEASE_URL: Final = "https://api.github.com/repos/BerriAI/litellm/releases/latest"
+CHANGELOG_URL: Final = "https://docs.litellm.ai/release_notes/"
+CHANGELOG_FEED_URL: Final = f"{CHANGELOG_URL}rss.xml"
RELEASE_BY_TAG_URL: Final = "https://api.github.com/repos/BerriAI/litellm/releases/tags/{tag}"
LATEST_RELEASE_FETCH_TIMEOUT_SECONDS: Final = 5
LATEST_RELEASE_CACHE_TTL_SECONDS: Final = 60 * 60
@@ -26,7 +28,7 @@ LATEST_RELEASE_CACHE_KEY: Final = "latest_release_info"
_RELEASE_BULLET_PATTERN: Final = re.compile(r"^\*\s+(?:([A-Za-z]+)(?:\([^)]*\))?!?:\s)?\S")
_NEW_CONTRIBUTOR_PATTERN: Final = re.compile(r"^\*\s+@\S+ made their first contribution\b")
-_STABLE_TAG_PATTERN: Final = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)$")
+_CHANGELOG_PAGE_PATTERN: Final = re.compile(rf"{re.escape(CHANGELOG_URL)}v(\d+)\.(\d+)\.(\d+)/")
_Bucket: TypeAlias = Literal["new_features", "bug_fixes", "other_updates"]
_PREFIX_BUCKETS: Final[Mapping[str, _Bucket]] = MappingProxyType({"feat": "new_features", "fix": "bug_fixes"})
@@ -45,6 +47,16 @@ class LatestReleaseUnavailable:
reason: str
+@dataclass(frozen=True, slots=True)
+class ChangelogEntry:
+ version: tuple[int, int, int]
+ url: str
+
+ @property
+ def tag(self) -> str:
+ return "v" + ".".join(str(part) for part in self.version)
+
+
class _GitHubRelease(LiteLLMBaseModel):
tag_name: str
html_url: str
@@ -91,9 +103,18 @@ def count_release_bullets(body: str) -> Mapping[_Bucket, int]:
return MappingProxyType(Counter(bucket for line in body.splitlines() if (bucket := _bucket_for(line)) is not None))
-def _stable_version(tag_name: str) -> tuple[int, int, int] | None:
- match: Final = _STABLE_TAG_PATTERN.match(tag_name)
- return None if match is None else (int(match.group(1)), int(match.group(2)), int(match.group(3)))
+def _changelog_entry(link: str) -> ChangelogEntry | None:
+ match: Final = _CHANGELOG_PAGE_PATTERN.match(link)
+ if match is None:
+ return None
+ return ChangelogEntry(version=(int(match.group(1)), int(match.group(2)), int(match.group(3))), url=link)
+
+
+def latest_changelog_entry(feed: bytes) -> ChangelogEntry | None:
+ """The highest stable release with a page in the changelog feed. Release candidate pages don't count."""
+ links: Final = (link.text or "" for link in ET.fromstring(feed).iterfind("channel/item/link"))
+ entries: Final = (entry for link in links if (entry := _changelog_entry(link)) is not None)
+ return max(entries, key=lambda entry: entry.version, default=None)
def earlier_patch_tags(version: tuple[int, int, int]) -> tuple[str, ...]:
@@ -102,7 +123,9 @@ def earlier_patch_tags(version: tuple[int, int, int]) -> tuple[str, ...]:
return tuple(f"v{major}.{minor}.{earlier}" for earlier in range(patch))
-def summarize_release_line(latest: _GitHubRelease, earlier: tuple[_GitHubRelease, ...]) -> LatestReleaseInfo:
+def summarize_release_line(
+ entry: ChangelogEntry, latest: _GitHubRelease, earlier: tuple[_GitHubRelease, ...]
+) -> LatestReleaseInfo:
"""A patch release's notes only list its backports, so count the whole minor line it belongs to."""
counts: Final = sum(
(Counter[_Bucket](count_release_bullets(release.body or "")) for release in (latest, *earlier)),
@@ -113,7 +136,7 @@ def summarize_release_line(latest: _GitHubRelease, earlier: tuple[_GitHubRelease
new_features=counts.get("new_features", 0),
bug_fixes=counts.get("bug_fixes", 0),
other_updates=counts.get("other_updates", 0),
- release_url=latest.html_url,
+ release_url=entry.url,
)
@@ -137,25 +160,38 @@ async def _fetch_earlier_release(client: _AsyncGetClient, tag: str) -> _GitHubRe
return release
+async def _fetch_changelog_entry(client: _AsyncGetClient) -> ChangelogEntry | LatestReleaseUnavailable:
+ response: Final = await client.get(CHANGELOG_FEED_URL, timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
+ if response.status_code != 200:
+ return LatestReleaseUnavailable(reason=f"Changelog feed responded with status {response.status_code}")
+ try:
+ entry: Final = latest_changelog_entry(response.content)
+ except ET.ParseError as e:
+ return LatestReleaseUnavailable(reason=f"Changelog feed was not valid XML: {e}")
+ return LatestReleaseUnavailable(reason="Changelog feed lists no stable release") if entry is None else entry
+
+
async def fetch_latest_release(client: _AsyncGetClient) -> LatestReleaseInfo | LatestReleaseUnavailable:
try:
+ entry: Final = await _fetch_changelog_entry(client)
+ if isinstance(entry, LatestReleaseUnavailable):
+ return entry
latest: Final = parse_github_release(
- await client.get(LATEST_RELEASE_URL, timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
+ await client.get(RELEASE_BY_TAG_URL.format(tag=entry.tag), timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
)
if isinstance(latest, LatestReleaseUnavailable):
return latest
- version: Final = _stable_version(latest.tag_name)
- if version is None:
- return summarize_release_line(latest, ())
earlier: Final = await asyncio.gather(
- *(_fetch_earlier_release(client, tag) for tag in earlier_patch_tags(version))
+ *(_fetch_earlier_release(client, tag) for tag in earlier_patch_tags(entry.version))
)
except httpx.HTTPError as e:
return LatestReleaseUnavailable(reason=f"{type(e).__name__}: {e}")
unavailable: Final = next((release for release in earlier if isinstance(release, LatestReleaseUnavailable)), None)
if unavailable is not None:
return unavailable
- return summarize_release_line(latest, tuple(release for release in earlier if isinstance(release, _GitHubRelease)))
+ return summarize_release_line(
+ entry, latest, tuple(release for release in earlier if isinstance(release, _GitHubRelease))
+ )
async def get_latest_release_info(
@@ -190,9 +226,9 @@ async def latest_release_info(
fetch_lock: Annotated[asyncio.Lock, Depends(_default_fetch_lock)],
) -> LatestReleaseInfo | None:
"""
- Latest stable LiteLLM GitHub release with the PR count of its whole minor line (e.g. v1.105.0 + v1.105.1)
- split into new features, bug fixes and other updates.
- Returns null when GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
+ Newest stable release on the LiteLLM changelog, linked to its changelog page, with the PR count of its whole
+ minor line (e.g. v1.105.0 + v1.105.1) from GitHub split into new features, bug fixes and other updates.
+ Returns null when the changelog or GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
"""
result: Final = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock)
if isinstance(result, LatestReleaseUnavailable):
diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py
index a1d48f730c0..5ad6e45e6d2 100644
--- a/litellm/types/decisions.py
+++ b/litellm/types/decisions.py
@@ -73,7 +73,7 @@ DecisionQuestion: TypeAlias = Annotated[
DecisionQuestionMap: TypeAlias = Annotated[
Mapping[Annotated[str, Field(min_length=1)], DecisionQuestion],
- Field(min_length=1, max_length=MAX_DECISION_QUESTIONS),
+ Field(min_length=1),
]
diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
index 1f02b3aaa5a..f4920505722 100644
--- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
+++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
@@ -2033,6 +2033,7 @@ async def test_anthropic_messages_with_extra_headers():
],
"model": "claude-haiku-4-5-20251001",
"stop_reason": "end_turn",
+ "stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 20},
}
@@ -2096,6 +2097,7 @@ async def test_anthropic_messages_with_thinking():
],
"model": "claude-haiku-4-5-20251001",
"stop_reason": "end_turn",
+ "stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 20},
}
@@ -2210,6 +2212,7 @@ async def test_anthropic_messages_bedrock_dynamic_region():
"content": [{"type": "text", "text": "This is a mock response"}],
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"stop_reason": "end_turn",
+ "stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 20},
}
diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py
index 7fbec3ba12e..457d5d5e1c8 100644
--- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py
+++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py
@@ -1,17 +1,23 @@
from collections.abc import Mapping
+from datetime import datetime, timezone
from pathlib import Path
from typing import Final
from unittest.mock import patch
+import httpx
import pytest
+from pydantic import JsonValue
import litellm
+from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.anthropic.pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from litellm.llms.anthropic.wif import get_anthropic_wif_token
from litellm.llms.minimax.messages.transformation import MinimaxMessagesConfig
+from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig
from litellm.llms.tencent.messages.transformation import TencentAnthropicMessagesConfig
+from litellm.types.llms.anthropic import AnthropicFinishReason
from litellm.types.router import GenericLiteLLMParams
from tests.unit.llms.anthropic.test_anthropic_wif import (
ScriptedPoster,
@@ -27,6 +33,52 @@ _WIF_PARAMS: Final[dict] = {
}
+@pytest.mark.parametrize("config", (AnthropicMessagesConfig(), OpenAILikeAnthropicMessagesConfig()), ids=type)
+@pytest.mark.parametrize(
+ ("stop_reason", "stop_fields", "expected_stop_sequence"),
+ (
+ pytest.param("end_turn", {}, None, id="end_turn_missing"),
+ pytest.param("max_tokens", {}, None, id="max_tokens_missing"),
+ pytest.param("tool_use", {}, None, id="tool_use_missing"),
+ pytest.param("end_turn", {"stop_sequence": None}, None, id="explicit_null"),
+ pytest.param("stop_sequence", {"stop_sequence": ", 7"}, ", 7", id="matched_stop"),
+ ),
+)
+def test_native_response_defaults_stop_sequence_without_changing_other_fields(
+ config: AnthropicMessagesConfig,
+ stop_reason: AnthropicFinishReason,
+ stop_fields: Mapping[str, str | None],
+ expected_stop_sequence: str | None,
+) -> None:
+ upstream: Final[dict[str, JsonValue]] = {
+ "id": "msg_test",
+ "type": "message",
+ "role": "assistant",
+ "model": "test-model",
+ "content": [{"type": "text", "text": "ok"}],
+ "stop_reason": stop_reason,
+ "usage": {"input_tokens": 19, "output_tokens": 42},
+ **stop_fields,
+ }
+ logging_obj: Final = Logging(
+ model="test-model",
+ messages=[],
+ stream=False,
+ call_type="anthropic_messages",
+ start_time=datetime(2026, 1, 1, tzinfo=timezone.utc),
+ litellm_call_id="test-call",
+ function_id="test-function",
+ )
+ response: Final = config.transform_anthropic_messages_response(
+ model="test-model", raw_response=httpx.Response(200, json=upstream), logging_obj=logging_obj
+ )
+
+ assert response == {
+ **upstream,
+ "stop_sequence": expected_stop_sequence,
+ }
+
+
def test_workload_identity_allowed_for_anthropic() -> None:
assert AnthropicMessagesConfig()._allows_workload_identity is True
diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py b/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py
index fd9aa75be74..4ed9054f3d3 100644
--- a/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py
+++ b/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py
@@ -26,7 +26,9 @@ from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
is_provider_error_chunk,
parse_anthropic_error_event,
)
+from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig
from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator
+from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig
from litellm.types.utils import Choices, Message, ModelResponse
@@ -1733,6 +1735,49 @@ async def _drained(stream: AsyncIterator[bytes]) -> list[bytes]:
return [chunk async for chunk in stream]
+@pytest.mark.asyncio
+@pytest.mark.parametrize("config", (AnthropicMessagesConfig(), OpenAILikeAnthropicMessagesConfig()), ids=type)
+@pytest.mark.parametrize("stop_sequence", (None, ", 7"), ids=("null", "matched_stop"))
+async def test_native_stream_preserves_nullable_stop_fields_across_chunk_boundaries(
+ config: AnthropicMessagesConfig, stop_sequence: str | None
+) -> None:
+ logging_obj: Final = _make_logging_obj("native_stream_stop_fields")
+ events: Final = (
+ {
+ "type": "message_start",
+ "message": {
+ "id": "msg_test",
+ "type": "message",
+ "role": "assistant",
+ "model": logging_obj.model,
+ "content": [],
+ "stop_reason": None,
+ "stop_sequence": None,
+ "usage": {"input_tokens": 19, "output_tokens": 0},
+ },
+ },
+ {
+ "type": "message_delta",
+ "delta": {
+ "stop_reason": "end_turn" if stop_sequence is None else "stop_sequence",
+ "stop_sequence": stop_sequence,
+ },
+ "usage": {"output_tokens": 0},
+ },
+ {"type": "message_stop"},
+ )
+ upstream: Final = b"".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events)
+ response: Final = httpx.Response(200, content=_chunked(upstream, 7))
+ stream: Final = config.get_async_streaming_response_iterator(
+ model=logging_obj.model,
+ httpx_response=response,
+ request_body={"model": logging_obj.model},
+ litellm_logging_obj=logging_obj,
+ )
+
+ assert b"".join(await _drained(stream)) == upstream
+
+
def _data_events(frames: Sequence[bytes]) -> list[Mapping[str, object]]:
return [
json.loads(line[len(b"data:") :]) for frame in frames for line in frame.splitlines() if line.startswith(b"data:")
diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py
index 021eb289ad7..4384507e841 100644
--- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py
+++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py
@@ -126,6 +126,51 @@ def test_proxy_decisions_route_returns_answers_and_cost(
assert upstream.calls[0].request.headers["authorization"] == "Bearer test-key"
+@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/systemone"))
+@pytest.mark.parametrize("question_count", (128, 129, 154))
+def test_proxy_systemone_preserves_large_question_batches(
+ client: TestClient,
+ respx_mock: respx.MockRouter,
+ endpoint: str,
+ question_count: int,
+) -> None:
+ questions: Final = {
+ f"q{index}": {"type": "noul", "instructions": f"Question {index}?"} for index in range(question_count)
+ }
+ answers: Final = {f"q{index}": {"type": "noul", "noul": index / question_count} for index in range(question_count)}
+ upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(
+ json={**_RESPONSE, "answers": answers}
+ )
+
+ response: Final = client.post(endpoint, json={**_REQUEST, "questions": questions})
+
+ assert response.status_code == 200, response.text
+ assert json.loads(upstream.calls[0].request.content) == {
+ **_REQUEST,
+ "model": "pplx-decider-v1-27b",
+ "questions": questions,
+ }
+ assert response.json()["answers"] == answers
+
+
+@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/systemone"))
+@pytest.mark.parametrize(
+ "questions",
+ ({}, {"": {"type": "noul", "instructions": "Is this a defect?"}}, {"is_defect": {"type": "noul"}}),
+ ids=("empty", "empty_name", "missing_instructions_or_criteria"),
+)
+def test_proxy_systemone_rejects_invalid_questions_before_http(
+ client: TestClient,
+ respx_mock: respx.MockRouter,
+ endpoint: str,
+ questions: Mapping[str, object],
+) -> None:
+ response: Final = client.post(endpoint, json={**_REQUEST, "questions": questions})
+
+ assert response.status_code == 400, response.text
+ assert len(respx_mock.calls) == 0
+
+
def test_proxy_decisions_dispatches_typesafe_deployment(
client: TestClient,
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py b/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py
index 25f1301dd9d..47f81fb55d1 100644
--- a/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py
+++ b/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py
@@ -12,10 +12,11 @@ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.proxy_server import app
from litellm.proxy.ui_crud_endpoints.latest_release_endpoints import (
+ CHANGELOG_FEED_URL,
+ CHANGELOG_URL,
LATEST_RELEASE_CACHE_KEY,
LATEST_RELEASE_CACHE_TTL_SECONDS,
LATEST_RELEASE_UNAVAILABLE_CACHE_TTL_SECONDS,
- LATEST_RELEASE_URL,
RELEASE_BY_TAG_URL,
LatestReleaseInfo,
LatestReleaseUnavailable,
@@ -49,37 +50,68 @@ SAMPLE_RELEASE: Final = {
"html_url": "https://github.com/BerriAI/litellm/releases/tag/v1.102.0",
"body": SAMPLE_BODY,
}
+
+
+def _changelog_page(folder: str) -> str:
+ return f"{CHANGELOG_URL}{folder}/{folder.removeprefix('v').replace('.', '-')}"
+
+
+def _feed(*folders: str) -> httpx.Response:
+ items: Final = "".join(
+ f"- {folder}{_changelog_page(folder)}
" for folder in folders
+ )
+ return httpx.Response(200, content=f'{items}'.encode())
+
+
EXPECTED_INFO: Final = {
"version": "1.102.0",
"new_features": 2,
"bug_fixes": 2,
"other_updates": 4,
- "release_url": SAMPLE_RELEASE["html_url"],
+ "release_url": _changelog_page("v1.102.0"),
}
-class _RecordingClient:
- def __init__(self, outcomes: list[httpx.Response | Exception]) -> None:
- self._outcomes = outcomes
- self.calls: list[tuple[str, float | None]] = []
-
- async def get(self, url: str, *, timeout: float | None = None) -> httpx.Response:
- self.calls.append((url, timeout))
- outcome = self._outcomes[min(len(self.calls) - 1, len(self._outcomes) - 1)]
- if isinstance(outcome, Exception):
- raise outcome
- return outcome
-
-
def _github_response(status: int = 200, payload: object = SAMPLE_RELEASE) -> httpx.Response:
return httpx.Response(status, content=json.dumps(payload).encode())
+def _tag_url(tag: str) -> str:
+ return RELEASE_BY_TAG_URL.format(tag=tag)
+
+
+class _ByUrl:
+ """Answers each URL from a fixed map; unknown URLs are 404, like a tag with no release."""
+
+ def __init__(self, routes: dict[str, httpx.Response | Exception]) -> None:
+ self._routes = routes
+ self.calls: list[tuple[str, float | None]] = []
+
+ async def get(self, url: str, *, timeout: float | None = None) -> httpx.Response:
+ self.calls.append((url, timeout))
+ outcome = self._routes.get(url, httpx.Response(404, content=b"{}"))
+ if isinstance(outcome, Exception):
+ raise outcome
+ return outcome
+
+ @property
+ def urls(self) -> list[str]:
+ return [url for url, _ in self.calls]
+
+
+def _minor_release_routes(**overrides: httpx.Response | Exception) -> dict[str, httpx.Response | Exception]:
+ return {CHANGELOG_FEED_URL: _feed("v1.102.0"), _tag_url("v1.102.0"): _github_response(), **overrides}
+
+
+def _minor_release(**overrides: httpx.Response | Exception) -> _ByUrl:
+ return _ByUrl(_minor_release_routes(**overrides))
+
+
def _fresh_cache() -> InMemoryCache:
return InMemoryCache(max_size_in_memory=1, default_ttl=LATEST_RELEASE_CACHE_TTL_SECONDS)
-def _override_dependencies(client: _RecordingClient, cache: InMemoryCache, role: LitellmUserRoles) -> None:
+def _override_dependencies(client: _ByUrl, cache: InMemoryCache, role: LitellmUserRoles) -> None:
async def auth() -> UserAPIKeyAuth:
return UserAPIKeyAuth(api_key="sk-test", user_id="test-user", user_role=role)
@@ -127,35 +159,68 @@ class TestCountReleaseBullets:
class TestGetLatestReleaseInfo:
@pytest.mark.asyncio
- async def test_fetches_and_parses_github_release(self):
- client = _RecordingClient([_github_response()])
+ async def test_announces_the_changelog_release_and_links_its_changelog_page(self):
+ client = _minor_release()
result = await get_latest_release_info(client=client, cache=_fresh_cache(), fetch_lock=asyncio.Lock())
assert isinstance(result, LatestReleaseInfo)
assert result.model_dump() == EXPECTED_INFO
- assert client.calls == [(LATEST_RELEASE_URL, 5)]
+ assert client.calls == [(CHANGELOG_FEED_URL, 5), (_tag_url("v1.102.0"), 5)]
+
+ @pytest.mark.asyncio
+ async def test_github_release_missing_from_the_changelog_is_not_announced(self):
+ automated: Final = _github_response(payload=_release("v1.103.0", "* feat(proxy): automated release\n"))
+ result = await _info(_minor_release(**{_tag_url("v1.103.0"): automated}))
+ assert isinstance(result, LatestReleaseInfo)
+ assert result.model_dump() == EXPECTED_INFO
+
+ @pytest.mark.asyncio
+ async def test_highest_stable_version_wins_regardless_of_feed_order(self):
+ client = _minor_release(**{CHANGELOG_FEED_URL: _feed("v1.101.6", "v1.103.0rc1", "v1.102.0", "v1.100.9")})
+ result = await _info(client)
+ assert isinstance(result, LatestReleaseInfo)
+ assert result.model_dump() == EXPECTED_INFO
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "feed",
+ [
+ httpx.Response(503, content=_feed("v1.102.0").content),
+ httpx.Response(200, content=b"not a feed"),
+ _feed("v1.103.0rc1", "v1.103.0-dev.1"),
+ httpx.Response(
+ 200,
+ content=f"- https://example.com/?{_changelog_page('v9.0.0')}"
+ "
- https://example.com/release_notes/v9.0.0/x
".encode(),
+ ),
+ ],
+ ids=["server_error", "not_xml", "only_prereleases", "foreign_links"],
+ )
+ async def test_bad_changelog_feeds_are_unavailable(self, feed: httpx.Response):
+ client = _minor_release(**{CHANGELOG_FEED_URL: feed})
+ result = await _info(client)
+ assert isinstance(result, LatestReleaseUnavailable)
+ assert client.urls == [CHANGELOG_FEED_URL]
@pytest.mark.asyncio
async def test_second_call_within_ttl_does_not_refetch(self):
- client = _RecordingClient([_github_response()])
+ client = _minor_release()
cache = _fresh_cache()
fetch_lock = asyncio.Lock()
first = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock)
second = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock)
assert first == second
- assert len(client.calls) == 1
+ assert len(client.calls) == 2
@pytest.mark.asyncio
async def test_success_is_cached_for_the_full_ttl(self):
cache = _fresh_cache()
- await get_latest_release_info(
- client=_RecordingClient([_github_response()]), cache=cache, fetch_lock=asyncio.Lock()
- )
+ await get_latest_release_info(client=_minor_release(), cache=cache, fetch_lock=asyncio.Lock())
remaining = await cache.async_get_ttl(LATEST_RELEASE_CACHE_KEY) - time.time()
assert LATEST_RELEASE_CACHE_TTL_SECONDS - 5 < remaining <= LATEST_RELEASE_CACHE_TTL_SECONDS
@pytest.mark.asyncio
async def test_failure_is_cached_briefly_so_github_is_not_hammered(self):
- client = _RecordingClient([httpx.ConnectError("boom")])
+ client = _ByUrl({CHANGELOG_FEED_URL: httpx.ConnectError("boom")})
cache = _fresh_cache()
fetch_lock = asyncio.Lock()
first = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock)
@@ -172,30 +237,29 @@ class TestGetLatestReleaseInfo:
@pytest.mark.parametrize(
"response",
[
+ _github_response(status=404, payload={"message": "Not Found"}),
_github_response(status=403, payload={"message": "rate limited"}),
_github_response(status=500, payload={}),
_github_response(payload={"tag_name": "v1.0.0"}),
httpx.Response(200, content=b"not json"),
+ httpx.ConnectError("boom"),
],
- ids=["rate_limited", "server_error", "missing_fields", "not_json"],
+ ids=["no_release_for_tag", "rate_limited", "server_error", "missing_fields", "not_json", "unreachable"],
)
- async def test_bad_github_responses_are_unavailable(self, response: httpx.Response):
- result = await get_latest_release_info(
- client=_RecordingClient([response]), cache=_fresh_cache(), fetch_lock=asyncio.Lock()
- )
+ async def test_bad_github_responses_are_unavailable(self, response: httpx.Response | Exception):
+ result = await _info(_minor_release(**{_tag_url("v1.102.0"): response}))
assert isinstance(result, LatestReleaseUnavailable)
@pytest.mark.asyncio
async def test_concurrent_misses_share_one_fetch(self):
event = asyncio.Event()
- class _BlockingClient(_RecordingClient):
+ class _BlockingClient(_ByUrl):
async def get(self, url: str, *, timeout: float | None = None) -> httpx.Response:
- self.calls.append((url, timeout))
await event.wait()
- return _github_response()
+ return await super().get(url, timeout=timeout)
- client = _BlockingClient([])
+ client = _BlockingClient(_minor_release_routes())
cache = _fresh_cache()
fetch_lock = asyncio.Lock()
tasks = [
@@ -208,19 +272,19 @@ class TestGetLatestReleaseInfo:
results = await asyncio.gather(*tasks)
expected: Final = LatestReleaseInfo.model_validate(EXPECTED_INFO)
assert results == [expected] * 5
- assert len(client.calls) == 1
+ assert len(client.calls) == 2
@pytest.mark.asyncio
async def test_failure_under_lock_is_also_coalesced(self):
event = asyncio.Event()
- class _FailingBlockingClient(_RecordingClient):
+ class _FailingBlockingClient(_ByUrl):
async def get(self, url: str, *, timeout: float | None = None) -> httpx.Response:
self.calls.append((url, timeout))
await event.wait()
raise httpx.ConnectError("boom")
- client = _FailingBlockingClient([])
+ client = _FailingBlockingClient({})
cache = _fresh_cache()
fetch_lock = asyncio.Lock()
tasks = [
@@ -245,40 +309,23 @@ def _release(tag: str, body: str | None, *, prerelease: bool = False) -> dict[st
}
-def _tag_url(tag: str) -> str:
- return RELEASE_BY_TAG_URL.format(tag=tag)
-
-
-class _GitHubByUrl:
- """Answers each GitHub URL from a fixed map; unknown URLs are 404, like a tag with no release."""
-
- def __init__(self, routes: dict[str, httpx.Response | Exception]) -> None:
- self._routes = routes
- self.urls: list[str] = []
-
- async def get(self, url: str, *, timeout: float | None = None) -> httpx.Response:
- self.urls.append(url)
- outcome = self._routes.get(url, httpx.Response(404, content=b"{}"))
- if isinstance(outcome, Exception):
- raise outcome
- return outcome
-
-
PATCH_RELEASE: Final = _release("v1.102.2", "* fix(proxy): second backport by @k in https://x/pull/10\n")
FIRST_PATCH: Final = _release("v1.102.1", "* fix(proxy): first backport by @k in https://x/pull/9\n")
-def _minor_line(**overrides: httpx.Response | Exception) -> dict[str, httpx.Response | Exception]:
- routes: dict[str, httpx.Response | Exception] = {
- LATEST_RELEASE_URL: _github_response(payload=PATCH_RELEASE),
- _tag_url("v1.102.0"): _github_response(payload=SAMPLE_RELEASE),
- _tag_url("v1.102.1"): _github_response(payload=FIRST_PATCH),
- }
- routes.update({_tag_url(tag): outcome for tag, outcome in overrides.items()})
- return routes
+def _minor_line(**overrides: httpx.Response | Exception) -> _ByUrl:
+ return _ByUrl(
+ {
+ CHANGELOG_FEED_URL: _feed("v1.102.2", "v1.101.6"),
+ _tag_url("v1.102.2"): _github_response(payload=PATCH_RELEASE),
+ _tag_url("v1.102.0"): _github_response(payload=SAMPLE_RELEASE),
+ _tag_url("v1.102.1"): _github_response(payload=FIRST_PATCH),
+ **{_tag_url(tag): outcome for tag, outcome in overrides.items()},
+ }
+ )
-async def _info(client: _GitHubByUrl) -> LatestReleaseInfo | LatestReleaseUnavailable:
+async def _info(client: _ByUrl) -> LatestReleaseInfo | LatestReleaseUnavailable:
return await get_latest_release_info(client=client, cache=_fresh_cache(), fetch_lock=asyncio.Lock())
@@ -289,7 +336,7 @@ class TestPatchReleaseAggregatesMinorLine:
@pytest.mark.asyncio
async def test_patch_release_counts_include_every_earlier_patch_on_its_line(self):
- client = _GitHubByUrl(_minor_line())
+ client = _minor_line()
result = await _info(client)
assert isinstance(result, LatestReleaseInfo)
assert result.model_dump() == {
@@ -297,23 +344,11 @@ class TestPatchReleaseAggregatesMinorLine:
"new_features": 2,
"bug_fixes": 4,
"other_updates": 4,
- "release_url": PATCH_RELEASE["html_url"],
+ "release_url": _changelog_page("v1.102.2"),
}
- assert sorted(client.urls) == sorted([LATEST_RELEASE_URL, _tag_url("v1.102.0"), _tag_url("v1.102.1")])
-
- @pytest.mark.asyncio
- async def test_minor_release_makes_a_single_request(self):
- client = _RecordingClient([_github_response()])
- await get_latest_release_info(client=client, cache=_fresh_cache(), fetch_lock=asyncio.Lock())
- assert client.calls == [(LATEST_RELEASE_URL, 5)]
-
- @pytest.mark.asyncio
- async def test_non_semver_latest_tag_counts_only_its_own_notes(self):
- client = _GitHubByUrl({LATEST_RELEASE_URL: _github_response(payload=_release("v1.102.1-hotfix", "* fix: x\n"))})
- result = await _info(client)
- assert isinstance(result, LatestReleaseInfo)
- assert (result.new_features, result.bug_fixes, result.other_updates) == (0, 1, 0)
- assert client.urls == [LATEST_RELEASE_URL]
+ assert sorted(client.urls) == sorted(
+ [CHANGELOG_FEED_URL, _tag_url("v1.102.2"), _tag_url("v1.102.0"), _tag_url("v1.102.1")]
+ )
@pytest.mark.asyncio
@pytest.mark.parametrize(
@@ -326,15 +361,13 @@ class TestPatchReleaseAggregatesMinorLine:
ids=["never_published", "prerelease", "null_body"],
)
async def test_missing_prerelease_or_empty_patch_adds_nothing(self, skipped: httpx.Response):
- result = await _info(_GitHubByUrl(_minor_line(**{"v1.102.1": skipped})))
+ result = await _info(_minor_line(**{"v1.102.1": skipped}))
assert isinstance(result, LatestReleaseInfo)
assert (result.new_features, result.bug_fixes, result.other_updates) == (2, 3, 4)
@pytest.mark.asyncio
async def test_latest_release_with_null_body_still_counts_its_line(self):
- routes = _minor_line()
- routes[LATEST_RELEASE_URL] = _github_response(payload=_release("v1.102.2", None))
- result = await _info(_GitHubByUrl(routes))
+ result = await _info(_minor_line(**{"v1.102.2": _github_response(payload=_release("v1.102.2", None))}))
assert isinstance(result, LatestReleaseInfo)
assert (result.new_features, result.bug_fixes, result.other_updates) == (2, 3, 4)
@@ -349,31 +382,31 @@ class TestPatchReleaseAggregatesMinorLine:
ids=["unreachable", "rate_limited", "not_json"],
)
async def test_earlier_patch_failure_is_unavailable(self, outcome: httpx.Response | Exception):
- result = await _info(_GitHubByUrl(_minor_line(**{"v1.102.0": outcome})))
+ result = await _info(_minor_line(**{"v1.102.0": outcome}))
assert isinstance(result, LatestReleaseUnavailable)
class TestLatestReleaseInfoEndpoint:
def test_returns_release_stats_for_authenticated_user(self, http_client):
- _override_dependencies(_RecordingClient([_github_response()]), _fresh_cache(), LitellmUserRoles.INTERNAL_USER)
+ _override_dependencies(_minor_release(), _fresh_cache(), LitellmUserRoles.INTERNAL_USER)
response = http_client.get("/get/latest_release_info")
assert response.status_code == 200
assert response.json() == EXPECTED_INFO
- def test_returns_null_when_github_is_unreachable(self, http_client):
+ def test_returns_null_when_the_changelog_is_unreachable(self, http_client):
_override_dependencies(
- _RecordingClient([httpx.ConnectError("boom")]), _fresh_cache(), LitellmUserRoles.PROXY_ADMIN
+ _ByUrl({CHANGELOG_FEED_URL: httpx.ConnectError("boom")}), _fresh_cache(), LitellmUserRoles.PROXY_ADMIN
)
response = http_client.get("/get/latest_release_info")
assert response.status_code == 200
assert response.json() is None
def test_repeated_requests_reuse_cache(self, http_client):
- client = _RecordingClient([_github_response()])
+ client = _minor_release()
_override_dependencies(client, _fresh_cache(), LitellmUserRoles.PROXY_ADMIN)
assert http_client.get("/get/latest_release_info").json() == EXPECTED_INFO
assert http_client.get("/get/latest_release_info").json() == EXPECTED_INFO
- assert len(client.calls) == 1
+ assert len(client.calls) == 2
def test_rejects_unauthenticated_requests(self, monkeypatch):
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", MASTER_KEY)
diff --git a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx
index e0c428a94a0..93d9b6018ac 100644
--- a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx
+++ b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx
@@ -20,7 +20,7 @@ const RELEASE: LatestReleaseInfo = {
new_features: 12,
bug_fixes: 30,
other_updates: 8,
- release_url: "https://github.com/BerriAI/litellm/releases/tag/v1.103.0",
+ release_url: "https://docs.litellm.ai/release_notes/v1.103.0/v1-103-0",
};
const bell = (unread: number) =>
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 2e2ff77963d..f3821a2bb46 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -5865,9 +5865,9 @@ export interface paths {
};
/**
* Latest Release Info
- * @description Latest stable LiteLLM GitHub release with the PR count of its whole minor line (e.g. v1.105.0 + v1.105.1)
- * split into new features, bug fixes and other updates.
- * Returns null when GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
+ * @description Newest stable release on the LiteLLM changelog, linked to its changelog page, with the PR count of its whole
+ * minor line (e.g. v1.105.0 + v1.105.1) from GitHub split into new features, bug fixes and other updates.
+ * Returns null when the changelog or GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
*/
get: operations["latest_release_info_get_latest_release_info_get"];
put?: never;
@@ -16025,7 +16025,7 @@ export interface paths {
put?: never;
/**
* Ask named questions about a state (System One format)
- * @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.
+ * @description Judge a `state` against one or more named `questions` and get one calibrated answer per question name.
*
* Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with
* `confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,
@@ -22346,7 +22346,7 @@ export interface paths {
put?: never;
/**
* Ask named questions about a state (System One format)
- * @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.
+ * @description Judge a `state` against one or more named `questions` and get one calibrated answer per question name.
*
* Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with
* `confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,