chore: sync decision rerank with main

Merge main without conflicts and preserve the existing PR diff

Validation passed with 867 focused tests and make check
This commit is contained in:
Ishaan 2026-10-23 11:22:55 -07:00
commit 4343c562ff
12 changed files with 340 additions and 124 deletions

View file

@ -75,6 +75,12 @@ def messages_carry_output_config(messages: Sequence[object]) -> bool:
_messages_carry_output_config = messages_carry_output_config
def _ensure_stop_sequence(response: AnthropicMessagesResponse) -> AnthropicMessagesResponse:
if "stop_sequence" not in response:
response["stop_sequence"] = None # rebind-ok: normalize the request-owned response in place
return response
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
_workload_identity_eligible: ClassVar[bool] = True
@ -715,14 +721,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> AnthropicMessagesResponse:
"""
No transformation is needed for Anthropic messages, since we want the response in the Anthropic /v1/messages API spec
"""
try:
raw_response_json: Final = raw_response.json()
except Exception:
raise AnthropicError(message=raw_response.text, status_code=raw_response.status_code)
return AnthropicMessagesResponse(**raw_response_json)
response: Final = AnthropicMessagesResponse(**raw_response_json)
return _ensure_stop_sequence(response)
def get_async_streaming_response_iterator(
self,

View file

@ -11407,7 +11407,7 @@
},
"/systemone": {
"post": {
"description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"operationId": "systemone_systemone_post",
"requestBody": {
"content": {
@ -11813,7 +11813,6 @@
]
},
"description": "Named noul, choice and score questions. Each key becomes a key in the answers",
"maxProperties": 128,
"minProperties": 1,
"propertyNames": {
"minLength": 1
@ -12814,7 +12813,7 @@
},
"/v1/systemone": {
"post": {
"description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"operationId": "systemone_v1_systemone_post",
"requestBody": {
"content": {
@ -13220,7 +13219,6 @@
]
},
"description": "Named noul, choice and score questions. Each key becomes a key in the answers",
"maxProperties": 128,
"minProperties": 1,
"propertyNames": {
"minLength": 1

View file

@ -228,7 +228,7 @@ async def systemone(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
):
"""
Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.
Judge a `state` against one or more named `questions` and get one calibrated answer per question name.
Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with
`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,

View file

@ -1,5 +1,6 @@
import asyncio
import re
import xml.etree.ElementTree as ET
from collections import Counter
from collections.abc import Awaitable, Mapping
from dataclasses import dataclass
@ -17,7 +18,8 @@ from litellm.types.llms.base import LiteLLMBaseModel
router: Final = APIRouter()
LATEST_RELEASE_URL: Final = "https://api.github.com/repos/BerriAI/litellm/releases/latest"
CHANGELOG_URL: Final = "https://docs.litellm.ai/release_notes/"
CHANGELOG_FEED_URL: Final = f"{CHANGELOG_URL}rss.xml"
RELEASE_BY_TAG_URL: Final = "https://api.github.com/repos/BerriAI/litellm/releases/tags/{tag}"
LATEST_RELEASE_FETCH_TIMEOUT_SECONDS: Final = 5
LATEST_RELEASE_CACHE_TTL_SECONDS: Final = 60 * 60
@ -26,7 +28,7 @@ LATEST_RELEASE_CACHE_KEY: Final = "latest_release_info"
_RELEASE_BULLET_PATTERN: Final = re.compile(r"^\*\s+(?:([A-Za-z]+)(?:\([^)]*\))?!?:\s)?\S")
_NEW_CONTRIBUTOR_PATTERN: Final = re.compile(r"^\*\s+@\S+ made their first contribution\b")
_STABLE_TAG_PATTERN: Final = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)$")
_CHANGELOG_PAGE_PATTERN: Final = re.compile(rf"{re.escape(CHANGELOG_URL)}v(\d+)\.(\d+)\.(\d+)/")
_Bucket: TypeAlias = Literal["new_features", "bug_fixes", "other_updates"]
_PREFIX_BUCKETS: Final[Mapping[str, _Bucket]] = MappingProxyType({"feat": "new_features", "fix": "bug_fixes"})
@ -45,6 +47,16 @@ class LatestReleaseUnavailable:
reason: str
@dataclass(frozen=True, slots=True)
class ChangelogEntry:
version: tuple[int, int, int]
url: str
@property
def tag(self) -> str:
return "v" + ".".join(str(part) for part in self.version)
class _GitHubRelease(LiteLLMBaseModel):
tag_name: str
html_url: str
@ -91,9 +103,18 @@ def count_release_bullets(body: str) -> Mapping[_Bucket, int]:
return MappingProxyType(Counter(bucket for line in body.splitlines() if (bucket := _bucket_for(line)) is not None))
def _stable_version(tag_name: str) -> tuple[int, int, int] | None:
match: Final = _STABLE_TAG_PATTERN.match(tag_name)
return None if match is None else (int(match.group(1)), int(match.group(2)), int(match.group(3)))
def _changelog_entry(link: str) -> ChangelogEntry | None:
match: Final = _CHANGELOG_PAGE_PATTERN.match(link)
if match is None:
return None
return ChangelogEntry(version=(int(match.group(1)), int(match.group(2)), int(match.group(3))), url=link)
def latest_changelog_entry(feed: bytes) -> ChangelogEntry | None:
"""The highest stable release with a page in the changelog feed. Release candidate pages don't count."""
links: Final = (link.text or "" for link in ET.fromstring(feed).iterfind("channel/item/link"))
entries: Final = (entry for link in links if (entry := _changelog_entry(link)) is not None)
return max(entries, key=lambda entry: entry.version, default=None)
def earlier_patch_tags(version: tuple[int, int, int]) -> tuple[str, ...]:
@ -102,7 +123,9 @@ def earlier_patch_tags(version: tuple[int, int, int]) -> tuple[str, ...]:
return tuple(f"v{major}.{minor}.{earlier}" for earlier in range(patch))
def summarize_release_line(latest: _GitHubRelease, earlier: tuple[_GitHubRelease, ...]) -> LatestReleaseInfo:
def summarize_release_line(
entry: ChangelogEntry, latest: _GitHubRelease, earlier: tuple[_GitHubRelease, ...]
) -> LatestReleaseInfo:
"""A patch release's notes only list its backports, so count the whole minor line it belongs to."""
counts: Final = sum(
(Counter[_Bucket](count_release_bullets(release.body or "")) for release in (latest, *earlier)),
@ -113,7 +136,7 @@ def summarize_release_line(latest: _GitHubRelease, earlier: tuple[_GitHubRelease
new_features=counts.get("new_features", 0),
bug_fixes=counts.get("bug_fixes", 0),
other_updates=counts.get("other_updates", 0),
release_url=latest.html_url,
release_url=entry.url,
)
@ -137,25 +160,38 @@ async def _fetch_earlier_release(client: _AsyncGetClient, tag: str) -> _GitHubRe
return release
async def _fetch_changelog_entry(client: _AsyncGetClient) -> ChangelogEntry | LatestReleaseUnavailable:
response: Final = await client.get(CHANGELOG_FEED_URL, timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
if response.status_code != 200:
return LatestReleaseUnavailable(reason=f"Changelog feed responded with status {response.status_code}")
try:
entry: Final = latest_changelog_entry(response.content)
except ET.ParseError as e:
return LatestReleaseUnavailable(reason=f"Changelog feed was not valid XML: {e}")
return LatestReleaseUnavailable(reason="Changelog feed lists no stable release") if entry is None else entry
async def fetch_latest_release(client: _AsyncGetClient) -> LatestReleaseInfo | LatestReleaseUnavailable:
try:
entry: Final = await _fetch_changelog_entry(client)
if isinstance(entry, LatestReleaseUnavailable):
return entry
latest: Final = parse_github_release(
await client.get(LATEST_RELEASE_URL, timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
await client.get(RELEASE_BY_TAG_URL.format(tag=entry.tag), timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
)
if isinstance(latest, LatestReleaseUnavailable):
return latest
version: Final = _stable_version(latest.tag_name)
if version is None:
return summarize_release_line(latest, ())
earlier: Final = await asyncio.gather(
*(_fetch_earlier_release(client, tag) for tag in earlier_patch_tags(version))
*(_fetch_earlier_release(client, tag) for tag in earlier_patch_tags(entry.version))
)
except httpx.HTTPError as e:
return LatestReleaseUnavailable(reason=f"{type(e).__name__}: {e}")
unavailable: Final = next((release for release in earlier if isinstance(release, LatestReleaseUnavailable)), None)
if unavailable is not None:
return unavailable
return summarize_release_line(latest, tuple(release for release in earlier if isinstance(release, _GitHubRelease)))
return summarize_release_line(
entry, latest, tuple(release for release in earlier if isinstance(release, _GitHubRelease))
)
async def get_latest_release_info(
@ -190,9 +226,9 @@ async def latest_release_info(
fetch_lock: Annotated[asyncio.Lock, Depends(_default_fetch_lock)],
) -> LatestReleaseInfo | None:
"""
Latest stable LiteLLM GitHub release with the PR count of its whole minor line (e.g. v1.105.0 + v1.105.1)
split into new features, bug fixes and other updates.
Returns null when GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
Newest stable release on the LiteLLM changelog, linked to its changelog page, with the PR count of its whole
minor line (e.g. v1.105.0 + v1.105.1) from GitHub split into new features, bug fixes and other updates.
Returns null when the changelog or GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
"""
result: Final = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock)
if isinstance(result, LatestReleaseUnavailable):

View file

@ -73,7 +73,7 @@ DecisionQuestion: TypeAlias = Annotated[
DecisionQuestionMap: TypeAlias = Annotated[
Mapping[Annotated[str, Field(min_length=1)], DecisionQuestion],
Field(min_length=1, max_length=MAX_DECISION_QUESTIONS),
Field(min_length=1),
]

View file

@ -2033,6 +2033,7 @@ async def test_anthropic_messages_with_extra_headers():
],
"model": "claude-haiku-4-5-20251001",
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 20},
}
@ -2096,6 +2097,7 @@ async def test_anthropic_messages_with_thinking():
],
"model": "claude-haiku-4-5-20251001",
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 20},
}
@ -2210,6 +2212,7 @@ async def test_anthropic_messages_bedrock_dynamic_region():
"content": [{"type": "text", "text": "This is a mock response"}],
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 20},
}

View file

@ -1,17 +1,23 @@
from collections.abc import Mapping
from datetime import datetime, timezone
from pathlib import Path
from typing import Final
from unittest.mock import patch
import httpx
import pytest
from pydantic import JsonValue
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.anthropic.pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from litellm.llms.anthropic.wif import get_anthropic_wif_token
from litellm.llms.minimax.messages.transformation import MinimaxMessagesConfig
from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig
from litellm.llms.tencent.messages.transformation import TencentAnthropicMessagesConfig
from litellm.types.llms.anthropic import AnthropicFinishReason
from litellm.types.router import GenericLiteLLMParams
from tests.unit.llms.anthropic.test_anthropic_wif import (
ScriptedPoster,
@ -27,6 +33,52 @@ _WIF_PARAMS: Final[dict] = {
}
@pytest.mark.parametrize("config", (AnthropicMessagesConfig(), OpenAILikeAnthropicMessagesConfig()), ids=type)
@pytest.mark.parametrize(
("stop_reason", "stop_fields", "expected_stop_sequence"),
(
pytest.param("end_turn", {}, None, id="end_turn_missing"),
pytest.param("max_tokens", {}, None, id="max_tokens_missing"),
pytest.param("tool_use", {}, None, id="tool_use_missing"),
pytest.param("end_turn", {"stop_sequence": None}, None, id="explicit_null"),
pytest.param("stop_sequence", {"stop_sequence": ", 7"}, ", 7", id="matched_stop"),
),
)
def test_native_response_defaults_stop_sequence_without_changing_other_fields(
config: AnthropicMessagesConfig,
stop_reason: AnthropicFinishReason,
stop_fields: Mapping[str, str | None],
expected_stop_sequence: str | None,
) -> None:
upstream: Final[dict[str, JsonValue]] = {
"id": "msg_test",
"type": "message",
"role": "assistant",
"model": "test-model",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": stop_reason,
"usage": {"input_tokens": 19, "output_tokens": 42},
**stop_fields,
}
logging_obj: Final = Logging(
model="test-model",
messages=[],
stream=False,
call_type="anthropic_messages",
start_time=datetime(2026, 1, 1, tzinfo=timezone.utc),
litellm_call_id="test-call",
function_id="test-function",
)
response: Final = config.transform_anthropic_messages_response(
model="test-model", raw_response=httpx.Response(200, json=upstream), logging_obj=logging_obj
)
assert response == {
**upstream,
"stop_sequence": expected_stop_sequence,
}
def test_workload_identity_allowed_for_anthropic() -> None:
assert AnthropicMessagesConfig()._allows_workload_identity is True

View file

@ -26,7 +26,9 @@ from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
is_provider_error_chunk,
parse_anthropic_error_event,
)
from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig
from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator
from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig
from litellm.types.utils import Choices, Message, ModelResponse
@ -1733,6 +1735,49 @@ async def _drained(stream: AsyncIterator[bytes]) -> list[bytes]:
return [chunk async for chunk in stream]
@pytest.mark.asyncio
@pytest.mark.parametrize("config", (AnthropicMessagesConfig(), OpenAILikeAnthropicMessagesConfig()), ids=type)
@pytest.mark.parametrize("stop_sequence", (None, ", 7"), ids=("null", "matched_stop"))
async def test_native_stream_preserves_nullable_stop_fields_across_chunk_boundaries(
config: AnthropicMessagesConfig, stop_sequence: str | None
) -> None:
logging_obj: Final = _make_logging_obj("native_stream_stop_fields")
events: Final = (
{
"type": "message_start",
"message": {
"id": "msg_test",
"type": "message",
"role": "assistant",
"model": logging_obj.model,
"content": [],
"stop_reason": None,
"stop_sequence": None,
"usage": {"input_tokens": 19, "output_tokens": 0},
},
},
{
"type": "message_delta",
"delta": {
"stop_reason": "end_turn" if stop_sequence is None else "stop_sequence",
"stop_sequence": stop_sequence,
},
"usage": {"output_tokens": 0},
},
{"type": "message_stop"},
)
upstream: Final = b"".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events)
response: Final = httpx.Response(200, content=_chunked(upstream, 7))
stream: Final = config.get_async_streaming_response_iterator(
model=logging_obj.model,
httpx_response=response,
request_body={"model": logging_obj.model},
litellm_logging_obj=logging_obj,
)
assert b"".join(await _drained(stream)) == upstream
def _data_events(frames: Sequence[bytes]) -> list[Mapping[str, object]]:
return [
json.loads(line[len(b"data:") :]) for frame in frames for line in frame.splitlines() if line.startswith(b"data:")

View file

@ -126,6 +126,51 @@ def test_proxy_decisions_route_returns_answers_and_cost(
assert upstream.calls[0].request.headers["authorization"] == "Bearer test-key"
@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/systemone"))
@pytest.mark.parametrize("question_count", (128, 129, 154))
def test_proxy_systemone_preserves_large_question_batches(
client: TestClient,
respx_mock: respx.MockRouter,
endpoint: str,
question_count: int,
) -> None:
questions: Final = {
f"q{index}": {"type": "noul", "instructions": f"Question {index}?"} for index in range(question_count)
}
answers: Final = {f"q{index}": {"type": "noul", "noul": index / question_count} for index in range(question_count)}
upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(
json={**_RESPONSE, "answers": answers}
)
response: Final = client.post(endpoint, json={**_REQUEST, "questions": questions})
assert response.status_code == 200, response.text
assert json.loads(upstream.calls[0].request.content) == {
**_REQUEST,
"model": "pplx-decider-v1-27b",
"questions": questions,
}
assert response.json()["answers"] == answers
@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/systemone"))
@pytest.mark.parametrize(
"questions",
({}, {"": {"type": "noul", "instructions": "Is this a defect?"}}, {"is_defect": {"type": "noul"}}),
ids=("empty", "empty_name", "missing_instructions_or_criteria"),
)
def test_proxy_systemone_rejects_invalid_questions_before_http(
client: TestClient,
respx_mock: respx.MockRouter,
endpoint: str,
questions: Mapping[str, object],
) -> None:
response: Final = client.post(endpoint, json={**_REQUEST, "questions": questions})
assert response.status_code == 400, response.text
assert len(respx_mock.calls) == 0
def test_proxy_decisions_dispatches_typesafe_deployment(
client: TestClient,
monkeypatch: pytest.MonkeyPatch,

View file

@ -12,10 +12,11 @@ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.proxy_server import app
from litellm.proxy.ui_crud_endpoints.latest_release_endpoints import (
CHANGELOG_FEED_URL,
CHANGELOG_URL,
LATEST_RELEASE_CACHE_KEY,
LATEST_RELEASE_CACHE_TTL_SECONDS,
LATEST_RELEASE_UNAVAILABLE_CACHE_TTL_SECONDS,
LATEST_RELEASE_URL,
RELEASE_BY_TAG_URL,
LatestReleaseInfo,
LatestReleaseUnavailable,
@ -49,37 +50,68 @@ SAMPLE_RELEASE: Final = {
"html_url": "https://github.com/BerriAI/litellm/releases/tag/v1.102.0",
"body": SAMPLE_BODY,
}
def _changelog_page(folder: str) -> str:
return f"{CHANGELOG_URL}{folder}/{folder.removeprefix('v').replace('.', '-')}"
def _feed(*folders: str) -> httpx.Response:
items: Final = "".join(
f"<item><title>{folder}</title><link>{_changelog_page(folder)}</link></item>" for folder in folders
)
return httpx.Response(200, content=f'<?xml version="1.0"?><rss><channel>{items}</channel></rss>'.encode())
EXPECTED_INFO: Final = {
"version": "1.102.0",
"new_features": 2,
"bug_fixes": 2,
"other_updates": 4,
"release_url": SAMPLE_RELEASE["html_url"],
"release_url": _changelog_page("v1.102.0"),
}
class _RecordingClient:
def __init__(self, outcomes: list[httpx.Response | Exception]) -> None:
self._outcomes = outcomes
self.calls: list[tuple[str, float | None]] = []
async def get(self, url: str, *, timeout: float | None = None) -> httpx.Response:
self.calls.append((url, timeout))
outcome = self._outcomes[min(len(self.calls) - 1, len(self._outcomes) - 1)]
if isinstance(outcome, Exception):
raise outcome
return outcome
def _github_response(status: int = 200, payload: object = SAMPLE_RELEASE) -> httpx.Response:
return httpx.Response(status, content=json.dumps(payload).encode())
def _tag_url(tag: str) -> str:
return RELEASE_BY_TAG_URL.format(tag=tag)
class _ByUrl:
"""Answers each URL from a fixed map; unknown URLs are 404, like a tag with no release."""
def __init__(self, routes: dict[str, httpx.Response | Exception]) -> None:
self._routes = routes
self.calls: list[tuple[str, float | None]] = []
async def get(self, url: str, *, timeout: float | None = None) -> httpx.Response:
self.calls.append((url, timeout))
outcome = self._routes.get(url, httpx.Response(404, content=b"{}"))
if isinstance(outcome, Exception):
raise outcome
return outcome
@property
def urls(self) -> list[str]:
return [url for url, _ in self.calls]
def _minor_release_routes(**overrides: httpx.Response | Exception) -> dict[str, httpx.Response | Exception]:
return {CHANGELOG_FEED_URL: _feed("v1.102.0"), _tag_url("v1.102.0"): _github_response(), **overrides}
def _minor_release(**overrides: httpx.Response | Exception) -> _ByUrl:
return _ByUrl(_minor_release_routes(**overrides))
def _fresh_cache() -> InMemoryCache:
return InMemoryCache(max_size_in_memory=1, default_ttl=LATEST_RELEASE_CACHE_TTL_SECONDS)
def _override_dependencies(client: _RecordingClient, cache: InMemoryCache, role: LitellmUserRoles) -> None:
def _override_dependencies(client: _ByUrl, cache: InMemoryCache, role: LitellmUserRoles) -> None:
async def auth() -> UserAPIKeyAuth:
return UserAPIKeyAuth(api_key="sk-test", user_id="test-user", user_role=role)
@ -127,35 +159,68 @@ class TestCountReleaseBullets:
class TestGetLatestReleaseInfo:
@pytest.mark.asyncio
async def test_fetches_and_parses_github_release(self):
client = _RecordingClient([_github_response()])
async def test_announces_the_changelog_release_and_links_its_changelog_page(self):
client = _minor_release()
result = await get_latest_release_info(client=client, cache=_fresh_cache(), fetch_lock=asyncio.Lock())
assert isinstance(result, LatestReleaseInfo)
assert result.model_dump() == EXPECTED_INFO
assert client.calls == [(LATEST_RELEASE_URL, 5)]
assert client.calls == [(CHANGELOG_FEED_URL, 5), (_tag_url("v1.102.0"), 5)]
@pytest.mark.asyncio
async def test_github_release_missing_from_the_changelog_is_not_announced(self):
automated: Final = _github_response(payload=_release("v1.103.0", "* feat(proxy): automated release\n"))
result = await _info(_minor_release(**{_tag_url("v1.103.0"): automated}))
assert isinstance(result, LatestReleaseInfo)
assert result.model_dump() == EXPECTED_INFO
@pytest.mark.asyncio
async def test_highest_stable_version_wins_regardless_of_feed_order(self):
client = _minor_release(**{CHANGELOG_FEED_URL: _feed("v1.101.6", "v1.103.0rc1", "v1.102.0", "v1.100.9")})
result = await _info(client)
assert isinstance(result, LatestReleaseInfo)
assert result.model_dump() == EXPECTED_INFO
@pytest.mark.asyncio
@pytest.mark.parametrize(
"feed",
[
httpx.Response(503, content=_feed("v1.102.0").content),
httpx.Response(200, content=b"<html>not a feed"),
_feed("v1.103.0rc1", "v1.103.0-dev.1"),
httpx.Response(
200,
content=f"<rss><channel><item><link>https://example.com/?{_changelog_page('v9.0.0')}</link>"
"</item><item><link>https://example.com/release_notes/v9.0.0/x</link></item></channel></rss>".encode(),
),
],
ids=["server_error", "not_xml", "only_prereleases", "foreign_links"],
)
async def test_bad_changelog_feeds_are_unavailable(self, feed: httpx.Response):
client = _minor_release(**{CHANGELOG_FEED_URL: feed})
result = await _info(client)
assert isinstance(result, LatestReleaseUnavailable)
assert client.urls == [CHANGELOG_FEED_URL]
@pytest.mark.asyncio
async def test_second_call_within_ttl_does_not_refetch(self):
client = _RecordingClient([_github_response()])
client = _minor_release()
cache = _fresh_cache()
fetch_lock = asyncio.Lock()
first = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock)
second = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock)
assert first == second
assert len(client.calls) == 1
assert len(client.calls) == 2
@pytest.mark.asyncio
async def test_success_is_cached_for_the_full_ttl(self):
cache = _fresh_cache()
await get_latest_release_info(
client=_RecordingClient([_github_response()]), cache=cache, fetch_lock=asyncio.Lock()
)
await get_latest_release_info(client=_minor_release(), cache=cache, fetch_lock=asyncio.Lock())
remaining = await cache.async_get_ttl(LATEST_RELEASE_CACHE_KEY) - time.time()
assert LATEST_RELEASE_CACHE_TTL_SECONDS - 5 < remaining <= LATEST_RELEASE_CACHE_TTL_SECONDS
@pytest.mark.asyncio
async def test_failure_is_cached_briefly_so_github_is_not_hammered(self):
client = _RecordingClient([httpx.ConnectError("boom")])
client = _ByUrl({CHANGELOG_FEED_URL: httpx.ConnectError("boom")})
cache = _fresh_cache()
fetch_lock = asyncio.Lock()
first = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock)
@ -172,30 +237,29 @@ class TestGetLatestReleaseInfo:
@pytest.mark.parametrize(
"response",
[
_github_response(status=404, payload={"message": "Not Found"}),
_github_response(status=403, payload={"message": "rate limited"}),
_github_response(status=500, payload={}),
_github_response(payload={"tag_name": "v1.0.0"}),
httpx.Response(200, content=b"not json"),
httpx.ConnectError("boom"),
],
ids=["rate_limited", "server_error", "missing_fields", "not_json"],
ids=["no_release_for_tag", "rate_limited", "server_error", "missing_fields", "not_json", "unreachable"],
)
async def test_bad_github_responses_are_unavailable(self, response: httpx.Response):
result = await get_latest_release_info(
client=_RecordingClient([response]), cache=_fresh_cache(), fetch_lock=asyncio.Lock()
)
async def test_bad_github_responses_are_unavailable(self, response: httpx.Response | Exception):
result = await _info(_minor_release(**{_tag_url("v1.102.0"): response}))
assert isinstance(result, LatestReleaseUnavailable)
@pytest.mark.asyncio
async def test_concurrent_misses_share_one_fetch(self):
event = asyncio.Event()
class _BlockingClient(_RecordingClient):
class _BlockingClient(_ByUrl):
async def get(self, url: str, *, timeout: float | None = None) -> httpx.Response:
self.calls.append((url, timeout))
await event.wait()
return _github_response()
return await super().get(url, timeout=timeout)
client = _BlockingClient([])
client = _BlockingClient(_minor_release_routes())
cache = _fresh_cache()
fetch_lock = asyncio.Lock()
tasks = [
@ -208,19 +272,19 @@ class TestGetLatestReleaseInfo:
results = await asyncio.gather(*tasks)
expected: Final = LatestReleaseInfo.model_validate(EXPECTED_INFO)
assert results == [expected] * 5
assert len(client.calls) == 1
assert len(client.calls) == 2
@pytest.mark.asyncio
async def test_failure_under_lock_is_also_coalesced(self):
event = asyncio.Event()
class _FailingBlockingClient(_RecordingClient):
class _FailingBlockingClient(_ByUrl):
async def get(self, url: str, *, timeout: float | None = None) -> httpx.Response:
self.calls.append((url, timeout))
await event.wait()
raise httpx.ConnectError("boom")
client = _FailingBlockingClient([])
client = _FailingBlockingClient({})
cache = _fresh_cache()
fetch_lock = asyncio.Lock()
tasks = [
@ -245,40 +309,23 @@ def _release(tag: str, body: str | None, *, prerelease: bool = False) -> dict[st
}
def _tag_url(tag: str) -> str:
return RELEASE_BY_TAG_URL.format(tag=tag)
class _GitHubByUrl:
"""Answers each GitHub URL from a fixed map; unknown URLs are 404, like a tag with no release."""
def __init__(self, routes: dict[str, httpx.Response | Exception]) -> None:
self._routes = routes
self.urls: list[str] = []
async def get(self, url: str, *, timeout: float | None = None) -> httpx.Response:
self.urls.append(url)
outcome = self._routes.get(url, httpx.Response(404, content=b"{}"))
if isinstance(outcome, Exception):
raise outcome
return outcome
PATCH_RELEASE: Final = _release("v1.102.2", "* fix(proxy): second backport by @k in https://x/pull/10\n")
FIRST_PATCH: Final = _release("v1.102.1", "* fix(proxy): first backport by @k in https://x/pull/9\n")
def _minor_line(**overrides: httpx.Response | Exception) -> dict[str, httpx.Response | Exception]:
routes: dict[str, httpx.Response | Exception] = {
LATEST_RELEASE_URL: _github_response(payload=PATCH_RELEASE),
_tag_url("v1.102.0"): _github_response(payload=SAMPLE_RELEASE),
_tag_url("v1.102.1"): _github_response(payload=FIRST_PATCH),
}
routes.update({_tag_url(tag): outcome for tag, outcome in overrides.items()})
return routes
def _minor_line(**overrides: httpx.Response | Exception) -> _ByUrl:
return _ByUrl(
{
CHANGELOG_FEED_URL: _feed("v1.102.2", "v1.101.6"),
_tag_url("v1.102.2"): _github_response(payload=PATCH_RELEASE),
_tag_url("v1.102.0"): _github_response(payload=SAMPLE_RELEASE),
_tag_url("v1.102.1"): _github_response(payload=FIRST_PATCH),
**{_tag_url(tag): outcome for tag, outcome in overrides.items()},
}
)
async def _info(client: _GitHubByUrl) -> LatestReleaseInfo | LatestReleaseUnavailable:
async def _info(client: _ByUrl) -> LatestReleaseInfo | LatestReleaseUnavailable:
return await get_latest_release_info(client=client, cache=_fresh_cache(), fetch_lock=asyncio.Lock())
@ -289,7 +336,7 @@ class TestPatchReleaseAggregatesMinorLine:
@pytest.mark.asyncio
async def test_patch_release_counts_include_every_earlier_patch_on_its_line(self):
client = _GitHubByUrl(_minor_line())
client = _minor_line()
result = await _info(client)
assert isinstance(result, LatestReleaseInfo)
assert result.model_dump() == {
@ -297,23 +344,11 @@ class TestPatchReleaseAggregatesMinorLine:
"new_features": 2,
"bug_fixes": 4,
"other_updates": 4,
"release_url": PATCH_RELEASE["html_url"],
"release_url": _changelog_page("v1.102.2"),
}
assert sorted(client.urls) == sorted([LATEST_RELEASE_URL, _tag_url("v1.102.0"), _tag_url("v1.102.1")])
@pytest.mark.asyncio
async def test_minor_release_makes_a_single_request(self):
client = _RecordingClient([_github_response()])
await get_latest_release_info(client=client, cache=_fresh_cache(), fetch_lock=asyncio.Lock())
assert client.calls == [(LATEST_RELEASE_URL, 5)]
@pytest.mark.asyncio
async def test_non_semver_latest_tag_counts_only_its_own_notes(self):
client = _GitHubByUrl({LATEST_RELEASE_URL: _github_response(payload=_release("v1.102.1-hotfix", "* fix: x\n"))})
result = await _info(client)
assert isinstance(result, LatestReleaseInfo)
assert (result.new_features, result.bug_fixes, result.other_updates) == (0, 1, 0)
assert client.urls == [LATEST_RELEASE_URL]
assert sorted(client.urls) == sorted(
[CHANGELOG_FEED_URL, _tag_url("v1.102.2"), _tag_url("v1.102.0"), _tag_url("v1.102.1")]
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
@ -326,15 +361,13 @@ class TestPatchReleaseAggregatesMinorLine:
ids=["never_published", "prerelease", "null_body"],
)
async def test_missing_prerelease_or_empty_patch_adds_nothing(self, skipped: httpx.Response):
result = await _info(_GitHubByUrl(_minor_line(**{"v1.102.1": skipped})))
result = await _info(_minor_line(**{"v1.102.1": skipped}))
assert isinstance(result, LatestReleaseInfo)
assert (result.new_features, result.bug_fixes, result.other_updates) == (2, 3, 4)
@pytest.mark.asyncio
async def test_latest_release_with_null_body_still_counts_its_line(self):
routes = _minor_line()
routes[LATEST_RELEASE_URL] = _github_response(payload=_release("v1.102.2", None))
result = await _info(_GitHubByUrl(routes))
result = await _info(_minor_line(**{"v1.102.2": _github_response(payload=_release("v1.102.2", None))}))
assert isinstance(result, LatestReleaseInfo)
assert (result.new_features, result.bug_fixes, result.other_updates) == (2, 3, 4)
@ -349,31 +382,31 @@ class TestPatchReleaseAggregatesMinorLine:
ids=["unreachable", "rate_limited", "not_json"],
)
async def test_earlier_patch_failure_is_unavailable(self, outcome: httpx.Response | Exception):
result = await _info(_GitHubByUrl(_minor_line(**{"v1.102.0": outcome})))
result = await _info(_minor_line(**{"v1.102.0": outcome}))
assert isinstance(result, LatestReleaseUnavailable)
class TestLatestReleaseInfoEndpoint:
def test_returns_release_stats_for_authenticated_user(self, http_client):
_override_dependencies(_RecordingClient([_github_response()]), _fresh_cache(), LitellmUserRoles.INTERNAL_USER)
_override_dependencies(_minor_release(), _fresh_cache(), LitellmUserRoles.INTERNAL_USER)
response = http_client.get("/get/latest_release_info")
assert response.status_code == 200
assert response.json() == EXPECTED_INFO
def test_returns_null_when_github_is_unreachable(self, http_client):
def test_returns_null_when_the_changelog_is_unreachable(self, http_client):
_override_dependencies(
_RecordingClient([httpx.ConnectError("boom")]), _fresh_cache(), LitellmUserRoles.PROXY_ADMIN
_ByUrl({CHANGELOG_FEED_URL: httpx.ConnectError("boom")}), _fresh_cache(), LitellmUserRoles.PROXY_ADMIN
)
response = http_client.get("/get/latest_release_info")
assert response.status_code == 200
assert response.json() is None
def test_repeated_requests_reuse_cache(self, http_client):
client = _RecordingClient([_github_response()])
client = _minor_release()
_override_dependencies(client, _fresh_cache(), LitellmUserRoles.PROXY_ADMIN)
assert http_client.get("/get/latest_release_info").json() == EXPECTED_INFO
assert http_client.get("/get/latest_release_info").json() == EXPECTED_INFO
assert len(client.calls) == 1
assert len(client.calls) == 2
def test_rejects_unauthenticated_requests(self, monkeypatch):
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", MASTER_KEY)

View file

@ -20,7 +20,7 @@ const RELEASE: LatestReleaseInfo = {
new_features: 12,
bug_fixes: 30,
other_updates: 8,
release_url: "https://github.com/BerriAI/litellm/releases/tag/v1.103.0",
release_url: "https://docs.litellm.ai/release_notes/v1.103.0/v1-103-0",
};
const bell = (unread: number) =>

View file

@ -5865,9 +5865,9 @@ export interface paths {
};
/**
* Latest Release Info
* @description Latest stable LiteLLM GitHub release with the PR count of its whole minor line (e.g. v1.105.0 + v1.105.1)
* split into new features, bug fixes and other updates.
* Returns null when GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
* @description Newest stable release on the LiteLLM changelog, linked to its changelog page, with the PR count of its whole
* minor line (e.g. v1.105.0 + v1.105.1) from GitHub split into new features, bug fixes and other updates.
* Returns null when the changelog or GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
*/
get: operations["latest_release_info_get_latest_release_info_get"];
put?: never;
@ -16025,7 +16025,7 @@ export interface paths {
put?: never;
/**
* Ask named questions about a state (System One format)
* @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.
* @description Judge a `state` against one or more named `questions` and get one calibrated answer per question name.
*
* Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with
* `confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,
@ -22346,7 +22346,7 @@ export interface paths {
put?: never;
/**
* Ask named questions about a state (System One format)
* @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.
* @description Judge a `state` against one or more named `questions` and get one calibrated answer per question name.
*
* Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with
* `confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,