From 67c4eb86b105a12bb2dec81c9750be2a936fc5ab Mon Sep 17 00:00:00 2001 From: ansh-agrawal Date: Tue, 11 Aug 2026 11:53:39 +0530 Subject: [PATCH 001/105] feat(proxy): add opt-in flag to require rpm/tpm for project models (create + update) --- .../management_endpoints/project_endpoints.py | 79 ++++++++ .../test_project_endpoints_prisma.py | 188 ++++++++++++++++++ 2 files changed, 267 insertions(+) diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py index 66fac8d76ee..9249150f6ce 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py @@ -180,6 +180,75 @@ def _check_team_project_limits( ) +def _project_models_missing_positive_quota( + models: list[str] | None, + rpm_limits: Mapping[str, object] | None, + tpm_limits: Mapping[str, object] | None, +) -> list[str]: + """Return the models that lack a positive `rpm` AND `tpm` quota. + + A valid quota is a positive integer; null, zero, and negative are rejected + because downstream rate limiters treat a non-positive limit as immediately + exhausted (every request blocked). + """ + + def _is_positive(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value > 0 + + rpm = rpm_limits or {} + tpm = tpm_limits or {} + return [model for model in (models or []) if not _is_positive(rpm.get(model)) or not _is_positive(tpm.get(model))] + + +def _raise_on_missing_project_model_quota(data: NewProjectRequest | UpdateProjectRequest) -> None: + """Require a positive `rpm`/`tpm` quota for every model on project CREATE. + + `model_rpm_limit`/`model_tpm_limit` are relocated into `metadata` by the request + model's `set_model_info` validator, so they are read from there. + + Only invoked when `general_settings.enforce_project_model_quota` is enabled + (default off), so it is opt-in and does not change behavior for existing users. + """ + metadata = data.metadata or {} + missing = _project_models_missing_positive_quota( + data.models, metadata.get("model_rpm_limit"), metadata.get("model_tpm_limit") + ) + if not missing: + return + raise HTTPException( + status_code=400, + detail={ + "error": f"models {missing} added to project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model." + }, + ) + + +def _raise_on_missing_project_model_quota_on_update(data: UpdateProjectRequest, existing_project: object) -> None: + """Require a positive `rpm`/`tpm` quota over the RESULTING state on project UPDATE. + + `/project/update` replaces `models` and `metadata` when they are provided, so the + check runs on what the project WILL look like: a partial update that doesn't touch + models/quota keeps the existing values, while one that adds a model or clears a + model's quota must leave every resulting model with a positive limit. + + Only invoked when `general_settings.enforce_project_model_quota` is enabled + (default off), so it is opt-in and does not change behavior for existing users. + """ + resulting_models = data.models if data.models is not None else (getattr(existing_project, "models", None) or []) + resulting_metadata = data.metadata if data.metadata is not None else (getattr(existing_project, "metadata", None) or {}) + missing = _project_models_missing_positive_quota( + resulting_models, resulting_metadata.get("model_rpm_limit"), resulting_metadata.get("model_tpm_limit") + ) + if not missing: + return + raise HTTPException( + status_code=400, + detail={ + "error": f"models {missing} would be left on the project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model." + }, + ) + + async def _create_budget_for_project( data: NewProjectRequest, user_id: str | None, @@ -327,6 +396,7 @@ async def new_project( ``` """ from litellm.proxy.proxy_server import ( + general_settings, litellm_proxy_admin_name, premium_user, prisma_client, @@ -374,6 +444,10 @@ async def new_project( data=data, ) + # Opt-in (default off): require rpm/tpm for every model added to the project. + if general_settings.get("enforce_project_model_quota", False): + _raise_on_missing_project_model_quota(data) + # Check if user has permission to create projects for this team # only team admins can create projects for their team has_permission = await _check_user_permission_for_project( @@ -512,6 +586,7 @@ async def update_project( ``` """ from litellm.proxy.proxy_server import ( + general_settings, litellm_proxy_admin_name, premium_user, prisma_client, @@ -616,6 +691,10 @@ async def update_project( data=data, ) + # Opt-in (default off): require rpm/tpm for every model the update would leave on the project. + if general_settings.get("enforce_project_model_quota", False): + _raise_on_missing_project_model_quota_on_update(data, existing_project) + # Prepare update data update_data = data.json(exclude_none=True, exclude={"project_id"}) update_data = prisma_client.jsonify_object(update_data) diff --git a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py b/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py index c29b4c68bb0..f4ac8a55092 100644 --- a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py +++ b/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py @@ -1039,3 +1039,191 @@ async def test_project_eviction_publishes_cross_worker_invalidation(monkeypatch) ) mock_publish.assert_awaited_once_with(cache_key=f"project_id:{project_id}") + + +def test_enforce_project_model_quota_missing_both_raises(): + """A model added to a project without rpm/tpm is rejected.""" + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota, + ) + + data = NewProjectRequest(team_id="test-team", models=["gpt-5.5"]) + with pytest.raises(Exception) as exc_info: + _raise_on_missing_project_model_quota(data) + assert "gpt-5.5" in str(exc_info.value.detail) + assert "rpm/tpm quota" in str(exc_info.value.detail) + + +def test_enforce_project_model_quota_missing_tpm_raises(): + """A model with rpm but no tpm is rejected.""" + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota, + ) + + data = NewProjectRequest( + team_id="test-team", + models=["gpt-5.5"], + model_rpm_limit={"gpt-5.5": 100}, + ) + with pytest.raises(Exception): + _raise_on_missing_project_model_quota(data) + + +def test_enforce_project_model_quota_all_present_passes(): + """A model with both rpm and tpm set passes.""" + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota, + ) + + data = NewProjectRequest( + team_id="test-team", + models=["gpt-5.5"], + model_rpm_limit={"gpt-5.5": 100}, + model_tpm_limit={"gpt-5.5": 1000}, + ) + # Should not raise. + _raise_on_missing_project_model_quota(data) + + +def test_enforce_project_model_quota_no_models_passes(): + """A project with no models has nothing to enforce.""" + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota, + ) + + data = NewProjectRequest(team_id="test-team") + # Should not raise. + _raise_on_missing_project_model_quota(data) + + +def test_enforce_project_model_quota_zero_rejected(): + """A zero quota is non-positive -> rejected (downstream treats it as exhausted).""" + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota, + ) + + data = NewProjectRequest( + team_id="test-team", + models=["gpt-5.5"], + model_rpm_limit={"gpt-5.5": 0}, + model_tpm_limit={"gpt-5.5": 1000}, + ) + with pytest.raises(Exception): + _raise_on_missing_project_model_quota(data) + + +def test_enforce_project_model_quota_negative_rejected(): + """A negative quota is non-positive -> rejected.""" + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota, + ) + + data = NewProjectRequest( + team_id="test-team", + models=["gpt-5.5"], + model_rpm_limit={"gpt-5.5": 100}, + model_tpm_limit={"gpt-5.5": -1}, + ) + with pytest.raises(Exception): + _raise_on_missing_project_model_quota(data) + + +def test_update_quota_adds_model_without_quota_rejected(): + """Adding a model via /project/update without quota is rejected (the bypass).""" + import types + + from litellm.proxy._types import UpdateProjectRequest + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota_on_update, + ) + + existing = types.SimpleNamespace(models=[], metadata={}) + data = UpdateProjectRequest(project_id="p", models=["gpt-5.5"]) # adds model, no quota + with pytest.raises(Exception): + _raise_on_missing_project_model_quota_on_update(data, existing) + + +def test_update_quota_adds_model_with_quota_passes(): + """Adding a model with a positive quota via update passes.""" + import types + + from litellm.proxy._types import UpdateProjectRequest + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota_on_update, + ) + + existing = types.SimpleNamespace(models=[], metadata={}) + data = UpdateProjectRequest( + project_id="p", + models=["gpt-5.5"], + model_rpm_limit={"gpt-5.5": 100}, + model_tpm_limit={"gpt-5.5": 1000}, + ) + # Should not raise. + _raise_on_missing_project_model_quota_on_update(data, existing) + + +def test_update_quota_partial_update_keeps_existing_valid_passes(): + """A partial update that doesn't touch models/quota keeps existing valid quota -> passes.""" + import types + + from litellm.proxy._types import UpdateProjectRequest + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota_on_update, + ) + + existing = types.SimpleNamespace( + models=["gpt-5.5"], + metadata={"model_rpm_limit": {"gpt-5.5": 100}, "model_tpm_limit": {"gpt-5.5": 1000}}, + ) + data = UpdateProjectRequest(project_id="p", description="unrelated change") + # Should not raise (existing quota is valid, update doesn't touch it). + _raise_on_missing_project_model_quota_on_update(data, existing) + + +def test_update_quota_existing_quotaless_model_rejected(): + """A project already holding a quota-less model is rejected on any update (fail-closed).""" + import types + + from litellm.proxy._types import UpdateProjectRequest + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota_on_update, + ) + + existing = types.SimpleNamespace(models=["gpt-5.5"], metadata={}) + data = UpdateProjectRequest(project_id="p", description="unrelated change") + with pytest.raises(Exception): + _raise_on_missing_project_model_quota_on_update(data, existing) + + +@pytest.mark.asyncio +async def test_new_project_flag_on_missing_rpm_tpm_returns_400(): + """End-to-end: with the flag on, POST /project/new rejects a model added without rpm/tpm.""" + from unittest.mock import AsyncMock, MagicMock, patch + + from fastapi import Request + + from litellm.proxy._types import LiteLLM_TeamTable + from litellm_enterprise.proxy.management_endpoints import project_endpoints as pe + + team = LiteLLM_TeamTable(team_id="test-team", models=["gpt-5.5"]) + data = NewProjectRequest(team_id="test-team", models=["gpt-5.5"]) # no rpm/tpm + + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.proxy_server.general_settings", {"enforce_project_model_quota": True}), + patch.object(pe, "_validate_team_exists", AsyncMock(return_value=team)), + patch.object(pe, "_check_user_permission_for_project", AsyncMock(return_value=True)), + ): + with pytest.raises(Exception) as exc_info: + await pe.new_project( + data=data, + http_request=Request(scope={"type": "http"}), + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234", user_id="1234" + ), + ) + + # new_project re-wraps the HTTPException, so assert on the string form. + assert "rpm/tpm quota" in str(exc_info.value) From 1d22faf4085d9ee5ceda513c352347273aeb79a5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 06:51:17 +0000 Subject: [PATCH 002/105] test(litellm_utils_tests): give aiohttp transport tests real assertions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_aiohttp_handler.py | 140 +++++++----------- 1 file changed, 52 insertions(+), 88 deletions(-) diff --git a/tests/litellm_utils_tests/test_aiohttp_handler.py b/tests/litellm_utils_tests/test_aiohttp_handler.py index 9fdac5ca23d..0257660611f 100644 --- a/tests/litellm_utils_tests/test_aiohttp_handler.py +++ b/tests/litellm_utils_tests/test_aiohttp_handler.py @@ -4,6 +4,8 @@ import time from datetime import datetime from unittest import mock +import httpx +from aiohttp import ClientSession from dotenv import load_dotenv from litellm.types.utils import StandardCallbackDynamicParams @@ -13,117 +15,79 @@ load_dotenv() import pytest import litellm +from litellm.llms.custom_httpx.aiohttp_transport import LiteLLMAiohttpTransport from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @pytest.mark.asyncio async def test_client_session_helper(): """Test that the client session helper handles event loop changes correctly""" - try: - # Create a transport with the new helper - transport = AsyncHTTPHandler._create_aiohttp_transport() - if transport is not None: - print("✅ Successfully created aiohttp transport with helper") + transport = AsyncHTTPHandler._create_aiohttp_transport() + assert isinstance(transport, LiteLLMAiohttpTransport) - # Test the helper function directly if it's a LiteLLMAiohttpTransport - if hasattr(transport, "_get_valid_client_session"): - session1 = transport._get_valid_client_session() # type: ignore - print(f"✅ First session created: {type(session1).__name__}") + session1 = transport._get_valid_client_session() + assert isinstance(session1, ClientSession) + assert session1.closed is False + assert getattr(session1, "_loop") is asyncio.get_running_loop() - # Call it again to test reuse - session2 = transport._get_valid_client_session() # type: ignore - print(f"✅ Second session call: {type(session2).__name__}") + # Within the same event loop the valid session is reused, not rebuilt + session2 = transport._get_valid_client_session() + assert session2 is session1 - # In the same event loop, should be the same session - print(f"✅ Same session reused: {session1 is session2}") - - return True - else: - print("ℹ️ No aiohttp transport available (probably missing httpx-aiohttp)") - return True - except Exception as e: - print(f"❌ Error: {e}") - import traceback - - traceback.print_exc() - return False + await session1.close() async def test_event_loop_robustness(): """Test behavior when event loops change (simulating CI/CD scenario)""" - try: - # Test session creation in multiple scenarios - transport = AsyncHTTPHandler._create_aiohttp_transport() + transport = AsyncHTTPHandler._create_aiohttp_transport() - if transport and hasattr(transport, "_get_valid_client_session"): - # Test 1: Normal usage - session = transport._get_valid_client_session() # type: ignore - print(f"✅ Normal session creation works: {session is not None}") + session = transport._get_valid_client_session() + assert isinstance(session, ClientSession) - # Test 2: Force recreation by setting client to a callable - from aiohttp import ClientSession + # A closed session must be replaced with a live one bound to this loop + await session.close() + session_after_close = transport._get_valid_client_session() + assert isinstance(session_after_close, ClientSession) + assert session_after_close is not session + assert session_after_close.closed is False - transport.client = lambda: ClientSession() # type: ignore - session2 = transport._get_valid_client_session() # type: ignore - print(f"✅ Session recreation after callable works: {session2 is not None}") + # A client that is a factory rather than a session must also be rebuilt + transport.client = lambda: ClientSession() # type: ignore[assignment] + session_after_factory = transport._get_valid_client_session() + assert isinstance(session_after_factory, ClientSession) + assert session_after_factory is not session_after_close + assert session_after_factory.closed is False + assert transport.client is session_after_factory - return True - else: - print("ℹ️ Transport not available or no helper method") - return True - - except Exception as e: - print(f"❌ Error in event loop robustness test: {e}") - import traceback - - traceback.print_exc() - return False + await session_after_close.close() + await session_after_factory.close() async def test_httpx_request_simulation(): """Test that the transport can handle a simulated HTTP request""" - try: - transport = AsyncHTTPHandler._create_aiohttp_transport() + transport = AsyncHTTPHandler._create_aiohttp_transport(ssl_verify=False) + request = httpx.Request("GET", "https://httpbin.org/headers") - if transport is not None: - print("✅ Transport created for request simulation") + # The per-request SSL override the request path reads must reflect ssl_verify + assert transport._ssl_verify is False - # Create a simple httpx request to test with - import httpx + session = transport._get_valid_client_session() + assert isinstance(session, ClientSession) + assert session.closed is False + assert callable(session.request) + assert session.connector is not None + assert session.connector._ssl is False - request = httpx.Request("GET", "https://httpbin.org/headers") + with mock.patch.object( + transport, "_make_aiohttp_request", new=mock.AsyncMock(side_effect=RuntimeError("boom")) + ) as mocked_request: + with pytest.raises(RuntimeError): + await transport.handle_async_request(request) - # Just test that we can get a valid session for this request context - if hasattr(transport, "_get_valid_client_session"): - session = transport._get_valid_client_session() # type: ignore - print(f"✅ Got valid session for request: {session is not None}") + assert mocked_request.call_count == 1 + call_kwargs = mocked_request.call_args.kwargs + assert call_kwargs["request"] is request + assert call_kwargs["ssl_verify"] is False + assert call_kwargs["client_session"] is session - # Test that session has required aiohttp methods - has_request_method = hasattr(session, "request") - print(f"✅ Session has request method: {has_request_method}") - - return has_request_method - - return True - else: - print("ℹ️ No transport available for request simulation") - return True - - except Exception as e: - print(f"❌ Error in request simulation: {e}") - return False - - -if __name__ == "__main__": - print("Testing client session helper and event loop handling fix...") - - result1 = asyncio.run(test_client_session_helper()) - result2 = asyncio.run(test_event_loop_robustness()) - result3 = asyncio.run(test_httpx_request_simulation()) - - if result1 and result2 and result3: - print( - "🎉 All tests passed! The helper function approach should fix the CI/CD event loop issues." - ) - else: - print("💥 Some tests failed") + await session.close() From cffde8d21851687e08575267315184cdcad63d77 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 06:57:46 +0000 Subject: [PATCH 003/105] chore(lint): ratchet TQ001 budget for the assertions added to the aiohttp transport tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- test-quality-budget.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-quality-budget.json b/test-quality-budget.json index 4a7bc7edff2..b586b59690d 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -1,6 +1,6 @@ { "TQ001": { - "limit": 744 + "limit": 741 }, "TQ002": { "limit": 742 From 90bc8acd86e274b206c26f0ff6bc3216da799df2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:12:06 -0700 Subject: [PATCH 004/105] feat(models): add daily Together AI model registry sync script and workflow --- .github/workflows/sync-together-ai-models.yml | 68 +++ scripts/sync_together_ai_models.py | 539 ++++++++++++++++++ .../fixtures/together_ai_sync/deprecations.md | 442 ++++++++++++++ .../together_ai_sync/models_serverless.json | 1 + .../test_sync_together_ai_models.py | 350 ++++++++++++ 5 files changed, 1400 insertions(+) create mode 100644 .github/workflows/sync-together-ai-models.yml create mode 100644 scripts/sync_together_ai_models.py create mode 100644 tests/test_litellm/fixtures/together_ai_sync/deprecations.md create mode 100644 tests/test_litellm/fixtures/together_ai_sync/models_serverless.json create mode 100644 tests/test_litellm/test_sync_together_ai_models.py diff --git a/.github/workflows/sync-together-ai-models.yml b/.github/workflows/sync-together-ai-models.yml new file mode 100644 index 00000000000..eca9ad7c968 --- /dev/null +++ b/.github/workflows/sync-together-ai-models.yml @@ -0,0 +1,68 @@ +name: Sync Together AI model registry + +on: + schedule: + - cron: "30 6 * * *" + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + sync_together_ai_models: + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + ref: litellm_internal_staging + persist-credentials: false + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + - name: Look for an already-open sync PR + id: existing + run: | + open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --json headRefName \ + --jq '[.[].headRefName | select(startswith("litellm_together_registry_sync_"))] | first // empty')" + echo "open_pr=$open_pr" >> "$GITHUB_OUTPUT" + if [ -n "$open_pr" ]; then + echo "An open sync PR already exists on branch $open_pr; skipping this run." + fi + env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + - name: Run the sync + if: steps.existing.outputs.open_pr == '' + run: | + uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md" + env: + TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} + - name: Regenerate the JSON schema + if: steps.existing.outputs.open_pr == '' + run: | + uv run --frozen python ci_cd/generate_model_prices_schema.py + - name: Create a pull request when the registry changed + if: steps.existing.outputs.open_pr == '' + run: | + if git diff --quiet; then + echo "Registry already in sync; no PR needed." + exit 0 + fi + branch="litellm_together_registry_sync_$(date +'%Y-%m-%d')" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -b "$branch" + git add model_prices_and_context_window.json \ + litellm/model_prices_and_context_window_backup.json \ + model_prices_and_context_window.schema.json + git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')" + gh auth setup-git + git push origin "$branch" + gh pr create --title "feat(models): sync together_ai model registry" \ + --body-file "$RUNNER_TEMP/pr_body.md" \ + --head "$branch" \ + --base litellm_internal_staging + env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} diff --git a/scripts/sync_together_ai_models.py b/scripts/sync_together_ai_models.py new file mode 100644 index 00000000000..a7f3f36b555 --- /dev/null +++ b/scripts/sync_together_ai_models.py @@ -0,0 +1,539 @@ +"""Sync the together_ai entries of model_prices_and_context_window.json with Together's live serverless catalog. + +Pulls ``GET https://api.together.ai/v1/models?serverless`` plus the deprecations doc, maps API fields onto +registry fields, merges the reviewed capability rules below for everything the API cannot express, and diffs +the result against the registry. Dry run (the default) prints the diff summary and the generated PR body; +``--write`` applies the changes to the root cost map and its ``litellm/`` backup copy. + +Policy highlights: +- Prices arrive per 1M tokens with float artifacts and are normalized to clean per-token values. +- A registry entry absent from the serverless catalog is marked with ``deprecation_date`` from the docs + deprecation table, never deleted; absences with no docs date are surfaced for a human call. +- Availability comes from the API: a model the docs list as removed but the API still serves stays live, + with the conflict surfaced as a warning. +- Manually curated values the API cannot express (``metadata.successor``, ``max_output_tokens`` on existing + entries, capability flags no rule covers) are never overwritten; conflicts are surfaced instead. +""" + +import argparse +import json +import os +import re +import sys +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import httpx +from pydantic import BaseModel, TypeAdapter, ValidationError + +MODELS_URL: Final = "https://api.together.ai/v1/models?serverless" +DEPRECATIONS_URL: Final = "https://docs.together.ai/docs/deprecations.md" +PROVIDER: Final = "together_ai" +PREFIX: Final = "together_ai/" +SOURCE_URL: Final = "https://docs.together.ai/docs/serverless-models" +COST_MAP_RELPATHS: Final = ( + "model_prices_and_context_window.json", + "litellm/model_prices_and_context_window_backup.json", +) + +TYPE_TO_MODE: Final = MappingProxyType({"chat": "chat", "embedding": "embedding", "moderation": "chat"}) + + +class SyncError(RuntimeError): + pass + + +class CatalogPricing(BaseModel): + input: float + output: float + cached_input: float | None = None + + +class CatalogModel(BaseModel): + id: str + type: str + context_length: int | None = None + pricing: CatalogPricing + + +CATALOG_ADAPTER: Final = TypeAdapter(list[CatalogModel]) + +RegistryEntry = dict[str, object] +CostMap = dict[str, object] + + +@dataclass(frozen=True, slots=True) +class CapabilityRule: + model_id: str + fields: Mapping[str, bool | int] + provenance: str + + +def _rule(model_id: str, provenance: str, **fields: bool | int) -> CapabilityRule: + return CapabilityRule(model_id=model_id, fields=MappingProxyType(dict(fields)), provenance=provenance) + + +_TOOLS: Final = MappingProxyType( + { + "supports_function_calling": True, + "supports_parallel_function_calling": True, + "supports_response_schema": True, + "supports_tool_choice": True, + } +) + +CAPABILITY_RULES: Final = ( + _rule( + "MiniMaxAI/MiniMax-M3", + "reviewed for the LIT-5968 backfill against https://www.together.ai/models/minimax-m3", + **_TOOLS, + supports_reasoning=True, + supports_vision=True, + ), + _rule("Prism-ML/Ternary-Bonsai-27B", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), + _rule( + "Qwen/Qwen3.5-9B", + "reviewed for the LIT-5968 backfill against https://www.together.ai/models/qwen3-5-9b", + **_TOOLS, + supports_reasoning=True, + supports_vision=True, + ), + _rule( + "Qwen/Qwen3.6-Plus", + "reviewed for the LIT-5968 backfill; hybrid reasoning model without a documented tools contract", + supports_reasoning=True, + ), + _rule("Qwen/Qwen3.7-Max", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), + _rule("Qwen/Qwen3.7-Plus", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), + _rule("Qwen/Qwen3.8-2.4T-A95B", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), + _rule("arize-ai/qwen-2-1.5b-instruct", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), + _rule( + "deepseek-ai/DeepSeek-V4-Flash-0731", + "reviewed for the LIT-5968 backfill against https://www.together.ai/models/deepseek-v4-flash", + **_TOOLS, + ), + _rule( + "deepseek-ai/DeepSeek-V4-Pro", + "reviewed for the LIT-5968 backfill against https://www.together.ai/models/deepseek-v4-pro", + **_TOOLS, + supports_reasoning=True, + ), + _rule( + "deepseek-ai/DeepSeek-V4-Pro-0813", + "reviewed for the LIT-5968 backfill against https://www.together.ai/models/deepseek-v4-pro", + **_TOOLS, + ), + _rule("google/gemma-3n-E4B-it", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), + _rule( + "google/gemma-4-31B-it", + "reviewed for the LIT-5968 backfill against https://www.together.ai/models/gemma-4-31b-it", + **_TOOLS, + supports_vision=True, + ), + _rule( + "intfloat/multilingual-e5-large-instruct", + "embedding dims per https://huggingface.co/intfloat/multilingual-e5-large-instruct", + output_vector_size=1024, + ), + _rule( + "meta-llama/Llama-3.3-70B-Instruct-Turbo", + "reviewed for the LIT-5968 backfill against https://docs.together.ai/docs/function-calling", + **_TOOLS, + ), + _rule( + "meta-llama/Llama-Guard-4-12B", + "moderation classifier with a chat-shaped API; no tools per the LIT-5968 backfill review", + ), + _rule("meta-models/Muse-Glimmer-30B", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), + _rule( + "moonshotai/Kimi-K2.7-Code", + "reviewed for the LIT-5968 backfill against https://www.together.ai/models/kimi-k2-7-code", + **_TOOLS, + supports_vision=True, + ), + _rule( + "moonshotai/Kimi-K3", + "reviewed for the LIT-5968 backfill against https://www.together.ai/models/kimi-k3", + **_TOOLS, + supports_reasoning=True, + supports_vision=True, + ), + _rule( + "nvidia/nemotron-3-ultra-550b-a55b", + "reviewed for the LIT-5968 backfill against https://www.together.ai/models/nemotron-3-ultra", + **_TOOLS, + supports_reasoning=True, + ), + _rule( + "openai/gpt-oss-120b", + "reviewed for the LIT-5968 backfill against https://www.together.ai/models/gpt-oss-120b", + **_TOOLS, + supports_reasoning=True, + ), + _rule( + "openai/gpt-oss-20b", + "reviewed for the LIT-5968 backfill against https://www.together.ai/models/gpt-oss-20b", + **_TOOLS, + ), + _rule("pearl-ai/gemma-4-31b-it", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), + _rule( + "thinkingmachines/Inkling", + "reviewed for the LIT-5968 backfill against https://www.together.ai/models/inkling", + **_TOOLS, + ), + _rule("thinkingmachines/Inkling-Small", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), + _rule( + "zai-org/GLM-5.2", + "reviewed for the LIT-5968 backfill against https://www.together.ai/models/glm-5-2", + **_TOOLS, + supports_reasoning=True, + ), +) + +RULES_BY_ID: Final = MappingProxyType({rule.model_id: rule for rule in CAPABILITY_RULES}) + + +@dataclass(frozen=True, slots=True) +class DeprecationDoc: + removal_dates: Mapping[str, str] + redirects: Mapping[str, str] + + +_REDIRECT_ROW: Final = re.compile(r"^\|\s*`([^`]+)`\s*\|\s*`([^`]+)`\s*\|") +_REMOVAL_ROW: Final = re.compile(r"^\|\s*(\d{4}-\d{2}-\d{2})\s*\|\s*`([^`]+)`\s*\|") + + +def _section(markdown: str, heading: str) -> str: + level: Final = heading.split(" ", 1)[0] + start: Final = markdown.find(f"\n{heading}\n") + if start < 0: + return "" + body: Final = markdown[start + 1 + len(heading) :] + next_heading: Final = re.search(rf"^{re.escape(level)} ", body, flags=re.MULTILINE) + return body[: next_heading.start()] if next_heading else body + + +def parse_deprecations(markdown: str) -> DeprecationDoc: + redirect_rows: Final = tuple( + m.groups() + for m in (_REDIRECT_ROW.match(line) for line in _section(markdown, "## Active model redirects").splitlines()) + if m + ) + inference: Final = _section(_section(markdown, "## Deprecation history"), "### Inference") + removal_rows: Final = tuple(m.groups() for m in (_REMOVAL_ROW.match(line) for line in inference.splitlines()) if m) + if not redirect_rows or not removal_rows: + raise SyncError( + "deprecations doc parsed to zero redirect or removal rows; the table format at " + f"{DEPRECATIONS_URL} changed and the parser needs updating" + ) + removal_dates: Final = {model: date for date, model in reversed(removal_rows)} + return DeprecationDoc( + removal_dates=MappingProxyType(dict(reversed(removal_dates.items()))), + redirects=MappingProxyType({original: target for original, target in redirect_rows}), + ) + + +def per_token(price_per_million: float) -> float: + return float(f"{price_per_million / 1e6:.6g}") + + +def _resolve_name(name: str, universe: frozenset[str]) -> str | None: + if name in universe: + return name + suffix_matches: Final = tuple(candidate for candidate in universe if candidate.endswith(f"/{name}")) + return suffix_matches[0] if len(suffix_matches) == 1 else None + + +def resolve_successor(model_id: str, doc: DeprecationDoc, live_ids: frozenset[str]) -> str | None: + canonical: Final = live_ids | frozenset(doc.removal_dates) + redirects: Final = { + (_resolve_name(raw_source, canonical) or raw_source): (_resolve_name(raw_target, canonical) or raw_target) + for raw_source, raw_target in doc.redirects.items() + } + seen: Final = set() + current = model_id # rebind-ok: walks the redirect chain + while current in redirects and current not in seen: + seen.add(current) + current = redirects[current] # rebind-ok: walks the redirect chain + return current if current != model_id and current in live_ids else None + + +@dataclass(frozen=True, slots=True) +class SyncOutcome: + cost_map: CostMap + added: tuple[str, ...] = () + updated: tuple[str, ...] = () + deprecated: tuple[str, ...] = () + reappeared: tuple[str, ...] = () + warnings: tuple[str, ...] = () + skipped_types: Mapping[str, int] = field(default_factory=dict) + + @property + def has_changes(self) -> bool: + return bool(self.added or self.updated or self.deprecated or self.reappeared) + + +def _api_fields(model: CatalogModel) -> RegistryEntry: + cached: Final = model.pricing.cached_input + return { + "input_cost_per_token": per_token(model.pricing.input), + "output_cost_per_token": per_token(model.pricing.output), + **({"cache_read_input_token_cost": per_token(cached), "supports_prompt_caching": True} if cached else {}), + **({"max_input_tokens": model.context_length} if model.context_length is not None else {}), + } + + +def _new_entry(model: CatalogModel, mode: str) -> RegistryEntry: + rule: Final = RULES_BY_ID.get(model.id) + length_fields: Final = ( + {} + if model.context_length is None + else {"max_input_tokens": model.context_length, "max_tokens": model.context_length} + | ({"max_output_tokens": model.context_length} if mode == "chat" else {}) + ) + merged: Final = { + **_api_fields(model), + **length_fields, + "litellm_provider": PROVIDER, + "mode": mode, + "source": SOURCE_URL, + **(dict(rule.fields) if rule else {}), + } + return dict(sorted(merged.items())) + + +def _updated_entry(entry: RegistryEntry, model: CatalogModel) -> tuple[RegistryEntry, tuple[str, ...]]: + rule: Final = RULES_BY_ID.get(model.id) + desired: Final = {**_api_fields(model), **(dict(rule.fields) if rule else {})} + dropped: Final = () if model.pricing.cached_input else ("cache_read_input_token_cost",) + changes: Final = tuple( + f"{name}: {entry.get(name)!r} -> {value!r}" for name, value in desired.items() if entry.get(name) != value + ) + tuple( + f"{name}: {entry[name]!r} removed (no longer in the catalog pricing)" for name in dropped if name in entry + ) + merged: Final = {name: value for name, value in {**entry, **desired}.items() if name not in dropped} + return dict(sorted(merged.items())), changes + + +def _with_new_keys_in_block(original: CostMap, result: CostMap, new_keys: Sequence[str]) -> CostMap: + provider_keys: Final = tuple(key for key in original if key.startswith(PREFIX)) + if not new_keys or not provider_keys: + return result + block_end: Final = provider_keys[-1] + return { + key: value + for existing in original + for key, value in ( + (existing, result[existing]), + *((new, result[new]) for new in sorted(new_keys) if existing == block_end), + ) + } + + +def compute_sync(cost_map: CostMap, catalog: Sequence[CatalogModel], doc: DeprecationDoc) -> SyncOutcome: + live_ids: Final = frozenset(model.id for model in catalog) + token_models: Final = {model.id: model for model in catalog if model.type in TYPE_TO_MODE} + skipped: Final = { + model.type: sum(1 for m in catalog if m.type == model.type) + for model in catalog + if model.type not in TYPE_TO_MODE + } + registry_ids: Final = {key.removeprefix(PREFIX): key for key in cost_map if key.startswith(PREFIX)} + + added: Final[list[str]] = [] + updated: Final[list[str]] = [] + deprecated: Final[list[str]] = [] + reappeared: Final[list[str]] = [] + warnings: Final[list[str]] = [] + result: Final[CostMap] = dict(cost_map) + + for model_id, model in sorted(token_models.items()): + mode: Final = TYPE_TO_MODE[model.type] + key: Final = f"{PREFIX}{model_id}" + if model_id in doc.removal_dates: + warnings.append( + f"`{key}` is listed as removed on {doc.removal_dates[model_id]} in the docs but the serverless " + "catalog still serves it; availability kept from the API" + ) + entry = result.get(key) + if not isinstance(entry, dict): + result[key] = _new_entry(model, mode) + added.append(key) + if model.type == "chat" and model_id not in RULES_BY_ID: + warnings.append( + f"`{key}` added without a capability rule; review its tools/vision/reasoning support and add one" + ) + continue + if entry.get("mode") != mode: + warnings.append( + f"`{key}` has curated mode {entry.get('mode')!r} but the catalog maps to {mode!r}; left unchanged" + ) + new_entry, changes = _updated_entry(entry, model) + if "deprecation_date" in new_entry: + new_entry.pop("deprecation_date") + reappeared.append(key) + if changes: + updated.append(f"{key}: " + "; ".join(changes)) + if changes or key in reappeared: + result[key] = new_entry + + for model_id, key in sorted(registry_ids.items()): + if model_id in token_models: + continue + entry = result.get(key) + if not isinstance(entry, dict): + continue + removal_date: Final = doc.removal_dates.get(model_id) + successor: Final = resolve_successor(model_id, doc, live_ids) + metadata = entry.get("metadata") + curated_successor: Final = metadata.get("successor") if isinstance(metadata, dict) else None + new_entry = dict(entry) + if removal_date is not None and entry.get("deprecation_date") != removal_date: + if "deprecation_date" in entry: + warnings.append( + f"`{key}` has curated deprecation_date {entry.get('deprecation_date')!r} but the docs list " + f"{removal_date!r}; left unchanged" + ) + else: + new_entry["deprecation_date"] = removal_date + if removal_date is None and "deprecation_date" not in entry: + warnings.append( + f"`{key}` is absent from the serverless catalog with no removal date in the docs; " + "needs a human deprecation call" + ) + if successor is not None: + desired_successor: Final = f"{PREFIX}{successor}" + if curated_successor is None: + new_entry["metadata"] = dict( + sorted({**(metadata if isinstance(metadata, dict) else {}), "successor": desired_successor}.items()) + ) + elif curated_successor != desired_successor: + warnings.append( + f"`{key}` has curated successor {curated_successor!r} but the docs redirects resolve to " + f"{desired_successor!r}; left unchanged" + ) + if new_entry != entry: + result[key] = dict(sorted(new_entry.items())) + deprecated.append(f"{key}: " + ", ".join(sorted(set(new_entry) - set(entry)) or ["updated"])) + + return SyncOutcome( + cost_map=_with_new_keys_in_block(cost_map, result, tuple(added)), + added=tuple(added), + updated=tuple(updated), + deprecated=tuple(deprecated), + reappeared=tuple(reappeared), + warnings=tuple(warnings), + skipped_types=MappingProxyType(skipped), + ) + + +def _section_block(title: str, lines: Sequence[str], backtick: bool) -> str: + bullets: Final = "\n".join(f"- `{line}`" if backtick else f"- {line}" for line in lines) or "- none" + return f"### {title} ({len(lines)})\n{bullets}\n" + + +def render_pr_body(outcome: SyncOutcome) -> str: + skipped: Final = ", ".join(f"{kind} ({count})" for kind, count in sorted(outcome.skipped_types.items())) or "none" + return ( + "Automated daily sync of the together_ai entries in model_prices_and_context_window.json against " + f"`GET {MODELS_URL}` and {DEPRECATIONS_URL} by scripts/sync_together_ai_models.py.\n" + "\n" + f"{_section_block('Added', outcome.added, backtick=True)}" + "\n" + f"{_section_block('Updated', outcome.updated, backtick=True)}" + "\n" + f"{_section_block('Marked deprecated', outcome.deprecated, backtick=True)}" + "\n" + f"{_section_block('Returned to the catalog', outcome.reappeared, backtick=True)}" + "\n" + f"{_section_block('Warnings needing a human call', outcome.warnings, backtick=False)}" + "\n" + f"Catalog model types outside the sync's token-pricing scope, skipped: {skipped}\n" + ) + + +def render_summary(outcome: SyncOutcome) -> str: + return ( + f"added={len(outcome.added)} updated={len(outcome.updated)} deprecated={len(outcome.deprecated)} " + f"reappeared={len(outcome.reappeared)} warnings={len(outcome.warnings)}" + ) + + +def load_catalog(raw: bytes) -> list[CatalogModel]: + parsed: Final = json.loads(raw) + entries: Final = parsed.get("data") if isinstance(parsed, dict) else parsed + try: + catalog: Final = CATALOG_ADAPTER.validate_python(entries) + except ValidationError as error: + raise SyncError(f"the catalog response no longer matches the expected shape: {error}") from error + if not any(model.type in TYPE_TO_MODE for model in catalog): + raise SyncError( + "the catalog response contains no token-priced models; refusing to mark the whole registry deprecated" + ) + return catalog + + +def _fetch(url: str, headers: Mapping[str, str]) -> bytes: + response: Final = httpx.get(url, headers=dict(headers), timeout=30, follow_redirects=True) + if response.status_code != 200: + raise SyncError(f"GET {url} returned {response.status_code}") + return response.content + + +def _serialize(cost_map: CostMap) -> str: + return json.dumps(cost_map, indent=4, ensure_ascii=False) + "\n" + + +def main(argv: Sequence[str]) -> int: + parser: Final = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--write", action="store_true", help="apply the sync to the cost map files (default: dry run)") + parser.add_argument("--models-json", type=Path, help="recorded catalog response to use instead of the live API") + parser.add_argument( + "--deprecations-md", type=Path, help="recorded deprecations doc to use instead of the live docs" + ) + parser.add_argument("--pr-body-file", type=Path, help="write the generated PR body to this path") + parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parent.parent) + args: Final = parser.parse_args(argv) + + if args.models_json is not None: + catalog_raw: Final = args.models_json.read_bytes() + else: + api_key: Final = os.environ.get("TOGETHER_API_KEY") + if not api_key: + raise SyncError("TOGETHER_API_KEY is not set and --models-json was not given") + catalog_raw = _fetch(MODELS_URL, {"Authorization": f"Bearer {api_key}"}) # rebind-ok: branch-dependent source + catalog: Final = load_catalog(catalog_raw) + markdown: Final = ( + args.deprecations_md.read_text() if args.deprecations_md is not None else _fetch(DEPRECATIONS_URL, {}).decode() + ) + doc: Final = parse_deprecations(markdown) + + cost_map_path: Final = args.repo_root / COST_MAP_RELPATHS[0] + cost_map: Final = json.loads(cost_map_path.read_text()) + outcome: Final = compute_sync(cost_map, catalog, doc) + body: Final = render_pr_body(outcome) + + if args.pr_body_file is not None: + args.pr_body_file.write_text(body) + if args.write and outcome.has_changes: + for relpath in COST_MAP_RELPATHS: + (args.repo_root / relpath).write_text(_serialize(outcome.cost_map)) + print(render_summary(outcome)) + print() + print(body) + if not args.write: + print("dry run: no files were touched") + elif not outcome.has_changes: + print("registry already in sync: no files were touched") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main(sys.argv[1:])) + except SyncError as error: + print(f"SYNC FAILED: {error}", file=sys.stderr) + raise SystemExit(1) from error diff --git a/tests/test_litellm/fixtures/together_ai_sync/deprecations.md b/tests/test_litellm/fixtures/together_ai_sync/deprecations.md new file mode 100644 index 00000000000..b75e0825cee --- /dev/null +++ b/tests/test_litellm/fixtures/together_ai_sync/deprecations.md @@ -0,0 +1,442 @@ +> ## Documentation Index +> Fetch the complete documentation index at: https://docs.together.ai/llms.txt +> Use this file to discover all available pages before exploring further. + +# Deprecations + +> Together AI's model lifecycle policy, including upgrades, redirects, and deprecation schedules. + +Together AI regularly updates the platform with new open-source models. This page describes the model lifecycle policy and lists active redirects and scheduled deprecations. + +## Model lifecycle policy + +Together AI follows a structured approach to introducing new models, upgrading existing models, and deprecating older versions, so you can rely on predictable behavior. + +### Model upgrades (redirects) + +An **upgrade** is a model release that is materially the same model lineage with targeted improvements and no fundamental changes to how developers use or reason about it. + +A model qualifies as an upgrade when **one or more** of the following are true (and none of the "new model" criteria apply): + +* Same modality and task profile (e.g., instruct → instruct, reasoning → reasoning). +* Same architecture family (e.g., DeepSeek-V3 → DeepSeek-V3-0324). +* Post-training or fine-tuning improvements, bug fixes, safety tuning, or small data refresh. +* Behavior is strongly compatible (prompting patterns and evals are similar). +* Pricing change is none or small (≤10% increase). + +**Outcome:** The current endpoint redirects to the upgraded version after a **3-day notice**. The old version remains available via dedicated endpoints. + +### New models (no redirect) + +A **new model** is a release with materially different capabilities, costs, or operating characteristics, so a silent redirect would be misleading. + +Any of the following triggers classification as a new model: + +* Modality shift (e.g., reasoning-only ↔ instruct/hybrid, text → multimodal). +* Architecture shift (e.g., Qwen3 → Qwen3-Next, Llama 3 → Llama 4). +* Large behavior shift (prompting patterns, output style, or verbosity materially different). +* Experimental flag by provider (e.g., DeepSeek-V3-Exp). +* Large price change (>10% increase or pricing structure change). +* Benchmark deltas that meaningfully change task positioning. +* Safety policy or system prompt changes that noticeably affect outputs. + +**Outcome:** No automatic redirect. Together AI announces the new model and deprecates the old one on a **2-week timeline** (both are available during this window). You must explicitly switch model IDs. + +## Active model redirects + +The following models are redirected to newer versions. Requests to the original model ID are automatically routed to the upgraded version: + +| Original model | Redirects to | Notes | +| :----------------------------------- | :---------------------------------------- | :---------------------------------------- | +| `mistralai/Mistral-7B-Instruct-v0.3` | `mistralai/Ministral-3-14B-Instruct-2512` | Same lineage, upgraded version | +| `Kimi-K2` | `Kimi-K2-0905` | Same architecture, improved post-training | +| `DeepSeek-V3` | `DeepSeek-V3.1` | Same architecture, targeted improvements | +| `DeepSeek-V3-0324` | `DeepSeek-V3.1` | Same architecture, targeted improvements | +| `DeepSeek-R1` | `DeepSeek-R1-0528` | Same architecture, targeted improvements | + + + If you need to use the original model version, you can always deploy it as a [dedicated endpoint](/docs/dedicated-endpoints). + + +## Deprecation policy + +| Model type | Deprecation notice | Notes | +| :--------------------------- | :---------------------------------- | :------------------------------------------------------- | +| Preview model | \<24 hours of notice, after 30 days | Clearly marked in docs and playground with "Preview" tag | +| Serverless endpoint | 2 or 3 weeks\* | | +| On-demand dedicated endpoint | 2 or 3 weeks\* | | + +\*Depends on usage and whether a newer version of the model is available. + +* If you use a model scheduled for deprecation, you receive an email notification. +* All changes appear on this page. +* Each deprecated model has a specified removal date. +* After the removal date, the model is no longer available via its serverless endpoint, but migration options are described below. + +## Migration options + +When a model is deprecated on the serverless platform, you have three options: + +1. **On-demand dedicated endpoint** (if supported): + * Reserved solely for you. You choose the underlying hardware. + * Charged on a price-per-minute basis. + * Endpoints can be dynamically spun up and down. +2. **Monthly reserved dedicated endpoint:** + * Reserved solely for you. + * Charged on a month-by-month basis. + * Can be requested via this [form](https://together.ai/monthly-reserved). +3. **Migrate to a newer serverless model:** + * Switch to an updated model on the serverless platform. + +## Migration steps + +1. Review the deprecation table below to find your current model. +2. Check if on-demand dedicated endpoints are supported for your model. +3. Decide on your preferred migration option. +4. If you choose a new serverless model, test your application thoroughly before migrating. +5. Update your API calls to use the new model or dedicated endpoint. + +## Deprecation history + +### Inference + +The table below lists all models removed from serverless inference, most recent first. + +| Removal date | Model | Supported by on-demand dedicated endpoints | +| :-------------------------- | :-------------------------------------------------- | :----------------------------------------- | +| 2026-08-21 | `deepcogito/cogito-v2-1-671b` | No | +| 2026-08-04 | `google/gemma-3n-E4B-it` | No | +| 2026-07-10 | `Qwen/Qwen3-235B-A22B-Instruct-2507-tput` | Yes | +| 2026-07-10 | `meta-llama/Meta-Llama-3-8B-Instruct-Lite` | No | +| 2026-07-10 | `zai-org/GLM-5.1` | Yes | +| 2026-06-29 | `Qwen/Qwen3.5-397B-A17B` | Yes | +| 2026-06-22 | `zai-org/GLM-5` | No | +| 2026-06-11 | `mistralai/Voxtral-Mini-3B-2507` | No | +| 2026-06-04 | `Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8` | Yes | +| 2026-05-27 | `black-forest-labs/FLUX.1-krea-dev` | No | +| 2026-05-21 | `moonshotai/Kimi-K2.5` | No | +| 2026-05-14 | `deepseek-ai/DeepSeek-R1` | No | +| 2026-05-14 | `deepseek-ai/DeepSeek-V3.1` | Yes | +| 2026-05-14 | `Qwen/Qwen3-Coder-Next-FP8` | Yes | +| 2026-04-16 | `Qwen/Qwen3-VL-8B-Instruct` | Yes | +| 2026-04-16 | `Qwen/Qwen3-235B-A22B-Thinking-2507` | Yes | +| 2026-04-16 | `mistralai/Mixtral-8x7B-Instruct-v0.1` | Yes | +| 2026-04-03 | `ServiceNow-AI/Apriel-1.5-15b-Thinker` | No | +| 2026-04-03 | `ServiceNow-AI/Apriel-1.6-15b-Thinker` | No | +| 2026-04-02 | `zai-org/GLM-4.5-Air-FP8` | No | +| 2026-04-02 | `zai-org/GLM-4.7` | No | +| 2026-04-02 | `mistralai/Mistral-Small-24B-Instruct-2501` | No | +| 2026-04-02 | `Qwen/Qwen3-Next-80B-A3B-Instruct` | Yes | +| 2026-03-31 | `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8` | Yes | +| 2026-03-06 | `mixedbread-ai/Mxbai-Rerank-Large-V2` | No | +| 2026-03-06 | `meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo` | Yes | +| 2026-03-06 | `Qwen/Qwen3-235B-A22B-Thinking-2507` | Yes | +| 2026-03-06 | `moonshotai/Kimi-K2-Thinking` | No | +| 2026-03-06 | `moonshotai/Kimi-K2-Instruct-0905` | No | +| 2026-03-06 | `meta-llama/Llama-3.2-3B-Instruct-Turbo` | No | +| 2026-02-25 | `black-forest-labs/FLUX.1-dev` | No | +| 2026-02-25 | `black-forest-labs/FLUX.1-dev-lora` | No | +| 2026-02-25 | `black-forest-labs/FLUX.1-Kontext-dev` | No | +| 2026-02-25 | `Qwen/Qwen3-VL-32B-Instruct` | No | +| 2026-02-25 | `meta-llama/Llama-3.2-3B-Instruct-Turbo-Classifier` | No | +| 2026-02-25 | `mistralai/Ministral-3-14B-Instruct` | No | +| 2026-02-25 | `Qwen/Qwen3-Next-80B-A3B-Thinking` | No | +| 2026-02-25 | `Alibaba-NLP/gte-modernbert-base` | No | +| 2026-02-25 | `BAAI/bge-base-en-v1.5-vllm` | No | +| 2026-02-25 | `meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` | No | +| 2026-02-25 | `meta-llama/Llama-Guard-3-11B-Vision-Turbo` | No | +| 2026-02-25 | `meta-llama/LlamaGuard-2-8b` | No | +| 2026-02-25 | `marin-community/Marin-8B-Instruct` | No | +| 2026-02-25 | `nvidia/Nvidia-Nemotron-Nano-9B-v2` | No | +| 2026-02-06 | `togethercomputer/m2-bert-80M-32k-retrieval` | No | +| 2026-02-06 | `Salesforce/Llama-Rank-V1` | No | +| 2026-02-06 | `togethercomputer/Refuel-Llm-V2` | No | +| 2026-02-06 | `togethercomputer/Refuel-Llm-V2-Small` | No | +| 2026-02-06 | `Qwen/Qwen3-235B-A22B-fp8-tput` | No | +| 2026-02-06 | `qwen-qwen2-5-14b-instruct-lora` | No | +| 2026-02-06 | `meta-llama/Llama-4-Scout-17B-16E-Instruct` | Yes | +| 2026-02-06 | `Qwen/Qwen2.5-72B-Instruct-Turbo` | No | +| 2026-02-06 | `meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo` | No | +| 2026-02-06 | `BAAI/bge-large-en-v1.5` | No | +| 2026-02-03 | `deepseek-ai/DeepSeek-R1-0528-tput` | No | +| 2026-01-05 | `Qwen/Qwen2.5-VL-72B-Instruct` | No | +| 2025-12-23 | `deepseek-ai/DeepSeek-R1-Distill-Llama-70B` | No | +| 2025-12-23 | `meta-llama/Meta-Llama-3-70B-Instruct-Turbo` | No | +| 2025-12-23 | `black-forest-labs/FLUX.1-schnell-free` | No | +| 2025-12-23 | `meta-llama/Meta-Llama-Guard-3-8B` | No | +| 2025-11-19 | `deepcogito/cogito-v2-preview-deepseek-671b` | No | +| 2025-07-25 | `arcee-ai/caller` | No | +| 2025-07-25 | `arcee-ai/arcee-blitz` | No | +| 2025-07-25 | `arcee-ai/virtuoso-medium-v2` | No | +| 2025-11-17 | `arcee-ai/virtuoso-large` | No | +| 2025-11-17 | `arcee-ai/maestro-reasoning` | No | +| 2025-11-17 | `arcee_ai/arcee-spotlight` | No | +| 2025-11-17 | `arcee-ai/coder-large` | No | +| 2025-11-13 | `deepseek-ai/DeepSeek-R1-Distill-Qwen-14B` | No | +| 2025-11-13 | `mistralai/Mistral-7B-Instruct-v0.1` | No | +| 2025-11-13 | `Qwen/Qwen2.5-Coder-32B-Instruct` | No | +| 2025-11-13 | `Qwen/QwQ-32B` | No | +| 2025-11-13 | `deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free` | No | +| 2025-11-13 | `meta-llama/Llama-3.3-70B-Instruct-Turbo-Free` | No | +| 2025-08-28 | `Qwen/Qwen2-VL-72B-Instruct` | No | +| 2025-08-28 | `nvidia/Llama-3.1-Nemotron-70B-Instruct-HF` | No | +| 2025-08-28 | `perplexity-ai/r1-1776` | No | +| 2025-08-28 | `meta-llama/Meta-Llama-3-8B-Instruct` | No | +| 2025-08-28 | `google/gemma-2-27b-it` | No | +| 2025-08-28 | `Qwen/Qwen2-72B-Instruct` | No | +| 2025-08-28 | `meta-llama/Llama-Vision-Free` | No | +| 2025-08-28 | `Qwen/Qwen2.5-14B` | No | +| 2025-08-28 | `meta-llama-llama-3-3-70b-instruct-lora` | No | +| 2025-08-28 | `meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo` | No | +| 2025-08-28 | `NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO` | No | +| 2025-08-28 | `deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B` | No | +| 2025-08-28 | `black-forest-labs/FLUX.1-depth` | No | +| 2025-08-28 | `black-forest-labs/FLUX.1-redux` | No | +| 2025-08-28 | `meta-llama/Llama-3-8b-chat-hf` | No | +| 2025-08-28 | `black-forest-labs/FLUX.1-canny` | No | +| 2025-08-28 | `meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo` | No | +| 2025-06-13 | `gryphe-mythomax-l2-13b` | No | +| 2025-06-13 | `mistralai-mixtral-8x22b-instruct-v0-1` | No | +| 2025-06-13 | `mistralai-mixtral-8x7b-v0-1` | No | +| 2025-06-13 | `togethercomputer-m2-bert-80m-2k-retrieval` | No | +| 2025-06-13 | `togethercomputer-m2-bert-80m-8k-retrieval` | No | +| 2025-06-13 | `whereisai-uae-large-v1` | No | +| 2025-06-13 | `google-gemma-2-9b-it` | No | +| 2025-06-13 | `google-gemma-2b-it` | No | +| 2025-06-13 | `gryphe-mythomax-l2-13b-lite` | No | +| 2025-05-16 | `meta-llama-llama-3-2-3b-instruct-turbo-lora` | No | +| 2025-05-16 | `meta-llama-meta-llama-3-8b-instruct-turbo` | No | +| 2025-04-24 | `meta-llama/Llama-2-13b-chat-hf` | No | +| 2025-04-24 | `meta-llama-meta-llama-3-70b-instruct-turbo` | No | +| 2025-04-24 | `meta-llama-meta-llama-3-1-8b-instruct-turbo-lora` | No | +| 2025-04-24 | `meta-llama-meta-llama-3-1-70b-instruct-turbo-lora` | No | +| 2025-04-24 | `meta-llama-llama-3-2-1b-instruct-lora` | No | +| 2025-04-24 | `microsoft-wizardlm-2-8x22b` | No | +| 2025-04-24 | `upstage-solar-10-7b-instruct-v1` | No | +| 2025-04-14 | `stabilityai/stable-diffusion-xl-base-1.0` | No | +| 2025-04-04 | `meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo-lora` | No | +| 2025-03-27 | `mistralai/Mistral-7B-v0.1` | No | +| 2025-03-25 | `Qwen/QwQ-32B-Preview` | No | +| 2025-03-13 | `databricks-dbrx-instruct` | No | +| 2025-03-11 | `meta-llama/Meta-Llama-3-70B-Instruct-Lite` | No | +| 2025-03-08 | `Meta-Llama/Llama-Guard-7b` | No | +| 2025-02-06 | `sentence-transformers/msmarco-bert-base-dot-v5` | No | +| 2025-02-06 | `bert-base-uncased` | No | +| 2024-10-29 | `Qwen/Qwen1.5-72B-Chat` | No | +| 2024-10-29 | `Qwen/Qwen1.5-110B-Chat` | No | +| 2024-10-07 | `NousResearch/Nous-Hermes-2-Yi-34B` | No | +| 2024-10-07 | `NousResearch/Hermes-3-Llama-3.1-405B-Turbo` | No | +| 2024-08-22 | `NousResearch/Nous-Hermes-2-Mistral-7B-DPO` | No | +| 2024-08-22 | `SG161222/Realistic_Vision_V3.0_VAE` | No | +| 2024-08-22 | `meta-llama/Llama-2-70b-chat-hf` | No | +| 2024-08-22 | `mistralai/Mixtral-8x22B` | No | +| 2024-08-22 | `Phind/Phind-CodeLlama-34B-v2` | No | +| 2024-08-22 | `meta-llama/Meta-Llama-3-70B` | No | +| 2024-08-22 | `teknium/OpenHermes-2p5-Mistral-7B` | No | +| 2024-08-22 | `openchat/openchat-3.5-1210` | No | +| 2024-08-22 | `WizardLM/WizardCoder-Python-34B-V1.0` | No | +| 2024-08-22 | `NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT` | No | +| 2024-08-22 | `NousResearch/Nous-Hermes-Llama2-13b` | No | +| 2024-08-22 | `zero-one-ai/Yi-34B-Chat` | No | +| 2024-08-22 | `codellama/CodeLlama-34b-Instruct-hf` | No | +| 2024-08-22 | `codellama/CodeLlama-34b-Python-hf` | No | +| 2024-08-22 | `teknium/OpenHermes-2-Mistral-7B` | No | +| 2024-08-22 | `Qwen/Qwen1.5-14B-Chat` | No | +| 2024-08-22 | `stabilityai/stable-diffusion-2-1` | No | +| 2024-08-22 | `meta-llama/Llama-3-8b-hf` | No | +| 2024-08-22 | `prompthero/openjourney` | No | +| 2024-08-22 | `runwayml/stable-diffusion-v1-5` | No | +| 2024-08-22 | `wavymulder/Analog-Diffusion` | No | +| 2024-08-22 | `Snowflake/snowflake-arctic-instruct` | No | +| 2024-08-22 | `deepseek-ai/deepseek-coder-33b-instruct` | No | +| 2024-08-22 | `Qwen/Qwen1.5-7B-Chat` | No | +| 2024-08-22 | `Qwen/Qwen1.5-32B-Chat` | No | +| 2024-08-22 | `cognitivecomputations/dolphin-2.5-mixtral-8x7b` | No | +| 2024-08-22 | `garage-bAInd/Platypus2-70B-instruct` | No | +| 2024-08-22 | `google/gemma-7b-it` | No | +| 2024-08-22 | `meta-llama/Llama-2-7b-chat-hf` | No | +| 2024-08-22 | `Qwen/Qwen1.5-32B` | No | +| 2024-08-22 | `Open-Orca/Mistral-7B-OpenOrca` | No | +| 2024-08-22 | `codellama/CodeLlama-13b-Instruct-hf` | No | +| 2024-08-22 | `NousResearch/Nous-Capybara-7B-V1p9` | No | +| 2024-08-22 | `lmsys/vicuna-13b-v1.5` | No | +| 2024-08-22 | `Undi95/ReMM-SLERP-L2-13B` | No | +| 2024-08-22 | `Undi95/Toppy-M-7B` | No | +| 2024-08-22 | `meta-llama/Llama-2-13b-hf` | No | +| 2024-08-22 | `codellama/CodeLlama-70b-Instruct-hf` | No | +| 2024-08-22 | `snorkelai/Snorkel-Mistral-PairRM-DPO` | No | +| 2024-08-22 | `togethercomputer/LLaMA-2-7B-32K-Instruct` | No | +| 2024-08-22 | `Austism/chronos-hermes-13b` | No | +| 2024-08-22 | `Qwen/Qwen1.5-72B` | No | +| 2024-08-22 | `zero-one-ai/Yi-34B` | No | +| 2024-08-22 | `codellama/CodeLlama-7b-Instruct-hf` | No | +| 2024-08-22 | `togethercomputer/evo-1-131k-base` | No | +| 2024-08-22 | `codellama/CodeLlama-70b-hf` | No | +| 2024-08-22 | `WizardLM/WizardLM-13B-V1.2` | No | +| 2024-08-22 | `meta-llama/Llama-2-7b-hf` | No | +| 2024-08-22 | `google/gemma-7b` | No | +| 2024-08-22 | `Qwen/Qwen1.5-1.8B-Chat` | No | +| 2024-08-22 | `Qwen/Qwen1.5-4B-Chat` | No | +| 2024-08-22 | `lmsys/vicuna-7b-v1.5` | No | +| 2024-08-22 | `zero-one-ai/Yi-6B` | No | +| 2024-08-22 | `Nexusflow/NexusRaven-V2-13B` | No | +| 2024-08-22 | `google/gemma-2b` | No | +| 2024-08-22 | `Qwen/Qwen1.5-7B` | No | +| 2024-08-22 | `NousResearch/Nous-Hermes-llama-2-7b` | No | +| 2024-08-22 | `togethercomputer/alpaca-7b` | No | +| 2024-08-22 | `Qwen/Qwen1.5-14B` | No | +| 2024-08-22 | `codellama/CodeLlama-70b-Python-hf` | No | +| 2024-08-22 | `Qwen/Qwen1.5-4B` | No | +| 2024-08-22 | `togethercomputer/StripedHyena-Hessian-7B` | No | +| 2024-08-22 | `allenai/OLMo-7B-Instruct` | No | +| 2024-08-22 | `togethercomputer/RedPajama-INCITE-7B-Instruct` | No | +| 2024-08-22 | `togethercomputer/LLaMA-2-7B-32K` | No | +| 2024-08-22 | `togethercomputer/RedPajama-INCITE-7B-Base` | No | +| 2024-08-22 | `Qwen/Qwen1.5-0.5B-Chat` | No | +| 2024-08-22 | `microsoft/phi-2` | No | +| 2024-08-22 | `Qwen/Qwen1.5-0.5B` | No | +| 2024-08-22 | `togethercomputer/RedPajama-INCITE-7B-Chat` | No | +| 2024-08-22 | `togethercomputer/RedPajama-INCITE-Chat-3B-v1` | No | +| 2024-08-22 | `togethercomputer/GPT-JT-Moderation-6B` | No | +| 2024-08-22 | `Qwen/Qwen1.5-1.8B` | No | +| 2024-08-22 | `togethercomputer/RedPajama-INCITE-Instruct-3B-v1` | No | +| 2024-08-22 | `togethercomputer/RedPajama-INCITE-Base-3B-v1` | No | +| 2024-08-22 | `WhereIsAI/UAE-Large-V1` | No | +| 2024-08-22 | `allenai/OLMo-7B` | No | +| 2024-08-22 | `togethercomputer/evo-1-8k-base` | No | +| 2024-08-22 | `WizardLM/WizardCoder-15B-V1.0` | No | +| 2024-08-22 | `codellama/CodeLlama-13b-Python-hf` | No | +| 2024-08-22 | `allenai-olmo-7b-twin-2t` | No | +| 2024-08-22 | `sentence-transformers/msmarco-bert-base-dot-v5` | No | +| 2024-08-22 | `codellama/CodeLlama-7b-Python-hf` | No | +| 2024-08-22 | `hazyresearch/M2-BERT-2k-Retrieval-Encoder-V1` | No | +| 2024-08-22 | `bert-base-uncased` | No | +| 2024-08-22 | `mistralai/Mistral-7B-Instruct-v0.1-json` | No | +| 2024-08-22 | `mistralai/Mistral-7B-Instruct-v0.1-tools` | No | +| 2024-08-22 | `togethercomputer-codellama-34b-instruct-json` | No | +| 2024-08-22 | `togethercomputer-codellama-34b-instruct-tools` | No | +| **Notes on model support:** | | | + +* The support column reflects the current [supported models](/docs/dedicated-endpoints/models) catalog for dedicated model inference and is updated automatically as the catalog changes. +* Models marked "Yes" can be deployed as on-demand dedicated endpoints, either under the listed ID or as the underlying base model of a serving variant (for example, a deprecated `-FP8` or `-Turbo` ID). +* Models marked "No" are not available as on-demand endpoints and require migration to a different model or a monthly reserved dedicated endpoint. + +### Fine-tuning + +The table below lists all models removed from the fine-tuning service, most recent first. These models can no longer be used as a base model for a fine-tuning job. Where a close equivalent exists, the suggested replacement is listed. A blank cell means there is no direct equivalent. See [Supported models](/docs/fine-tuning/supported-models) for the full list of models available today. + +| Removal date | Model | Suggested replacement | +| :----------- | :------------------------------------------------------ | :------------------------------------------------ | +| 2026-07-29 | `nvidia/NVIDIA-Nemotron-Nano-9B-v2` | `Qwen/Qwen3.5-9B` | +| 2026-07-29 | `Qwen/Qwen3-Next-80B-A3B-Instruct` | `Qwen/Qwen3.5-122B-A10B` | +| 2026-07-29 | `Qwen/Qwen3-Next-80B-A3B-Thinking` | `Qwen/Qwen3.5-122B-A10B` | +| 2026-07-29 | `Qwen/Qwen3-0.6B` | `Qwen/Qwen3.5-0.8B` | +| 2026-07-29 | `Qwen/Qwen3-0.6B-Base` | `Qwen/Qwen3.5-0.8B` | +| 2026-07-29 | `Qwen/Qwen3-1.7B` | `Qwen/Qwen3.5-2B` | +| 2026-07-29 | `Qwen/Qwen3-1.7B-Base` | `Qwen/Qwen3.5-2B` | +| 2026-07-29 | `Qwen/Qwen3-4B` | `Qwen/Qwen3.5-4B` | +| 2026-07-29 | `Qwen/Qwen3-4B-Base` | `Qwen/Qwen3.5-4B` | +| 2026-07-29 | `Qwen/Qwen3-8B` | `Qwen/Qwen3.5-9B` | +| 2026-07-29 | `Qwen/Qwen3-8B-Base` | `Qwen/Qwen3.5-9B` | +| 2026-07-29 | `Qwen/Qwen3-14B` | `Qwen/Qwen3.5-27B` | +| 2026-07-29 | `Qwen/Qwen3-14B-Base` | `Qwen/Qwen3.5-27B` | +| 2026-07-29 | `Qwen/Qwen3-32B` | `Qwen/Qwen3.5-27B` | +| 2026-07-29 | `Qwen/Qwen3-30B-A3B-Base` | `Qwen/Qwen3.6-35B-A3B` | +| 2026-07-29 | `Qwen/Qwen3-30B-A3B` | `Qwen/Qwen3.6-35B-A3B` | +| 2026-07-29 | `Qwen/Qwen3-30B-A3B-Instruct-2507` | `Qwen/Qwen3.6-35B-A3B` | +| 2026-07-29 | `Qwen/Qwen3-235B-A22B` | `Qwen/Qwen3.5-397B-A17B` | +| 2026-07-29 | `Qwen/Qwen3-235B-A22B-Instruct-2507` | `Qwen/Qwen3.5-397B-A17B` | +| 2026-07-29 | `Qwen/Qwen3-Coder-30B-A3B-Instruct` | `Qwen/Qwen3.6-35B-A3B` | +| 2026-07-29 | `Qwen/Qwen3-Coder-480B-A35B-Instruct` | | +| 2026-07-29 | `Qwen/Qwen3-VL-8B-Instruct` | `Qwen/Qwen3.5-9B` | +| 2026-07-29 | `Qwen/Qwen3-VL-32B-Instruct` | | +| 2026-07-29 | `Qwen/Qwen3-VL-30B-A3B-Instruct` | `Qwen/Qwen3.5-4B` | +| 2026-07-29 | `Qwen/Qwen3-VL-235B-A22B-Instruct` | | +| 2026-07-29 | `Qwen/Qwen2.5-72B-Instruct` | `meta-llama/Llama-3.3-70B-Instruct-Reference` | +| 2026-07-29 | `Qwen/Qwen2.5-72B` | `meta-llama/Llama-3.3-70B-Instruct-Reference` | +| 2026-07-29 | `Qwen/Qwen2.5-32B-Instruct` | `Qwen/Qwen3.5-27B` | +| 2026-07-29 | `Qwen/Qwen2.5-32B` | `Qwen/Qwen3.5-27B` | +| 2026-07-29 | `Qwen/Qwen2.5-14B-Instruct` | `Qwen/Qwen3.5-27B` | +| 2026-07-29 | `Qwen/Qwen2.5-14B` | `Qwen/Qwen3.5-27B` | +| 2026-07-29 | `Qwen/Qwen2.5-7B-Instruct` | `Qwen/Qwen3.5-9B` | +| 2026-07-29 | `Qwen/Qwen2.5-7B` | `Qwen/Qwen3.5-9B` | +| 2026-07-29 | `Qwen/Qwen2.5-3B-Instruct` | `Qwen/Qwen3.5-4B` | +| 2026-07-29 | `Qwen/Qwen2.5-3B` | `Qwen/Qwen3.5-4B` | +| 2026-07-29 | `Qwen/Qwen2.5-1.5B-Instruct` | `Qwen/Qwen3.5-2B` | +| 2026-07-29 | `Qwen/Qwen2.5-1.5B` | `Qwen/Qwen3.5-2B` | +| 2026-07-29 | `Qwen/Qwen2-72B-Instruct` | `meta-llama/Llama-3.3-70B-Instruct-Reference` | +| 2026-07-29 | `Qwen/Qwen2-72B` | `meta-llama/Llama-3.3-70B-Instruct-Reference` | +| 2026-07-29 | `Qwen/Qwen2-7B-Instruct` | `Qwen/Qwen3.5-9B` | +| 2026-07-29 | `Qwen/Qwen2-7B` | `Qwen/Qwen3.5-9B` | +| 2026-07-29 | `Qwen/Qwen2-1.5B-Instruct` | `Qwen/Qwen3.5-2B` | +| 2026-07-29 | `Qwen/Qwen2-1.5B` | `Qwen/Qwen3.5-2B` | +| 2026-07-29 | `moonshotai/Kimi-K2.5` | `moonshotai/Kimi-K2.6` | +| 2026-07-29 | `moonshotai/Kimi-K2-Thinking` | `moonshotai/Kimi-K2.6` | +| 2026-07-29 | `moonshotai/Kimi-K2-Instruct-0905` | `moonshotai/Kimi-K2.6` | +| 2026-07-29 | `moonshotai/Kimi-K2-Instruct` | `moonshotai/Kimi-K2.6` | +| 2026-07-29 | `moonshotai/Kimi-K2-Base` | `moonshotai/Kimi-K2.6` | +| 2026-07-29 | `zai-org/GLM-5` | `zai-org/GLM-5.1` | +| 2026-07-29 | `zai-org/GLM-4.7` | `zai-org/GLM-5.1` | +| 2026-07-29 | `zai-org/GLM-4.6` | `zai-org/GLM-5.1` | +| 2026-07-29 | `deepseek-ai/DeepSeek-R1-0528` | `deepseek-ai/DeepSeek-V3.1` | +| 2026-07-29 | `deepseek-ai/DeepSeek-R1` | `deepseek-ai/DeepSeek-V3.1` | +| 2026-07-29 | `deepseek-ai/DeepSeek-V3-0324` | `deepseek-ai/DeepSeek-V3.1` | +| 2026-07-29 | `deepseek-ai/DeepSeek-V3` | `deepseek-ai/DeepSeek-V3.1` | +| 2026-07-29 | `deepseek-ai/DeepSeek-V3.1-Base` | `deepseek-ai/DeepSeek-V3.1` | +| 2026-07-29 | `deepseek-ai/DeepSeek-V3-Base` | `deepseek-ai/DeepSeek-V3.1` | +| 2026-07-29 | `deepseek-ai/DeepSeek-R1-Distill-Llama-70B` | `meta-llama/Llama-3.3-70B-Instruct-Reference` | +| 2026-07-29 | `deepseek-ai/DeepSeek-R1-Distill-Llama-70B-32k` | `meta-llama/Llama-3.3-70B-Instruct-Reference` | +| 2026-07-29 | `deepseek-ai/DeepSeek-R1-Distill-Llama-70B-131k` | `meta-llama/Llama-3.3-70B-Instruct-Reference` | +| 2026-07-29 | `deepseek-ai/DeepSeek-R1-Distill-Qwen-14B` | `Qwen/Qwen3.5-27B` | +| 2026-07-29 | `deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B` | `Qwen/Qwen3.5-2B` | +| 2026-07-29 | `meta-llama/Llama-4-Scout-17B-16E` | `meta-llama/Llama-4-Scout-17B-16E-Instruct` | +| 2026-07-29 | `meta-llama/Llama-4-Maverick-17B-128E` | `meta-llama/Llama-4-Maverick-17B-128E-Instruct` | +| 2026-07-29 | `meta-llama/Llama-3.3-70B-32k-Instruct-Reference` | `meta-llama/Llama-3.3-70B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Llama-3.3-70B-131k-Instruct-Reference` | `meta-llama/Llama-3.3-70B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Llama-3.2-3B-Instruct` | `meta-llama/Meta-Llama-3.1-8B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Llama-3.2-3B` | `meta-llama/Meta-Llama-3.1-8B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Llama-3.2-1B-Instruct` | `meta-llama/Meta-Llama-3.1-8B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Llama-3.2-1B` | `meta-llama/Meta-Llama-3.1-8B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Meta-Llama-3.1-8B-131k-Instruct-Reference` | `meta-llama/Meta-Llama-3.1-8B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Meta-Llama-3.1-8B-Reference` | `meta-llama/Meta-Llama-3.1-8B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Meta-Llama-3.1-8B-131k-Reference` | `meta-llama/Meta-Llama-3.1-8B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Meta-Llama-3.1-70B-Instruct-Reference` | `meta-llama/Llama-3.3-70B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Meta-Llama-3.1-70B-32k-Instruct-Reference` | `meta-llama/Llama-3.3-70B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Meta-Llama-3.1-70B-131k-Instruct-Reference` | `meta-llama/Llama-3.3-70B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Meta-Llama-3.1-70B-Reference` | `meta-llama/Llama-3.3-70B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Meta-Llama-3.1-70B-32k-Reference` | `meta-llama/Llama-3.3-70B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Meta-Llama-3.1-70B-131k-Reference` | `meta-llama/Llama-3.3-70B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Meta-Llama-3.1-405B-Instruct-Reference` | | +| 2026-07-29 | `meta-llama/Meta-Llama-3.1-405B-Reference` | | +| 2026-07-29 | `meta-llama/Meta-Llama-3.1-405B-10k-Instruct-Reference` | | +| 2026-07-29 | `meta-llama/Meta-Llama-3.1-405B-10k-Reference` | | +| 2026-07-29 | `meta-llama/Meta-Llama-3.1-405B-8k-Instruct-Reference` | | +| 2026-07-29 | `meta-llama/Meta-Llama-3.1-405B-8k-Reference` | | +| 2026-07-29 | `meta-llama/Meta-Llama-3-8B-Instruct` | `meta-llama/Meta-Llama-3.1-8B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Meta-Llama-3-8B` | `meta-llama/Meta-Llama-3.1-8B-Instruct-Reference` | +| 2026-07-29 | `meta-llama/Meta-Llama-3-70B-Instruct` | `meta-llama/Llama-3.3-70B-Instruct-Reference` | +| 2026-07-29 | `google/gemma-3-270m` | `Qwen/Qwen3.5-0.8B` | +| 2026-07-29 | `google/gemma-3-270m-it` | `Qwen/Qwen3.5-0.8B` | +| 2026-07-29 | `google/gemma-3-1b-it` | `google/gemma-4-26B-A4B-it` | +| 2026-07-29 | `google/gemma-3-1b-pt` | `google/gemma-4-26B-A4B-it` | +| 2026-07-29 | `google/gemma-3-4b-it` | `google/gemma-4-26B-A4B-it` | +| 2026-07-29 | `google/gemma-3-4b-it-VLM` | `google/gemma-4-26B-A4B-it` | +| 2026-07-29 | `google/gemma-3-4b-pt` | `google/gemma-4-26B-A4B-it` | +| 2026-07-29 | `google/gemma-3-12b-it` | `google/gemma-4-26B-A4B-it` | +| 2026-07-29 | `google/gemma-3-12b-it-VLM` | `google/gemma-4-31B-it-VLM` | +| 2026-07-29 | `google/gemma-3-12b-pt` | `google/gemma-4-26B-A4B-it` | +| 2026-07-29 | `google/gemma-3-27b-it` | `google/gemma-4-31B-it` | +| 2026-07-29 | `google/gemma-3-27b-it-VLM` | `google/gemma-4-31B-it-VLM` | +| 2026-07-29 | `google/gemma-3-27b-pt` | `google/gemma-4-31B-it` | +| 2026-07-29 | `mistralai/Mixtral-8x7B-v0.1` | `mistralai/Mixtral-8x7B-Instruct-v0.1` | +| 2026-07-29 | `mistralai/Mistral-7B-Instruct-v0.2` | `mistralai/Mixtral-8x7B-Instruct-v0.1` | +| 2026-07-29 | `mistralai/Mistral-7B-v0.1` | `mistralai/Mixtral-8x7B-Instruct-v0.1` | +| 2026-07-29 | `togethercomputer/llama-2-7b-chat` | `meta-llama/Meta-Llama-3.1-8B-Instruct-Reference` | + +## Recommended actions + +* Regularly check this page for updates on model deprecations. +* Plan your migration well in advance of the removal date to ensure a smooth transition. +* If you have any questions or need assistance with migration, contact the Together AI support team. + +For the most up-to-date information on model availability, support, and recommended alternatives, check the API documentation or contact the Together AI support team. diff --git a/tests/test_litellm/fixtures/together_ai_sync/models_serverless.json b/tests/test_litellm/fixtures/together_ai_sync/models_serverless.json new file mode 100644 index 00000000000..4988f0820bc --- /dev/null +++ b/tests/test_litellm/fixtures/together_ai_sync/models_serverless.json @@ -0,0 +1 @@ +[{"id":"moonshotai/Kimi-K3","uuid":"endpoint-kk-moonshotai-kimi-k3","object":"model","created":1785049898,"type":"chat","running":false,"display_name":"Kimi K3","organization":"Moonshot AI","link":"https://huggingface.co/moonshotai","license":"other","context_length":1048576,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":3,"output":15,"base":0,"finetune":0,"cached_input":0.3,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"zai-org/GLM-5.2","uuid":"endpoint-83348bee-b0fb-4aad-8ba4-72545469cb9e","object":"model","created":0,"type":"chat","running":false,"display_name":"GLM 5.2","organization":"Zai Org","link":"https://huggingface.co/api/models/nvidia/GLM-5.2-NVFP4","context_length":1048575,"config":{"chat_template":"[gMASK]\n{%- set effective_reasoning_effort = 'high' if reasoning_effort is defined and reasoning_effort == 'high' else 'max' -%}\n{%- if (enable_thinking is not defined or enable_thinking) and effective_reasoning_effort is not none -%}<|system|>Reasoning Effort: {{ effective_reasoning_effort | capitalize }}{%- endif -%}\n{%- if tools -%}\n{%- macro tool_to_json(tool) -%}\n {%- set ns_tool = namespace(first=true) -%}\n {{ '{' -}}\n {%- for k, v in tool.items() -%}\n {%- if k != 'defer_loading' and k != 'strict' -%}\n {%- if not ns_tool.first -%}{{- ', ' -}}{%- endif -%}\n {%- set ns_tool.first = false -%}\n \"{{ k }}\": {{ v | tojson(ensure_ascii=False) }}\n {%- endif -%}\n {%- endfor -%}\n {{- '}' -}}\n{%- endmacro -%}\n<|system|>\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{% for tool in tools %}\n{%- if 'function' in tool -%}\n {%- set tool = tool['function'] -%}\n{%- endif -%}\n{% if tool.defer_loading is not defined or not tool.defer_loading %}\n{{ tool_to_json(tool) }}\n{% endif %}\n{% endfor %}\n\n\nFor each function call, output the function name and arguments within the following XML format:\n{function-name}{arg-key-1}{arg-value-1}{arg-key-2}{arg-value-2}...{%- endif -%}\n{%- macro visible_text(content) -%}\n {%- if content is string -%}\n {{- content }}\n {%- elif content is iterable and content is not mapping -%}\n {%- for item in content -%}\n {%- if item is mapping and item.type == 'text' -%}\n {{- item.text }}\n {%- elif item is string -%}\n {{- item }}\n {%- elif item is mapping and item.type in ['image', 'image_url', 'video', 'video_url', 'audio', 'audio_url', 'input_audio'] -%}\n {%- set media_type = item.type | replace('_url', '') | replace('input_', '') -%}\n {{- \"You are unable to process this \" ~ media_type ~ \" because you don't have multi-modal input ability. Try different methods.\" }}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {{- content }}\n {%- endif -%}\n{%- endmacro -%}\n{%- set ns = namespace(last_user_index=-1) -%}\n{%- for m in messages %}\n {%- if m.role == 'user' %}\n {%- set ns.last_user_index = loop.index0 -%}\n {%- endif %}\n{%- endfor %}\n{%- for m in messages -%}\n{%- if m.role == 'user' -%}<|user|>{{ visible_text(m.content) }}\n{%- elif m.role == 'assistant' -%}\n<|assistant|>\n{%- set content = visible_text(m.content) %}\n{%- if m.reasoning_content is string %}\n {%- set reasoning_content = m.reasoning_content %}\n{%- elif '' in content %}\n {%- set reasoning_content = content.split('')[0].split('')[-1] %}\n {%- set content = content.split('')[-1] %}\n{%- endif %}\n{%- if ((clear_thinking is defined and not clear_thinking) or loop.index0 > ns.last_user_index) and reasoning_content is defined -%}\n{{ '' + reasoning_content + ''}}\n{%- else -%}\n{{ '' }}\n{%- endif -%}\n{%- if content.strip() -%}\n{{ content.strip() }}\n{%- endif -%}\n{% if m.tool_calls %}\n{% for tc in m.tool_calls %}\n{%- if tc.function %}\n {%- set tc = tc.function %}\n{%- endif %}\n{{- '' + tc.name -}}\n{% set _args = tc.arguments %}{% for k, v in _args.items() %}{{ k }}{{ v | tojson(ensure_ascii=False) if v is not string else v }}{% endfor %}{% endfor %}\n{% endif %}\n{%- elif m.role == 'tool' -%}\n{%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|observation|>' -}}\n{%- endif %}\n{%- if m.content is string -%}\n {{- '' + m.content + '' -}}\n{%- elif m.content is iterable and m.content is not mapping and m.content and m.content.0.type == \"tool_reference\" -%}\n {{- '\\n' -}}\n {% for tr in m.content %}\n {%- for tool in tools -%}\n {%- if 'function' in tool -%}\n {%- set tool = tool['function'] -%}\n {%- endif -%}\n {%- if tool.name == tr.name -%}\n {{- tool_to_json(tool) + '\\n' -}}\n {%- endif -%}\n {%- endfor -%}\n {%- endfor -%}\n {{- '' -}}\n{%- elif m.content is iterable and m.content is not mapping and m.content and m.content.0 is mapping and m.content.0.output is defined -%}\n {%- for tr in m.content -%}\n {{- '' + tr.output + '' -}}\n {%- endfor -%}\n{%- else -%}\n {{- '' + visible_text(m.content) + '' -}}\n{% endif -%}\n{%- elif m.role == 'system' -%}\n<|system|>{{ visible_text(m.content) }}\n{%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n <|assistant|>{{- '' if (enable_thinking is defined and not enable_thinking) else '' -}}\n{%- endif -%}\n","stop":[],"bos_token":null,"eos_token":"<|endoftext|>"},"pricing":{"hourly":0,"input":1.4,"output":4.4,"base":0,"finetune":0,"cached_input":0.25999999999999995,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"meta-models/Muse-Glimmer-30B","uuid":"endpoint-3da50849-cf6c-4e18-b44a-0dec9a699874","object":"model","created":0,"type":"chat","running":false,"display_name":"Muse Glimmer 30B","organization":"Meta","link":"https://huggingface.co/api/models/togethercomputer/onyx_final_hf-fp8-mlp","context_length":131072,"config":{"chat_template":null,"stop":["<|end_of_text|>"],"bos_token":"<|begin_of_text|>","eos_token":"<|end_of_text|>"},"pricing":{"hourly":0,"input":0.35,"output":1.5,"base":0,"finetune":0,"cached_input":0.04,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"Qwen/Qwen3.8-2.4T-A95B","uuid":"endpoint-494c76e2-129e-41ee-9ab9-e25c9a3ff08c","object":"model","created":0,"type":"chat","running":false,"display_name":"Qwen3.8-2.4T-A95B","organization":"Qwen","context_length":1010000,"config":{"chat_template":"{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n {%- if content is string %}\n {{- content }}\n {%- elif content is iterable and content is not mapping %}\n {%- for item in content %}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain images.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Picture ' ~ image_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n {%- elif 'video' in item or item.type == 'video' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain videos.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Video ' ~ video_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n {%- elif 'text' in item %}\n {{- item.text }}\n {%- else %}\n {{- raise_exception('Unexpected item type in content.') }}\n {%- endif %}\n {%- endfor %}\n {%- elif content is none or content is undefined %}\n {{- '' }}\n {%- else %}\n {{- raise_exception('Unexpected content type.') }}\n {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- set reasoning_instructions = '' %}\n{%- if enable_thinking is undefined or enable_thinking is true %}\n {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}\n {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}\n {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }}\n {%- endif %}\n {%- if resolved_reasoning_effort == 'xhigh' %}\n {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}\n {%- elif resolved_reasoning_effort == 'low' %}\n {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}\n {%- endif %}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {%- if reasoning_instructions %}\n {{- reasoning_instructions + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n\\n\\n\\nvalue_1\\n\\n\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n\\n\\n\\n\\n\\nReminder:\\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n' }}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {%- if content %}\n {{- '\\n\\n' + content }}\n {%- endif %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {%- if content %}\n {{- '<|im_start|>system\\n' + (reasoning_instructions + '\\n\\n' if reasoning_instructions else '') + content + '<|im_end|>\\n' }}\n {%- elif reasoning_instructions %}\n {{- '<|im_start|>system\\n' + reasoning_instructions + '<|im_end|>\\n' }}\n {%- endif %}\n {%- elif reasoning_instructions %}\n {{- '<|im_start|>system\\n' + reasoning_instructions + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" %}\n {%- set content = render_content(message.content, false)|trim %}\n {%- if not(content.startswith('') and content.endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n {%- set content = render_content(message.content, true)|trim %}\n {%- if message.role == \"system\" %}\n {%- if not loop.first %}\n {{- raise_exception('System message must be at the beginning.') }}\n {%- endif %}\n {%- elif message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- endif %}\n {%- set reasoning_content = reasoning_content|trim %}\n {%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content + '\\n\\n\\n' + content }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if loop.first %}\n {%- if content|trim %}\n {{- '\\n\\n\\n\\n' }}\n {%- else %}\n {{- '\\n\\n' }}\n {%- endif %}\n {%- else %}\n {{- '\\n\\n\\n' }}\n {%- endif %}\n {%- if tool_call.arguments is defined and tool_call.arguments != '' %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '\\n' }}\n {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}\n {{- args_value }}\n {{- '\\n\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception('Unexpected message role.') }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '\\n\\n\\n\\n' }}\n {%- else %}\n {{- '\\n' }}\n {%- endif %}\n{%- endif %}","stop":["<|im_end|>"],"bos_token":null,"eos_token":"<|im_end|>"},"pricing":{"hourly":0,"input":2.5,"output":6.25,"base":0,"finetune":0,"cached_input":0.5,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"deepseek-ai/DeepSeek-V4-Pro-0813","object":"model","created":1786804181,"type":"chat","running":false,"display_name":"DeepSeek V4 Pro 0813","organization":"DeepSeek","context_length":1048576,"config":{"chat_template":null,"stop":["<｜end▁of▁sentence｜>"],"bos_token":"<｜begin▁of▁sentence｜>","eos_token":"<｜end▁of▁sentence｜>"},"pricing":{"hourly":0,"input":1.32,"output":3.96,"base":0,"finetune":0,"cached_input":0.12999999999999998,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"deepseek-ai/DeepSeek-V4-Flash-0731","uuid":"endpoint-59e1bfe8-dcfd-4902-8e59-8e9585cfab4e","object":"model","created":0,"type":"chat","running":false,"display_name":"Deepseek V4 Flash 0731","organization":"Deepseek AI","context_length":1048576,"config":{"chat_template":null,"stop":["<｜end▁of▁sentence｜>"],"bos_token":"<｜begin▁of▁sentence｜>","eos_token":"<｜end▁of▁sentence｜>"},"pricing":{"hourly":0,"input":0.13999999999999999,"output":0.27999999999999997,"base":0,"finetune":0,"cached_input":0.030000000000000002,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"thinkingmachines/Inkling","uuid":"endpoint-8b0aa8da-8d35-4a01-be0b-eca731d64568","object":"model","created":0,"type":"chat","running":false,"display_name":"Inkling FP4","organization":"Thinking Machines","link":"https://huggingface.co/api/models/thinkingmachines/Inkling-NVFP4","license":"apache-2.0","context_length":524288,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":1,"output":4.05,"base":0,"finetune":0,"cached_input":0.17,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"MiniMaxAI/MiniMax-M3","uuid":"endpoint-5dea048e-3527-4287-8da8-5e61214b9f64","object":"model","created":0,"type":"chat","running":false,"display_name":"MiniMax M3","organization":"MiniMaxAI","context_length":524288,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0.3,"output":1.2,"base":0,"finetune":0,"cached_input":0.060000000000000005,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"thinkingmachines/Inkling-Small","object":"model","created":1785387855,"type":"chat","running":false,"display_name":"Inkling Small","organization":"Thinking Machines","link":"https://huggingface.co/api/models/thinkingmachines/Inkling-Small","context_length":524288,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0.5,"output":1.2,"base":0,"finetune":0,"cached_input":0.1,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"moonshotai/Kimi-K2.7-Code","uuid":"endpoint-b8ae5f69-a244-43dd-a6ac-957653518387","object":"model","created":0,"type":"chat","running":false,"display_name":"Kimi K2.7 Code","organization":"Moonshot AI","link":"https://huggingface.co/api/models/togethercomputer/Kimi-K2.7-Code-FP4","context_length":262144,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0.95,"output":4,"base":0,"finetune":0,"cached_input":0.19,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"deepseek-ai/DeepSeek-V4-Pro","uuid":"endpoint-94151073-7212-43f8-9357-42a6043e1eef","object":"model","created":0,"type":"chat","running":false,"display_name":"Deepseek V4 Pro","organization":"Deepseek","context_length":512000,"config":{"chat_template":null,"stop":["<｜end▁of▁sentence｜>"],"bos_token":"<｜begin▁of▁sentence｜>","eos_token":"<｜end▁of▁sentence｜>"},"pricing":{"hourly":0,"input":1.74,"output":3.48,"base":0,"finetune":0,"cached_input":0.2,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"nvidia/nemotron-3-ultra-550b-a55b","uuid":"endpoint-0f2ee6f7-0ad9-42e9-89df-cab8904dc46c","object":"model","created":0,"type":"chat","running":false,"display_name":"NVIDIA Nemotron 3 Ultra 550B A55B NVFP4","organization":"NVIDIA","context_length":512288,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0.6,"output":3.6,"base":0,"finetune":0,"cached_input":0.2,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"Qwen/Qwen3.7-Max","uuid":"endpoint-ba47b6c3-f84c-435c-9d86-d8142b17031b","object":"model","created":1779386434,"type":"chat","running":false,"display_name":"Qwen3.7 Max","organization":"Qwen","context_length":1000000,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":1.25,"output":3.75,"base":0,"finetune":0,"cached_input":0.125,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"google/gemma-4-31B-it","uuid":"endpoint-155df9cc-8c2f-4a04-8840-728681211a34","object":"model","created":0,"type":"chat","running":false,"display_name":"Gemma 4 31B-it FP8","organization":"Google","link":"https://huggingface.co/api/models/google/gemma-4-31B-it","license":"apache-2.0","context_length":262144,"config":{"chat_template":null,"stop":[""],"bos_token":"","eos_token":""},"pricing":{"hourly":0,"input":0.39,"output":0.9700000000000001,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"pearl-ai/gemma-4-31b-it","object":"model","created":1778777629,"type":"chat","running":false,"display_name":"Pearl-ai Gemma-4-31B-it-pearl","organization":"pearl.ai","link":"https://huggingface.co/pearl-ai/Gemma-4-31B-it-pearl","context_length":262144,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0.27999999999999997,"output":0.86,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"openai/gpt-oss-120b","uuid":"endpoint-cf361a3e-47d0-4dfc-851a-97098881e6a2","object":"model","created":1754414557,"type":"chat","running":false,"display_name":"OpenAI GPT-OSS 120B","organization":"OpenAI","link":"https://huggingface.co/openai/gpt-oss-120b","license":"other","context_length":131072,"config":{"chat_template":null,"stop":["<|return|>"],"bos_token":"<|startoftext|>","eos_token":"<|return|>"},"pricing":{"hourly":0,"input":0.15,"output":0.6,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"openai/gpt-oss-20b","uuid":"endpoint-f382c20a-6806-4ac2-abfb-d00d7a0b0c2b","object":"model","created":1774480577,"type":"chat","running":false,"display_name":"OpenAI GPT-OSS 20B","organization":"OpenAI","link":"https://huggingface.co/api/models/openai/gpt-oss-20b","license":"apache-2.0","context_length":131072,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0.05,"output":0.2,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"Qwen/Qwen3.5-9B","uuid":"endpoint-71bb7894-08d4-4882-bb72-7c257c234513","object":"model","created":0,"type":"chat","running":false,"display_name":"Qwen3.5 9B FP8","organization":"Qwen","link":"https://huggingface.co/api/models/togethercomputer/Qwen3.5-9B-FP8-MLP","context_length":262144,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":"<|im_end|>"},"pricing":{"hourly":0,"input":0.17,"output":0.25,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"meta-llama/Llama-3.3-70B-Instruct-Turbo","object":"model","created":1733466629,"type":"chat","running":false,"display_name":"Meta Llama 3.3 70B Instruct Turbo","organization":"Meta","link":"https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct","license":"Llama-3.3 (Other)","context_length":131072,"config":{"chat_template":"{{- bos_token }}\n{%- if custom_tools is defined %}\n {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n {#- Extract the first user message so we can plug it in here #}\n {%- if messages | length != 0 %}\n {%- set first_user_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n {%- else %}\n {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n {%- elif 'tool_calls' in message %}\n {%- if not message.tool_calls|length == 1 %}\n {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n {%- endif %}\n {%- set tool_call = message.tool_calls[0].function %}\n {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n {%- for arg_name, arg_val in tool_call.arguments | items %}\n {{- arg_name + '=\"' + arg_val + '\"' }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- endif %}\n {%- endfor %}\n {{- \")\" }}\n {%- else %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n {{- '\"parameters\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- \"}\" }}\n {%- endif %}\n {%- if builtin_tools is defined %}\n {#- This means we're in ipython mode #}\n {{- \"<|eom_id|>\" }}\n {%- else %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n {%- if message.content is mapping or message.content is iterable %}\n {{- message.content | tojson }}\n {%- else %}\n {{- message.content }}\n {%- endif %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n","stop":["<|eot_id|>","<|eom_id|>"],"bos_token":"<|begin_of_text|>","eos_token":"<|eot_id|>"},"pricing":{"hourly":0,"input":1.0399999999999998,"output":1.0399999999999998,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"google/gemma-3n-E4B-it","uuid":"endpoint-290b90f1-cdb9-46c1-a919-9a73822375c3","object":"model","created":1750955040,"type":"chat","running":false,"display_name":"Gemma 3N E4B Instruct","organization":"Google","link":"https://huggingface.co/google/gemma-3n-E4B-it","license":"gemma","context_length":32768,"config":{"chat_template":null,"stop":[""],"bos_token":"","eos_token":""},"pricing":{"hourly":0,"input":0.060000000000000005,"output":0.12000000000000001,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"hexgrad/Kokoro-82M","object":"model","created":1773163054,"type":"audio","running":false,"display_name":"Kokoro 82M","organization":"Hexgrad","link":"https://huggingface.co/hexgrad/Kokoro-82M","license":"apache2","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":4,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"canopylabs/orpheus-3b-0.1-ft","object":"model","created":1755731205,"type":"audio","running":false,"display_name":"Orpheus 3B 0.1 FT","organization":"Canopy Labs","link":"https://huggingface.co/canopylabs/orpheus-3b-0.1-ft","license":"apache2","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":15,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"openai/whisper-large-v3","uuid":"endpoint-b0eaec1e-3edb-48c3-85a9-1af9b5ce09fb","object":"model","created":0,"type":"transcribe","running":false,"display_name":"Whisper large-v3","organization":"OpenAI","link":"https://huggingface.co/openai/whisper-large-v3","license":"apache2","context_length":1,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0.27,"output":0.85,"base":0,"finetune":0,"image_pixel":0,"transcribe":{"price_per_minute":0.0015},"image":0,"video":0}},{"id":"black-forest-labs/FLUX.1-kontext-pro","object":"model","created":0,"type":"image","running":false,"display_name":"FLUX.1 Kontext [pro]","organization":"Black Forest Labs","context_length":0,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":{"price_per_megapixel":0.04,"min_steps":0},"transcribe":0,"image":0,"video":0}},{"id":"black-forest-labs/FLUX.1-kontext-max","object":"model","created":0,"type":"image","running":false,"display_name":"FLUX.1 Kontext [max]","organization":"Black Forest Labs","context_length":0,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":{"price_per_megapixel":0.08,"min_steps":0},"transcribe":0,"image":0,"video":0}},{"id":"black-forest-labs/FLUX.2-dev","uuid":"endpoint-268047b1-b295-4d9b-bc9f-239d375768ab","object":"model","created":1764086551,"type":"image","running":false,"display_name":"FLUX.2 [dev]","organization":"Black Forest Labs","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.0154,"example_description":"starting price per image"},"video":0}},{"id":"black-forest-labs/FLUX.2-flex","uuid":"endpoint-3d15053d-a558-487c-b0f8-068e9dfd781f","object":"model","created":1764090764,"type":"image","running":false,"display_name":"FLUX.2 [flex]","organization":"Black Forest Labs","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.03,"example_description":"per text-to-image"},"video":0}},{"id":"black-forest-labs/FLUX.2-pro","uuid":"endpoint-f6f3da91-6f41-4b38-b61c-40f60902b714","object":"model","created":1764070232,"type":"image","running":false,"display_name":"FLUX.2 [pro]","organization":"Black Forest Labs","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.03,"example_description":"per text-to-image image"},"video":0}},{"id":"black-forest-labs/FLUX.2-max","object":"model","created":0,"type":"image","running":false,"display_name":"FLUX.2 [max]","organization":"Black Forest Labs","context_length":0,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":{"price_per_megapixel":0.07,"min_steps":50},"transcribe":0,"image":0,"video":0}},{"id":"black-forest-labs/FLUX.1.1-pro","uuid":"endpoint-071376f6-db8a-44cf-9706-7ba0c9c14833","object":"model","created":0,"type":"image","running":false,"display_name":"FLUX1.1 [pro]","organization":"Black Forest Labs","link":"https://huggingface.co/black-forest-labs/FLUX.1-schnell","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":{"price_per_megapixel":0.04,"min_steps":0},"transcribe":0,"image":0,"video":0}},{"id":"meta-llama/Llama-Guard-4-12B","uuid":"endpoint-2095a2fc-bb95-48df-b94f-f95ccb22fa93","object":"model","created":1746038486,"type":"moderation","running":false,"display_name":"Llama Guard 4 12B","organization":"Meta","link":"https://huggingface.co/meta-llama/Llama-Guard-4-12B","license":"other llama4","context_length":1048576,"config":{"chat_template":"{%- if messages | length % 2 == 0 -%}\n {%- set role = 'Agent' -%}\n{%- else -%}\n {%- set role = 'User' -%}\n{%- endif -%}\n\n{%- if categories is not defined -%}\n {%- set ns = namespace(contains_image=false) -%}\n\n {%- for message in messages -%}\n {%- for content in message['content'] | selectattr('type', 'equalto', 'image') -%}\n {%- set ns.contains_image = true -%}\n {%- endfor -%}\n {%- endfor -%}\n\n {%- if ns.contains_image -%}\n {%- set categories = {\n \"S1\": \"Violent Crimes.\",\n \"S2\": \"Non-Violent Crimes.\",\n \"S3\": \"Sex Crimes.\",\n \"S4\": \"Child Exploitation.\",\n \"S5\": \"Defamation.\",\n \"S6\": \"Specialized Advice.\",\n \"S7\": \"Privacy.\",\n \"S8\": \"Intellectual Property.\",\n \"S9\": \"Indiscriminate Weapons.\",\n \"S10\": \"Hate.\",\n \"S11\": \"Self-Harm.\",\n \"S12\": \"Sexual Content.\",\n \"S13\": \"Elections.\",\n } -%}\n {%- else -%}\n {%- set categories = {\n \"S1\": \"Violent Crimes.\",\n \"S2\": \"Non-Violent Crimes.\",\n \"S3\": \"Sex Crimes.\",\n \"S4\": \"Child Exploitation.\",\n \"S5\": \"Defamation.\",\n \"S6\": \"Specialized Advice.\",\n \"S7\": \"Privacy.\",\n \"S8\": \"Intellectual Property.\",\n \"S9\": \"Indiscriminate Weapons.\",\n \"S10\": \"Hate.\",\n \"S11\": \"Self-Harm.\",\n \"S12\": \"Sexual Content.\",\n \"S13\": \"Elections.\",\n \"S14\": \"Code Interpreter Abuse.\",\n } -%}\n {%- endif -%}\n{%- endif -%}\n\n{{ \"<|begin_of_text|><|header_start|>user<|header_end|>\\n\" }}\n{{ \"Task: Check if there is unsafe content in '\" + role + \"' messages in conversations according to our safety policy with the below categories.\\n\" }}\n{{ \"\\n\" }}\n\n{%- for key in categories -%}\n {%- if key not in excluded_category_keys -%}\n {{ key + \": \" + categories[key] + \"\\n\" }}\n {%- endif -%}\n{%- endfor -%}\n\n{{ \"\\n\" }}\n{{ \"\\n\" }}\n\n{%- for message in messages -%}\n {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}\n {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}\n {%- endif -%}\n\n {%- set content = message['content'] -%}\n\n {%- if message['role'] == 'user' -%}\n {%- set role = 'User' -%}\n {%- elif message['role'] == 'assistant' -%}\n {%- set role = 'Agent' -%}\n {%- endif -%}\n\n {#— #1 Accumulate all images for this message —#}\n {%- set ns = namespace(img_tokens='') -%}\n {%- for img in message.content | selectattr('type', 'equalto', 'image') -%}\n {%- set ns.img_tokens = ns.img_tokens + '<|image|>' -%}\n {%- endfor -%}\n\n {#— #2 Then emit all text —#}\n {%- for txt in message.content | selectattr('type','equalto','text') -%}\n {{ role + ': ' }}{{ ns.img_tokens }}{{ txt.text | trim }}\\n\\n\n\n {#— clear the buffer so only first text prints images —#}\n {%- if loop.first -%}\n {%- set ns.img_tokens = '' -%}\n {%- endif -%}\n {%- endfor -%}\n{%- endfor -%}\n\n{{ \"\\n\" }}\n{{ \"Provide your safety assessment for ONLY THE LAST \" + role + \" message in the above conversation:\" }}\n{{ \" - First line must read 'safe' or 'unsafe'.\" }}\n{{ \" - If unsafe, a second line must include a comma-separated list of violated categories. <|eot|><|header_start|>assistant<|header_end|>\" }}","stop":["<|eot|>"],"bos_token":"<|begin_of_text|>","eos_token":"<|eot|>"},"pricing":{"hourly":0,"input":0.2,"output":0.2,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"intfloat/multilingual-e5-large-instruct","uuid":"endpoint-b1b563e5-5ec2-4577-9017-16b52ac5c841","object":"model","created":1745513588,"type":"embedding","running":false,"display_name":"Multilingual E5 Large Instruct","organization":"Intfloat","link":"https://huggingface.co/api/models/intfloat/multilingual-e5-large-instruct","license":"mit","context_length":514,"config":{"chat_template":null,"stop":[""],"bos_token":"","eos_token":""},"pricing":{"hourly":0,"input":0.02,"output":0.02,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"arize-ai/qwen-2-1.5b-instruct","uuid":"endpoint-22ce9f16-299a-47cc-b88f-c59cfb1d235e","object":"model","created":1745522693,"type":"chat","running":false,"display_name":"Arize AI Qwen 2 1.5B Instruct","organization":"Togethercomputer","link":"https://huggingface.co/api/models/togethercomputer/arize-ai-qwen-2-1.5b-instruct","context_length":32768,"config":{"chat_template":"{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}","stop":["<|im_end|>"],"bos_token":"<|endoftext|>","eos_token":"<|im_end|>"},"pricing":{"hourly":0,"input":0.1,"output":0.1,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"nvidia/parakeet-tdt-0.6b-v3","uuid":"endpoint-3fbe0c47-5c71-4f52-92fb-abaff932f05f","object":"model","created":0,"type":"transcribe","running":false,"display_name":"Nvidia Parakeet TDT 0.6B V3","organization":"Nvidia","link":"https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3","context_length":448,"config":{"chat_template":null,"stop":["<|endoftext|>"],"bos_token":"<|endoftext|>","eos_token":"<|endoftext|>"},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":{"price_per_minute":0.0015},"image":0,"video":0}},{"id":"openai/gpt-image-1.5","uuid":"endpoint-11f45afc-3f72-41d1-b93e-902e220f4d5a","object":"model","created":1765980893,"type":"image","running":false,"display_name":"GPT Image 1.5","organization":"OpenAI","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.034,"example_description":"/opt/homebrew/bin/zsh.009 - /opt/homebrew/bin/zsh.199 per image based on quality"},"video":0}},{"id":"Wan-AI/Wan2.6-image","uuid":"endpoint-7dc7f98d-c562-4b5a-b710-c24875a6b471","object":"model","created":1769618722,"type":"image","running":false,"display_name":"Wan 2.6 Image","organization":"Wan-AI","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.03,"example_description":"per output image"},"video":0}},{"id":"google/veo-3.0-fast-audio","uuid":"endpoint-8bdb9924-b64e-4f44-ad5f-c979e578e7f4","object":"model","created":1759884907,"type":"video","running":false,"display_name":"Google Veo 3.0 Fast + Audio","organization":"Google","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":1.2,"example_description":"1080p / 8s"}}},{"id":"vidu/vidu-q1","uuid":"endpoint-fea0b805-4d7e-45ec-8b1b-856c932f152c","object":"model","created":1759884996,"type":"video","running":false,"display_name":"Vidu Q1","organization":"Vidu","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.22,"example_description":"1080p / 5s"}}},{"id":"cartesia/sonic","object":"model","created":1773696454,"type":"audio","running":false,"display_name":"Cartesia Sonic","organization":"Cartesia","link":"https://www.cartesia.ai","context_length":0,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":65,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"ByteDance-Seed/Seedream-3.0","uuid":"endpoint-c2769196-9347-46e4-815a-9c7abf5b8d50","object":"model","created":1759884740,"type":"image","running":false,"display_name":"ByteDance Seedream 3.0","organization":"ByteDance","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.018,"example_description":"720x1280"},"video":0}},{"id":"ByteDance-Seed/Seedream-4.0","uuid":"endpoint-e27a4640-becc-4a5a-92f4-3940b7be23e8","object":"model","created":1759884757,"type":"image","running":false,"display_name":"ByteDance Seedream 4.0","organization":"ByteDance","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.03,"example_description":"720x1280"},"video":0}},{"id":"Rundiffusion/Juggernaut-Lightning-Flux","uuid":"endpoint-63c3e50f-b9eb-41e3-a3ed-7242665874e4","object":"model","created":1759884814,"type":"image","running":false,"display_name":"Juggernaut Lightning Flux by RunDiffusion","organization":"RunDiffusion","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.0017,"example_description":"720x1280"},"video":0}},{"id":"google/veo-3.0-audio","uuid":"endpoint-ced52ba5-3cb0-46a3-aa92-d7a2f59d6bd9","object":"model","created":1759884892,"type":"video","running":false,"display_name":"Google Veo 3.0 + Audio","organization":"Google","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":3.2,"example_description":"720p / 8s"}}},{"id":"kwaivgI/kling-2.1-master","uuid":"endpoint-5e489acf-5401-4843-97b7-8a830648bd3c","object":"model","created":1759884953,"type":"video","running":false,"display_name":"Kling 2.1 Master","organization":"kwaivgI","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.924,"example_description":"1080p / 5s"}}},{"id":"ideogram/ideogram-3.0","uuid":"endpoint-3d82f587-56ba-45df-817d-854cd2117f41","object":"model","created":1759884808,"type":"image","running":false,"display_name":"Ideogram 3.0","organization":"ideogram","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.06,"example_description":"720x1280"},"video":0}},{"id":"kwaivgI/kling-2.1-pro","uuid":"endpoint-8fa3e87a-9f35-45fc-8157-8ed046498ba6","object":"model","created":1759884948,"type":"video","running":false,"display_name":"Kling 2.1 Pro","organization":"kwaivgI","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.3234,"example_description":"1080p / 5s"}}},{"id":"google/veo-2.0","uuid":"endpoint-ad40ee70-5f82-4283-b2d8-2813a2773022","object":"model","created":1759884886,"type":"video","running":false,"display_name":"Google Veo 2.0","organization":"Google","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":2.5,"example_description":"720p / 5s"}}},{"id":"openai/sora-2","uuid":"endpoint-c4adc1b3-6ac2-491a-b4b0-e0c3b3fea40f","object":"model","created":1760480340,"type":"video","running":false,"display_name":"Sora 2","organization":"OpenAI","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.8,"example_description":"720p / 8s"}}},{"id":"kwaivgI/kling-2.1-standard","uuid":"endpoint-09e526e5-8428-4841-8242-c883b8600a8c","object":"model","created":1759884940,"type":"video","running":false,"display_name":"Kling 2.1 Standard","organization":"kwaivgI","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.1848,"example_description":"720p / 5s"}}},{"id":"google/veo-3.0-fast","uuid":"endpoint-92bc9b5a-365e-48e2-bc37-e278671310cb","object":"model","created":1759884913,"type":"video","running":false,"display_name":"Google Veo 3.0 Fast","organization":"Google","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.8,"example_description":"1080p / 8s"}}},{"id":"google/gemini-3-pro-image","uuid":"endpoint-d2f07d30-6a03-4f98-a52d-cdc5461cf639","object":"model","created":1763662095,"type":"image","running":false,"display_name":"Gemini 3 (Nano Banana Pro)","organization":"Google","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.134,"example_description":"1080p & 2K resolutions costs $0.134/image and 4K resolutions costs $0.24 per image"},"video":0}},{"id":"vidu/vidu-2.0","uuid":"endpoint-31518301-3076-47c8-b42f-542569955820","object":"model","created":1759885002,"type":"video","running":false,"display_name":"Vidu 2.0","organization":"Vidu","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.8,"example_description":"720p / 8s"}}},{"id":"openai/sora-2-pro","uuid":"endpoint-03b9298b-8624-4c29-8055-941df060eda4","object":"model","created":1760480692,"type":"video","running":false,"display_name":"Sora 2 Pro","organization":"OpenAI","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":3,"example_description":"1080p / 8s"}}},{"id":"pixverse/pixverse-v5","uuid":"endpoint-1588b5bc-5923-4672-be92-3199a579a18f","object":"model","created":1759884975,"type":"video","running":false,"display_name":"PixVerse v5","organization":"PixVerse","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.299,"example_description":"1080p / 5s"}}},{"id":"stabilityai/stable-diffusion-xl-base-1.0","uuid":"endpoint-5bbe64a1-3798-4ad5-bfd5-aee40eca9564","object":"model","created":1759884771,"type":"image","running":false,"display_name":"SD XL","organization":"stabilityai","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.0019,"example_description":"720x1280"},"video":0}},{"id":"ByteDance/Seedance-1.0-lite","uuid":"endpoint-5467de41-51aa-4d08-98b5-8cd34dc19906","object":"model","created":1759884873,"type":"video","running":false,"display_name":"ByteDance Seedance 1.0 Lite","organization":"ByteDance","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.143,"example_description":"720p / 5s"}}},{"id":"cartesia/sonic-3","object":"model","created":1774464715,"type":"audio","running":false,"display_name":"Cartesia Sonic 3","organization":"Cartesia","link":"https://www.cartesia.ai","context_length":448,"config":{"chat_template":null,"stop":["<|endoftext|>"],"bos_token":"<|endoftext|>","eos_token":"<|endoftext|>"},"pricing":{"hourly":0,"input":65,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"ByteDance/Seedance-1.0-pro","uuid":"endpoint-9419195a-e048-4865-bf8b-89343a3e9b84","object":"model","created":1759884879,"type":"video","running":false,"display_name":"ByteDance Seedance 1.0 Pro","organization":"ByteDance","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.565,"example_description":"720p / 5s"}}},{"id":"google/imagen-4.0-fast","uuid":"endpoint-3ba3bc6f-fe2b-4446-9ec0-71e82ac3348d","object":"model","created":1759884793,"type":"image","running":false,"display_name":"Google Imagen 4.0 Fast","organization":"Google","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.02,"example_description":"720x1280"},"video":0}},{"id":"google/flash-image-2.5","uuid":"endpoint-e9655a27-b014-43b4-bff1-b343a0206e07","object":"model","created":1759884801,"type":"image","running":false,"display_name":"Gemini Flash Image 2.5 (Nano Banana)","organization":"Google","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.039,"example_description":"720x1280"},"video":0}},{"id":"minimax/hailuo-02","uuid":"endpoint-68520084-c967-42b6-bff4-a63b660bd0cf","object":"model","created":1759884967,"type":"video","running":false,"display_name":"MiniMax Hailuo 02","organization":"MiniMaxAI","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.56,"example_description":"768p / 10s"}}},{"id":"google/imagen-4.0-ultra","uuid":"endpoint-40d2690e-57a7-4e89-987d-2a3e44c1302d","object":"model","created":1759884786,"type":"image","running":false,"display_name":"Google Imagen 4.0 Ultra","organization":"Google","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.06,"example_description":"720x1280"},"video":0}},{"id":"google/imagen-4.0-preview","uuid":"endpoint-b6561013-bc17-4aa3-9a76-89174973977b","object":"model","created":1759884778,"type":"image","running":false,"display_name":"Google Imagen 4.0 Preview","organization":"Google","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.04,"example_description":"720x1280"},"video":0}},{"id":"RunDiffusion/Juggernaut-pro-flux","uuid":"endpoint-1f51e977-a298-40aa-a0c6-d5865c37bc38","object":"model","created":1759884821,"type":"image","running":false,"display_name":"Juggernaut Pro Flux by RunDiffusion 1.0.0","organization":"RunDiffusion","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.0049,"example_description":"720x1280"},"video":0}},{"id":"Qwen/Qwen-Image","uuid":"endpoint-d4d29f48-ce86-4533-863a-23e9245f6570","object":"model","created":1759884857,"type":"image","running":false,"display_name":"Qwen Image","organization":"Qwen","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.0058,"example_description":"720x1280"},"video":0}},{"id":"google/veo-3.0","uuid":"endpoint-test-duplicate-001","object":"model","created":1778817876,"type":"video","running":false,"display_name":"Duplicate Test","organization":"Google","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.08,"example_description":"test"}}},{"id":"kwaivgI/kling-1.6-standard","uuid":"endpoint-9f6794ed-52f7-414f-8974-d3b1ffb8702f","object":"model","created":1759884920,"type":"video","running":false,"display_name":"Kling 1.6 Standard","organization":"kwaivgI","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.185,"example_description":"720p / 5s"}}},{"id":"minimax/video-01-director","uuid":"endpoint-d5929bff-e81e-4bab-8b20-17cb99936a68","object":"model","created":1759884960,"type":"video","running":false,"display_name":"MiniMax 01 Director","organization":"MiniMaxAI","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{}}},{"id":"cartesia/sonic-2","object":"model","created":1774464715,"type":"audio","running":false,"display_name":"Cartesia Sonic 2","organization":"Cartesia","context_length":448,"config":{"chat_template":null,"stop":["<|endoftext|>"],"bos_token":"<|endoftext|>","eos_token":"<|endoftext|>"},"pricing":{"hourly":0,"input":65,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"pixverse/pixverse-v5.6","uuid":"endpoint-5e8550be-7faf-411e-81ee-92773d4a1304","object":"model","created":1769621066,"type":"video","running":false,"display_name":"PixVerse v5.6","organization":"PixVerse","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.1326,"example_description":"$0.1031 - $0.221 per 5 sec video without audio. Audio is an additional $0.1326"}}},{"id":"Qwen/Qwen-Image-2.0-Pro","uuid":"endpoint-ea16bed3-cfd1-477b-ad95-1ac0f28bfec2","object":"model","created":1773318281,"type":"image","running":false,"display_name":"Qwen Image 2.0 Pro","organization":"Qwen","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.075,"example_description":"per image"},"video":0}},{"id":"google/flash-image-3.1","uuid":"endpoint-f0e10a8e-9250-4bcc-b1a9-ae34f3ecdaec","object":"model","created":1772535344,"type":"image","running":false,"display_name":"Gemini 3.1 Flash Image (Nano Banana 2)","organization":"Google","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.04657,"example_description":"0.04657 for 512x512. For every input image used, it's an additional $0.00028. When using grounded search, $0.014 will be added on top."},"video":0}},{"id":"Qwen/Qwen-Image-2.0","uuid":"endpoint-9bd5c294-1a2e-4ffb-bf28-482e01eee56f","object":"model","created":1773251084,"type":"image","running":false,"display_name":"Qwen Image 2.0","organization":"Qwen","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.035,"example_description":"per image"},"video":0}},{"id":"Wan-AI/wan2.7-t2v","uuid":"endpoint-4e24da5f-2274-44ad-8bf3-36dc47a8114a","object":"model","created":1775245808,"type":"video","running":false,"display_name":"Wan 2.7 T2V","organization":"Wan-AI","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.1,"example_description":"per 5 seconds of video"}}},{"id":"Wan-AI/wan2.7-i2v","uuid":"endpoint-47e29650-3293-4538-bc90-fa3f07b159dc","object":"model","created":1775254675,"type":"video","running":false,"display_name":"Wan 2.7 I2V","organization":"Wan-AI","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.1,"example_description":"per 5 seconds of video"}}},{"id":"Wan-AI/wan2.7-r2v","uuid":"endpoint-819be224-66c1-424d-8d79-7d527bcf278c","object":"model","created":1775257231,"type":"video","running":false,"display_name":"Wan 2.7 R2V","organization":"Wan-AI","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.1,"example_description":"per 5 seconds of video"}}},{"id":"vidu/vidu-q3","uuid":"endpoint-002dc245-03bd-4e03-bdb0-e3fd55e25aba","object":"model","created":1776175177,"type":"video","running":false,"display_name":"Vidu Q3","organization":"Vidu","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.0975,"example_description":"0.0455 - 0.1040 per second depending on resolution"}}},{"id":"vidu/vidu-q3-turbo","uuid":"endpoint-1381491a-63c3-4513-abdc-15005e5e85a3","object":"model","created":1776175206,"type":"video","running":false,"display_name":"Vidu Q3 Turbo","organization":"Vidu","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.195,"example_description":"0.13 - 0.26 per second depending on resolution"}}},{"id":"google/veo-3.1-test-debug","uuid":"endpoint-test-debug-001","object":"model","created":0,"type":"video","running":false,"display_name":"Veo 3.1 Debug Test","organization":"Google","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.08,"example_description":"test"}}},{"id":"pixverse/pixverse-v6","uuid":"endpoint-9782553a-d1f6-4641-b70f-cf3664e95a8a","object":"model","created":1776953730,"type":"video","running":false,"display_name":"PixVerse v6","organization":"PixVerse","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.09,"example_description":"0.090/s at 1080p without audio. 0.115/s with audio"}}},{"id":"ByteDance/Seedance-2.0","uuid":"endpoint-1d17df31-ca97-4848-869e-be0f68b096a7","object":"model","created":1776942761,"type":"video","running":false,"display_name":"ByteDance Seedance 2.0","organization":"ByteDance","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.16,"example_description":"Text/Image to Video at 720P: $0.16/sec & Video-to-Video at 720P: from $0.28/sec"}}},{"id":"Qwen/Qwen3.6-Plus","uuid":"endpoint-78f9d01e-0c22-47dc-b2b2-6aa0e2f3570c-v2","object":"model","created":1777340375,"type":"chat","running":false,"display_name":"Qwen3.6 Plus","organization":"Qwen","context_length":1000000,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0.5,"output":3,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"HappyHorse/HappyHorse-1.0-T2V","object":"model","created":1777283507,"type":"video","running":false,"display_name":"","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.1,"example_description":"per 5 seconds of video"}}},{"id":"alibaba/happyhorse-1.0-t2v","uuid":"endpoint-e65e99d1-97f1-443f-94e2-dd139e102897","object":"model","created":1777714549,"type":"video","running":false,"display_name":"HappyHorse 1.0 T2V","organization":"Alibaba","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.24,"example_description":"Text to Video at 720P: $0.14/sec and $0.24/sec at 1080p"}}},{"id":"alibaba/happyhorse-1.0-r2v","uuid":"endpoint-320deb45-9a43-46b2-8393-32b466ce9bce","object":"model","created":1777717813,"type":"video","running":false,"display_name":"HappyHorse 1.0 R2V","organization":"Alibaba","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.24,"example_description":"Text/Image to Video at 720P: $0.14/sec and $0.24/sec at 1080p"}}},{"id":"alibaba/happyhorse-1.0-i2v","uuid":"endpoint-0fdc51d3-6dd3-4f2c-bce8-418ab47b36ea","object":"model","created":1777717851,"type":"video","running":false,"display_name":"HappyHorse 1.0 I2V","organization":"Alibaba","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.24,"example_description":"Text/Image to Video at 720P: $0.14/sec and $0.24/sec at 1080p"}}},{"id":"ByteDance/Seedream-5.0-lite","uuid":"endpoint-90244fc5-096f-4bca-b5f2-79664175e2c4","object":"model","created":1778252567,"type":"image","running":false,"display_name":"ByteDance Seedream 5.0 Lite","organization":"ByteDance","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.035,"example_description":"Pricing is $0.035 for both 2K & 3K outputs"},"video":0}},{"id":"google/veo-3.1","uuid":"endpoint-b0a69f31-f14c-4825-9c01-cf20b5aeece9","object":"model","created":1776790993,"type":"video","running":false,"display_name":"Veo 3.1","organization":"Google","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.08,"example_description":"0.08/ per 4s at 720p without audio. .60/s with audio"}}},{"id":"google/veo-3.1-lite","uuid":"endpoint-0a06c93a-68ce-48f6-bfbf-d9a0337a073b","object":"model","created":1778615460,"type":"video","running":false,"display_name":"Veo 3.1 Lite","organization":"Google","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.05,"example_description":"0.05/s at 1080p without audio. 0.80/s with audio."}}},{"id":"nvidia/nemotron-3.5-asr-streaming-0.6b","uuid":"endpoint-cd9d043d-92ac-4320-af6a-2638e934861a","object":"model","created":0,"type":"transcribe","running":false,"display_name":"Nvidia Nemotron 3.5 ASR Streaming 0.6B","organization":"Nvidia","link":"https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b","license":"apache-2.0","context_length":448,"config":{"chat_template":null,"stop":[],"bos_token":"<|endoftext|>","eos_token":"<|endoftext|>"},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":{"price_per_minute":0.0015},"image":0,"video":0}},{"id":"nvidia/nemotron-3-asr-streaming-0.6b","uuid":"endpoint-614e0569-b81e-4234-b08e-976d81913415","object":"model","created":0,"type":"transcribe","running":false,"display_name":"Nvidia Nemotron 3 ASR Streaming 0.6B","organization":"Nvidia","link":"https://huggingface.co/nvidia/nemotron-speech-streaming-en-0.6b","license":"apache-2.0","context_length":448,"config":{"chat_template":null,"stop":[],"bos_token":"<|endoftext|>","eos_token":"<|endoftext|>"},"pricing":{"hourly":0,"input":0.45,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":{"price_per_minute":0.0015},"image":0,"video":0}},{"id":"ideogram/ideogram-4.0","uuid":"endpoint-0304633d-06c9-4d89-a093-eaf52cc62aae","object":"model","created":1780584367,"type":"image","running":false,"display_name":"Ideogram 4.0","organization":"ideogram","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.06,"example_description":"per image price ranging from 0.03 - 0.10 per based on size and quality"},"video":0}},{"id":"openai/gpt-image-2","uuid":"endpoint-3a75d1cd-a76f-4277-b7f6-a6c62d05901b","object":"model","created":1776938977,"type":"image","running":false,"display_name":"GPT Image 2","organization":"OpenAI","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.053,"example_description":"0.006 - 0.165 per image based on size and quality"},"video":0}},{"id":"Qwen/Qwen3.7-Plus","uuid":"endpoint-ddc9fb60-6793-469c-ab42-a6db76013f67","object":"model","created":1781532368,"type":"chat","running":false,"display_name":"Qwen3.7 Plus","organization":"Qwen","context_length":1000000,"config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0.32,"output":1.28,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"alibaba/happyhorse-1.1-t2v","uuid":"endpoint-bae418aa-f3a0-42b7-bf16-25639335bee5","object":"model","created":1782485613,"type":"video","running":false,"display_name":"HappyHorse 1.1 T2V","organization":"Alibaba","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.14,"example_description":"Text to Video at 720P: $0.14/sec and $0.18/sec at 1080p"}}},{"id":"alibaba/happyhorse-1.1-i2v","uuid":"endpoint-1d482f72-1593-4648-949f-09481c618521","object":"model","created":1782485593,"type":"video","running":false,"display_name":"HappyHorse 1.1 I2V","organization":"Alibaba","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.14,"example_description":"Text/Image to Video at 720P: $0.14/sec and $0.18/sec at 1080p"}}},{"id":"alibaba/happyhorse-1.1-r2v","uuid":"endpoint-87cf37d3-6892-40ce-b1ff-56d5aeb80c44","object":"model","created":1782485628,"type":"video","running":false,"display_name":"HappyHorse 1.1 R2V","organization":"Alibaba","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.14,"example_description":"Text/Image to Video at 720P: $0.14/sec and $0.18/sec at 1080p"}}},{"id":"google/flash-image-3.1-lite","uuid":"endpoint-acb856f2-4ab1-440e-ba58-2bd6cea1b536","object":"model","created":1782846618,"type":"image","running":false,"display_name":"Gemini 3.1 Flash-Lite Image (Nano Banana 2 Lite)","organization":"Google","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.069,"example_description":"price per image"},"video":0}},{"id":"Prism-ML/Ternary-Bonsai-27B","uuid":"endpoint-6c5092a2-b920-4be3-9e45-1c5cb7eee78f","object":"model","created":0,"type":"chat","running":false,"display_name":"Ternary Bonsai 27B","organization":"Prism Ml","link":"https://huggingface.co/api/models/prism-ml/Ternary-Bonsai-27B-AWQ-4bit","license":"apache-2.0","context_length":262144,"config":{"chat_template":null,"stop":["<|im_end|>"],"bos_token":null,"eos_token":"<|im_end|>"},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":0}},{"id":"prunaai/p-image-ideogram","uuid":"endpoint-c045bc1c-6174-4d1c-bee0-716fed7e4609","object":"model","created":1785844762,"type":"image","running":false,"display_name":"P-Image-Ideogram","organization":"Pruna AI","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":{"example_price":0.00225,"example_description":"Pricing starts at $0.00225 per image"},"video":0}},{"id":"black-forest-labs/FLUX-3","uuid":"endpoint-bec520ab-d414-4fad-aad8-d801da1cff65","object":"model","created":1785896986,"type":"video","running":false,"display_name":"FLUX 3","organization":"Black Forest Labs","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.17,"example_description":"T2V @ 720p is $0.17/s, T2V @ 1080p is $0.29/s, V2V @720 is $0.43/s, V2V @1080p is $0.54/s"}}},{"id":"ByteDance/Seedance-2.5","uuid":"endpoint-d0ba33d4-1c4e-43db-9f3f-c8a4c2885dad","object":"model","created":1786388202,"type":"video","running":false,"display_name":"ByteDance Seedance 2.5","organization":"ByteDance","config":{"chat_template":null,"stop":[],"bos_token":null,"eos_token":null},"pricing":{"hourly":0,"input":0,"output":0,"base":0,"finetune":0,"image_pixel":0,"transcribe":0,"image":0,"video":{"example_price":0.115,"example_description":"480P: $0.115/sec & 720P: from $0.249/sec"}}}] \ No newline at end of file diff --git a/tests/test_litellm/test_sync_together_ai_models.py b/tests/test_litellm/test_sync_together_ai_models.py new file mode 100644 index 00000000000..747cba87078 --- /dev/null +++ b/tests/test_litellm/test_sync_together_ai_models.py @@ -0,0 +1,350 @@ +import importlib.util +import json +from pathlib import Path +from types import MappingProxyType + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "sync_together_ai_models.py" +FIXTURES = Path(__file__).resolve().parent / "fixtures" / "together_ai_sync" + +_spec = importlib.util.spec_from_file_location("sync_together_ai_models", SCRIPT) +assert _spec is not None and _spec.loader is not None +sync = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(sync) + +RECORDED_CATALOG = sync.load_catalog(FIXTURES.joinpath("models_serverless.json").read_bytes()) +RECORDED_DOC = sync.parse_deprecations(FIXTURES.joinpath("deprecations.md").read_text()) + + +def _doc(removal_dates: dict[str, str], redirects: dict[str, str] | None = None) -> object: + return sync.DeprecationDoc( + removal_dates=MappingProxyType(removal_dates), + redirects=MappingProxyType(redirects or {}), + ) + + +def _chat_model(model_id: str, ctx: int = 4096, price: float = 1.0, cached: float | None = None) -> object: + return sync.CatalogModel( + id=model_id, + type="chat", + context_length=ctx, + pricing=sync.CatalogPricing(input=price, output=price, cached_input=cached), + ) + + +@pytest.mark.parametrize( + ("per_million", "expected"), + [ + (3, 3e-06), + (15, 1.5e-05), + (1.4, 1.4e-06), + (0.25999999999999995, 2.6e-07), + (0.060000000000000005, 6e-08), + (1.0399999999999998, 1.04e-06), + (0, 0.0), + ], +) +def test_per_token_normalizes_float_artifacts(per_million: float, expected: float) -> None: + assert sync.per_token(per_million) == expected + + +def test_parse_deprecations_recorded_fixture() -> None: + assert dict(RECORDED_DOC.redirects) == { + "mistralai/Mistral-7B-Instruct-v0.3": "mistralai/Ministral-3-14B-Instruct-2512", + "Kimi-K2": "Kimi-K2-0905", + "DeepSeek-V3": "DeepSeek-V3.1", + "DeepSeek-V3-0324": "DeepSeek-V3.1", + "DeepSeek-R1": "DeepSeek-R1-0528", + } + assert len(RECORDED_DOC.removal_dates) == 208 + assert RECORDED_DOC.removal_dates["google/gemma-3n-E4B-it"] == "2026-08-04" + + +def test_parse_deprecations_duplicate_rows_keep_most_recent_date() -> None: + assert RECORDED_DOC.removal_dates["Qwen/Qwen3-235B-A22B-Thinking-2507"] == "2026-04-16" + + +@pytest.mark.parametrize( + "markdown", + [ + "# Deprecations\n\nNothing here anymore.\n", + "\n## Active model redirects\n\n| A | B |\n| --- | --- |\n| `x` | `y` |\n\n## Something else\n", + "\n## Deprecation history\n\n### Inference\n\n| Date | Model | R |\n| --- | --- | --- |\n| 2026-01-01 | `m` | No |\n", + ], +) +def test_parse_deprecations_raises_when_a_table_parses_empty(markdown: str) -> None: + with pytest.raises(sync.SyncError): + sync.parse_deprecations(markdown) + + +def test_load_catalog_raises_on_shape_change() -> None: + with pytest.raises(sync.SyncError): + sync.load_catalog(b'[{"id": "x", "type": "chat"}]') + + +def test_load_catalog_raises_when_no_token_models_remain() -> None: + only_video = json.dumps([{"id": "v", "type": "video", "pricing": {"input": 0, "output": 0}}]).encode() + with pytest.raises(sync.SyncError): + sync.load_catalog(only_video) + + +def test_recorded_catalog_counts() -> None: + assert len(RECORDED_CATALOG) == 102 + assert sum(1 for model in RECORDED_CATALOG if model.type in sync.TYPE_TO_MODE) == 26 + assert sum(1 for model in RECORDED_CATALOG if model.pricing.cached_input) == 13 + + +def test_added_chat_model_matches_reviewed_registry_shape() -> None: + outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) + assert len(outcome.added) == 26 + assert not outcome.deprecated + assert outcome.cost_map["together_ai/moonshotai/Kimi-K3"] == { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "together_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://docs.together.ai/docs/serverless-models", + "supports_function_calling": True, + "supports_parallel_function_calling": True, + "supports_prompt_caching": True, + "supports_reasoning": True, + "supports_response_schema": True, + "supports_tool_choice": True, + "supports_vision": True, + } + + +def test_added_embedding_model_has_no_output_token_cap() -> None: + outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) + assert outcome.cost_map["together_ai/intfloat/multilingual-e5-large-instruct"] == { + "input_cost_per_token": 2e-08, + "litellm_provider": "together_ai", + "max_input_tokens": 514, + "max_tokens": 514, + "mode": "embedding", + "output_cost_per_token": 2e-08, + "output_vector_size": 1024, + "source": "https://docs.together.ai/docs/serverless-models", + } + + +def test_moderation_type_maps_to_chat_mode() -> None: + outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) + guard = outcome.cost_map["together_ai/meta-llama/Llama-Guard-4-12B"] + assert guard["mode"] == "chat" + assert guard["max_output_tokens"] == 1048576 + + +def test_docs_removed_but_live_model_stays_live_with_warning() -> None: + outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) + gemma = outcome.cost_map["together_ai/google/gemma-3n-E4B-it"] + assert "deprecation_date" not in gemma + assert any("gemma-3n-E4B-it" in warning and "2026-08-04" in warning for warning in outcome.warnings) + + +def test_price_change_updates_api_fields_and_keeps_curated_ones() -> None: + registry = { + "together_ai/acme/chat-1": { + "input_cost_per_token": 9e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 4096, + "max_output_tokens": 2048, + "max_tokens": 2048, + "mode": "chat", + "output_cost_per_token": 9e-07, + "supports_audio_input": True, + } + } + outcome = sync.compute_sync(registry, [_chat_model("acme/chat-1", ctx=8192, price=2.0)], _doc({"x": "2026-01-01"})) + entry = outcome.cost_map["together_ai/acme/chat-1"] + assert entry["input_cost_per_token"] == 2e-06 + assert entry["max_input_tokens"] == 8192 + assert entry["max_output_tokens"] == 2048 + assert entry["supports_audio_input"] is True + assert len(outcome.updated) == 1 + assert "input_cost_per_token" in outcome.updated[0] + + +def test_cached_input_appearing_and_disappearing() -> None: + registry = { + "together_ai/acme/chat-1": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "together_ai", + "max_input_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1e-06, + "supports_prompt_caching": True, + }, + "together_ai/acme/chat-2": { + "input_cost_per_token": 1e-06, + "litellm_provider": "together_ai", + "max_input_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1e-06, + }, + } + catalog = [_chat_model("acme/chat-1"), _chat_model("acme/chat-2", cached=0.25999999999999995)] + outcome = sync.compute_sync(registry, catalog, _doc({"x": "2026-01-01"})) + assert "cache_read_input_token_cost" not in outcome.cost_map["together_ai/acme/chat-1"] + assert outcome.cost_map["together_ai/acme/chat-2"]["cache_read_input_token_cost"] == 2.6e-07 + assert outcome.cost_map["together_ai/acme/chat-2"]["supports_prompt_caching"] is True + + +def test_capability_rule_backfills_existing_entry() -> None: + registry = { + "together_ai/moonshotai/Kimi-K3": { + "input_cost_per_token": 3e-06, + "litellm_provider": "together_ai", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + } + } + kimi = next(model for model in RECORDED_CATALOG if model.id == "moonshotai/Kimi-K3") + outcome = sync.compute_sync(registry, [kimi], _doc({"x": "2026-01-01"})) + assert outcome.cost_map["together_ai/moonshotai/Kimi-K3"]["supports_reasoning"] is True + assert any("supports_reasoning" in line for line in outcome.updated) + + +def test_disappeared_model_gets_docs_date_and_is_never_deleted() -> None: + registry = { + "together_ai/acme/gone": { + "input_cost_per_token": 1e-06, + "litellm_provider": "together_ai", + "mode": "chat", + "output_cost_per_token": 1e-06, + } + } + outcome = sync.compute_sync(registry, [_chat_model("acme/alive")], _doc({"acme/gone": "2026-07-01"})) + assert outcome.cost_map["together_ai/acme/gone"]["deprecation_date"] == "2026-07-01" + assert outcome.deprecated == ("together_ai/acme/gone: deprecation_date",) + + +def test_disappeared_model_without_docs_date_warns_instead() -> None: + registry = { + "together_ai/acme/gone": { + "input_cost_per_token": 1e-06, + "litellm_provider": "together_ai", + "mode": "chat", + "output_cost_per_token": 1e-06, + } + } + outcome = sync.compute_sync(registry, [_chat_model("acme/alive")], _doc({"other": "2026-07-01"})) + assert "deprecation_date" not in outcome.cost_map["together_ai/acme/gone"] + assert not outcome.deprecated + assert any("acme/gone" in warning and "human" in warning for warning in outcome.warnings) + + +def test_curated_deprecation_date_is_never_overwritten() -> None: + registry = { + "together_ai/acme/gone": { + "deprecation_date": "2026-06-15", + "input_cost_per_token": 1e-06, + "litellm_provider": "together_ai", + "mode": "chat", + "output_cost_per_token": 1e-06, + } + } + outcome = sync.compute_sync(registry, [_chat_model("acme/alive")], _doc({"acme/gone": "2026-07-01"})) + assert outcome.cost_map["together_ai/acme/gone"]["deprecation_date"] == "2026-06-15" + assert any("2026-06-15" in warning and "2026-07-01" in warning for warning in outcome.warnings) + + +def test_redirect_chain_resolves_to_final_live_model() -> None: + doc = _doc({"acme/a": "2026-01-01"}, redirects={"acme/a": "acme/b", "acme/b": "acme/c"}) + live = frozenset({"acme/c"}) + assert sync.resolve_successor("acme/a", doc, live) == "acme/c" + + +def test_redirect_dead_end_yields_no_successor() -> None: + doc = _doc({"acme/a": "2026-01-01"}, redirects={"acme/a": "acme/b"}) + assert sync.resolve_successor("acme/a", doc, frozenset({"acme/other"})) is None + + +def test_redirect_short_names_resolve_by_unique_suffix() -> None: + doc = _doc({"moonshotai/Kimi-K2": "2026-01-01"}, redirects={"Kimi-K2": "Kimi-K2-0905"}) + live = frozenset({"moonshotai/Kimi-K2-0905"}) + assert sync.resolve_successor("moonshotai/Kimi-K2", doc, live) == "moonshotai/Kimi-K2-0905" + + +def test_successor_written_only_when_not_curated() -> None: + registry = { + "together_ai/acme/a": { + "input_cost_per_token": 1e-06, + "litellm_provider": "together_ai", + "mode": "chat", + "output_cost_per_token": 1e-06, + }, + "together_ai/acme/b": { + "input_cost_per_token": 1e-06, + "litellm_provider": "together_ai", + "metadata": {"successor": "together_ai/acme/curated"}, + "mode": "chat", + "output_cost_per_token": 1e-06, + }, + } + doc = _doc({"acme/a": "2026-01-01", "acme/b": "2026-01-01"}, redirects={"acme/a": "acme/c", "acme/b": "acme/c"}) + outcome = sync.compute_sync(registry, [_chat_model("acme/c")], doc) + assert outcome.cost_map["together_ai/acme/a"]["metadata"] == {"successor": "together_ai/acme/c"} + assert outcome.cost_map["together_ai/acme/b"]["metadata"] == {"successor": "together_ai/acme/curated"} + assert any("acme/curated" in warning for warning in outcome.warnings) + + +def test_reappearance_clears_deprecation_date() -> None: + registry = { + "together_ai/acme/back": { + "deprecation_date": "2026-05-01", + "input_cost_per_token": 1e-06, + "litellm_provider": "together_ai", + "max_input_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1e-06, + } + } + outcome = sync.compute_sync(registry, [_chat_model("acme/back")], _doc({"x": "2026-01-01"})) + assert "deprecation_date" not in outcome.cost_map["together_ai/acme/back"] + assert outcome.reappeared == ("together_ai/acme/back",) + + +def test_new_chat_model_without_rule_is_flagged() -> None: + outcome = sync.compute_sync({}, [_chat_model("acme/unreviewed")], _doc({"x": "2026-01-01"})) + assert any("acme/unreviewed" in warning and "capability rule" in warning for warning in outcome.warnings) + + +def test_new_keys_land_at_the_end_of_the_provider_block() -> None: + registry = { + "aaa": {"mode": "chat"}, + "together_ai/acme/old": { + "input_cost_per_token": 1e-06, + "litellm_provider": "together_ai", + "mode": "chat", + "output_cost_per_token": 1e-06, + }, + "zzz": {"mode": "chat"}, + } + outcome = sync.compute_sync(registry, [_chat_model("acme/old"), _chat_model("acme/new")], _doc({"x": "2026-01-01"})) + assert list(outcome.cost_map) == ["aaa", "together_ai/acme/old", "together_ai/acme/new", "zzz"] + + +def test_sync_is_idempotent_over_the_repo_cost_map() -> None: + cost_map = json.loads((ROOT / "model_prices_and_context_window.json").read_text()) + first = sync.compute_sync(cost_map, RECORDED_CATALOG, RECORDED_DOC) + second = sync.compute_sync(first.cost_map, RECORDED_CATALOG, RECORDED_DOC) + assert not second.has_changes + assert second.cost_map == first.cost_map + + +def test_pr_body_lists_every_section_and_the_skipped_types() -> None: + outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) + body = sync.render_pr_body(outcome) + assert "### Added (26)" in body + assert "### Warnings needing a human call" in body + assert "image (29)" in body + assert "video (38)" in body From 6373ea090ef97c5d13f25a941f4b96aeb23317a8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:25:46 -0700 Subject: [PATCH 005/105] fix(scripts): drop supports_prompt_caching when cached pricing leaves the together_ai catalog --- scripts/sync_together_ai_models.py | 2 +- tests/test_litellm/test_sync_together_ai_models.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/sync_together_ai_models.py b/scripts/sync_together_ai_models.py index a7f3f36b555..97b308fb660 100644 --- a/scripts/sync_together_ai_models.py +++ b/scripts/sync_together_ai_models.py @@ -308,7 +308,7 @@ def _new_entry(model: CatalogModel, mode: str) -> RegistryEntry: def _updated_entry(entry: RegistryEntry, model: CatalogModel) -> tuple[RegistryEntry, tuple[str, ...]]: rule: Final = RULES_BY_ID.get(model.id) desired: Final = {**_api_fields(model), **(dict(rule.fields) if rule else {})} - dropped: Final = () if model.pricing.cached_input else ("cache_read_input_token_cost",) + dropped: Final = () if model.pricing.cached_input else ("cache_read_input_token_cost", "supports_prompt_caching") changes: Final = tuple( f"{name}: {entry.get(name)!r} -> {value!r}" for name, value in desired.items() if entry.get(name) != value ) + tuple( diff --git a/tests/test_litellm/test_sync_together_ai_models.py b/tests/test_litellm/test_sync_together_ai_models.py index 747cba87078..7c1287e94b8 100644 --- a/tests/test_litellm/test_sync_together_ai_models.py +++ b/tests/test_litellm/test_sync_together_ai_models.py @@ -193,6 +193,7 @@ def test_cached_input_appearing_and_disappearing() -> None: catalog = [_chat_model("acme/chat-1"), _chat_model("acme/chat-2", cached=0.25999999999999995)] outcome = sync.compute_sync(registry, catalog, _doc({"x": "2026-01-01"})) assert "cache_read_input_token_cost" not in outcome.cost_map["together_ai/acme/chat-1"] + assert "supports_prompt_caching" not in outcome.cost_map["together_ai/acme/chat-1"] assert outcome.cost_map["together_ai/acme/chat-2"]["cache_read_input_token_cost"] == 2.6e-07 assert outcome.cost_map["together_ai/acme/chat-2"]["supports_prompt_caching"] is True From 56c2cceeaa394ce6439cafb07b84666aea7fd715 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:42:28 -0700 Subject: [PATCH 006/105] fix(ci): raise the open-PR listing limit so the sync-PR guard sees every open PR --- .github/workflows/sync-together-ai-models.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-together-ai-models.yml b/.github/workflows/sync-together-ai-models.yml index eca9ad7c968..f1a8a841d0f 100644 --- a/.github/workflows/sync-together-ai-models.yml +++ b/.github/workflows/sync-together-ai-models.yml @@ -25,7 +25,7 @@ jobs: - name: Look for an already-open sync PR id: existing run: | - open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --json headRefName \ + open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --limit 1000 --json headRefName \ --jq '[.[].headRefName | select(startswith("litellm_together_registry_sync_"))] | first // empty')" echo "open_pr=$open_pr" >> "$GITHUB_OUTPUT" if [ -n "$open_pr" ]; then From 0e999e32c96ab2b46b693d34aa8340f7b6a20bab Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:01:20 -0700 Subject: [PATCH 007/105] test: fix staging CI regressions from #38182, #38144, #38265, #37962, and #37969 - test_custom_callback_input: audio redaction assertion expects None content (redaction leaves None untouched, gpt-audio-1.5 returns content=None) - local_testing conftest: drain GLOBAL_LOGGING_WORKER in isolate_litellm_state teardown so mocked-router tests stop leaking pending logging tasks into test_gcs_pub_sub - test_together_ai: tools is always a supported param now; only response_format is gated by function-calling support - test_keys: /team/new omits models instead of sending null (422), so the key's team really exists and auth no longer raises TeamNotFoundError - test_team_delete_member_add_race: per-test unique team and user ids so xdist workers sharing one Postgres stop deleting each other's team mid-race --- tests/llm_translation/test_together_ai.py | 9 +- tests/local_testing/conftest.py | 3 + .../test_custom_callback_input.py | 9 +- .../test_team_delete_member_add_race.py | 91 ++++++++++--------- tests/test_keys.py | 3 +- 5 files changed, 64 insertions(+), 51 deletions(-) diff --git a/tests/llm_translation/test_together_ai.py b/tests/llm_translation/test_together_ai.py index c371caefa5e..85f6eb29db2 100644 --- a/tests/llm_translation/test_together_ai.py +++ b/tests/llm_translation/test_together_ai.py @@ -23,14 +23,14 @@ class TestTogetherAI(BaseLLMChatTest): pass @pytest.mark.parametrize( - "model, expected_bool", + "model, supports_response_format", [ ("meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", True), ("nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", False), ], ) def test_get_supported_response_format_together_ai( - self, model: str, expected_bool: bool + self, model: str, supports_response_format: bool ) -> None: os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") @@ -40,9 +40,8 @@ class TestTogetherAI(BaseLLMChatTest): # Mapped provider assert isinstance(optional_params, list) - if expected_bool: + assert "tools" in optional_params + if supports_response_format: assert "response_format" in optional_params - assert "tools" in optional_params else: assert "response_format" not in optional_params - assert "tools" not in optional_params diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py index 4f142664827..ee93009a198 100644 --- a/tests/local_testing/conftest.py +++ b/tests/local_testing/conftest.py @@ -11,12 +11,14 @@ # these true defaults before every test, preventing cross-test contamination # under xdist where module reload is skipped. +import asyncio import importlib import os import pytest import litellm +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER # ``litellm.model_cost`` is loaded at import time from the URL pinned to ``main`` # (``LITELLM_MODEL_COST_MAP_URL``). The in-tree backup ships with this branch @@ -220,6 +222,7 @@ def isolate_litellm_state(): yield # ---- Teardown: restore saved state ---- + asyncio.run(GLOBAL_LOGGING_WORKER.clear_queue()) if hasattr(litellm, "in_memory_llm_clients_cache"): litellm.in_memory_llm_clients_cache.flush_cache() diff --git a/tests/local_testing/test_custom_callback_input.py b/tests/local_testing/test_custom_callback_input.py index 745bfe94e1a..f0f24a6e6b2 100644 --- a/tests/local_testing/test_custom_callback_input.py +++ b/tests/local_testing/test_custom_callback_input.py @@ -1236,10 +1236,11 @@ def test_standard_logging_payload_audio(turn_off_message_logging, stream): assert "redacted-by-litellm" == slobject["messages"][0]["content"] response = slobject["response"] if "choices" in response: - assert ( - response["choices"][0]["message"]["content"] - == "redacted-by-litellm" - ) + redacted_content = response["choices"][0]["message"]["content"] + if stream: + assert redacted_content == "redacted-by-litellm" + else: + assert redacted_content is None assert response["choices"][0]["message"].get("audio") is None else: assert response["text"] == "redacted-by-litellm" diff --git a/tests/proxy_admin_ui_tests/test_team_delete_member_add_race.py b/tests/proxy_admin_ui_tests/test_team_delete_member_add_race.py index 30544a8bb81..7577570be48 100644 --- a/tests/proxy_admin_ui_tests/test_team_delete_member_add_race.py +++ b/tests/proxy_admin_ui_tests/test_team_delete_member_add_race.py @@ -17,6 +17,7 @@ test is provably blocked on it rather than hoping a sleep lands in the right gap import asyncio import json import os +import uuid from contextlib import asynccontextmanager from datetime import timedelta from unittest.mock import MagicMock @@ -34,16 +35,21 @@ from litellm.proxy._types import ( from litellm.caching.caching import DualCache from litellm.proxy.utils import PrismaClient, ProxyLogging -TEAM = "lit5544-race-team" -USER = "lit5544-race-user" _DELETE_SEEDED = 'DELETE FROM "LiteLLM_TeamMembership" WHERE team_id = $1' _DELETE_USER = 'DELETE FROM "LiteLLM_UserTable" WHERE user_id = $1' _DELETE_TEAM = 'DELETE FROM "LiteLLM_TeamTable" WHERE team_id = $1' _LOCK_SQL = "SELECT pg_advisory_xact_lock(hashtext($1)) IS NULL AS locked" +def _race_ids() -> tuple[str, str]: + """Unique per test: xdist workers share one Postgres, so a shared id lets one worker's + cleanup delete the team another worker is mid-race on.""" + suffix = uuid.uuid4().hex[:8] + return f"lit5544-race-team-{suffix}", f"lit5544-race-user-{suffix}" + + @asynccontextmanager -async def _clean_db(): +async def _clean_db(team_id: str, user_id: str): """Connects inside the running test's loop: an async fixture would be torn up on a different loop than the test body, which prisma's engine lock refuses outright.""" from prisma import Prisma @@ -54,14 +60,14 @@ async def _clean_db(): db = Prisma() await db.connect() try: - await db.execute_raw(_DELETE_SEEDED, TEAM) - await db.execute_raw(_DELETE_USER, USER) - await db.execute_raw(_DELETE_TEAM, TEAM) + await db.execute_raw(_DELETE_SEEDED, team_id) + await db.execute_raw(_DELETE_USER, user_id) + await db.execute_raw(_DELETE_TEAM, team_id) yield db finally: - await db.execute_raw(_DELETE_SEEDED, TEAM) - await db.execute_raw(_DELETE_USER, USER) - await db.execute_raw(_DELETE_TEAM, TEAM) + await db.execute_raw(_DELETE_SEEDED, team_id) + await db.execute_raw(_DELETE_USER, user_id) + await db.execute_raw(_DELETE_TEAM, team_id) await db.disconnect() @@ -95,8 +101,9 @@ async def test_member_add_blocked_by_delete_writes_no_dangling_reference(): _add_team_members_to_team, ) - async with _clean_db() as db: - await db.litellm_teamtable.create(data={"team_id": TEAM, "team_alias": TEAM, "members_with_roles": "[]"}) + team_id, user_id = _race_ids() + async with _clean_db(team_id, user_id) as db: + await db.litellm_teamtable.create(data={"team_id": team_id, "team_alias": team_id, "members_with_roles": "[]"}) async with _real_prisma_client() as prisma_client: from prisma import Prisma @@ -109,11 +116,11 @@ async def test_member_add_blocked_by_delete_writes_no_dangling_reference(): lock_acquired.set() await _add_team_members_to_team( data=TeamMemberAddRequest( - team_id=TEAM, - member=Member(user_id=USER, role="user"), + team_id=team_id, + member=Member(user_id=user_id, role="user"), max_budget_in_team=5.0, ), - complete_team_data=LiteLLM_TeamTable(team_id=TEAM, members_with_roles=[]), + complete_team_data=LiteLLM_TeamTable(team_id=team_id, members_with_roles=[]), prisma_client=prisma_client, user_api_key_dict=_admin_auth(), litellm_proxy_admin_name="lit5544-admin", @@ -121,14 +128,14 @@ async def test_member_add_blocked_by_delete_writes_no_dangling_reference(): try: async with blocker.tx(timeout=timedelta(seconds=30)) as held: - await held.query_raw(_LOCK_SQL, TEAM) + await held.query_raw(_LOCK_SQL, team_id) task = asyncio.create_task(add_member()) await lock_acquired.wait() await asyncio.sleep(0.2) assert not task.done(), "member_add did not wait on the team's advisory lock" # the delete wins the race: strip the team row while the lock is held - await held.execute_raw(_DELETE_TEAM, TEAM) + await held.execute_raw(_DELETE_TEAM, team_id) with pytest.raises(HTTPException) as exc_info: await asyncio.wait_for(task, timeout=30) @@ -136,10 +143,10 @@ async def test_member_add_blocked_by_delete_writes_no_dangling_reference(): finally: await blocker.disconnect() - user_row = await db.litellm_usertable.find_unique(where={"user_id": USER}) + user_row = await db.litellm_usertable.find_unique(where={"user_id": user_id}) assert user_row is None, "member_add must not have written a user row for a team that was gone under its lock" - membership_row = await db.litellm_teammembership.find_first(where={"team_id": TEAM, "user_id": USER}) + membership_row = await db.litellm_teammembership.find_first(where={"team_id": team_id, "user_id": user_id}) assert membership_row is None @@ -156,16 +163,17 @@ async def test_member_delete_blocked_by_member_add_removes_from_the_fresh_roster from litellm.proxy._types import TeamMemberDeleteRequest from litellm.proxy.management_endpoints.team_endpoints import team_member_delete - other_user = f"{USER}-other" - seeded_roster = '[{"user_id": "%s", "user_email": null, "role": "user"}]' % USER + team_id, user_id = _race_ids() + other_user = f"{user_id}-other" + seeded_roster = '[{"user_id": "%s", "user_email": null, "role": "user"}]' % user_id winning_add_roster = ( '[{"user_id": "%s", "user_email": null, "role": "user"}, ' - '{"user_id": "%s", "user_email": null, "role": "user"}]' % (USER, other_user) + '{"user_id": "%s", "user_email": null, "role": "user"}]' % (user_id, other_user) ) - async with _clean_db() as db: + async with _clean_db(team_id, user_id) as db: await db.litellm_teamtable.create( - data={"team_id": TEAM, "team_alias": TEAM, "members_with_roles": seeded_roster} + data={"team_id": team_id, "team_alias": team_id, "members_with_roles": seeded_roster} ) async with _real_prisma_client() as prisma_client: @@ -182,13 +190,13 @@ async def test_member_delete_blocked_by_member_add_removes_from_the_fresh_roster async def run_delete(): lock_acquired.set() return await team_member_delete( - data=TeamMemberDeleteRequest(team_id=TEAM, user_id=USER), + data=TeamMemberDeleteRequest(team_id=team_id, user_id=user_id), user_api_key_dict=_admin_auth(), ) try: async with blocker.tx(timeout=timedelta(seconds=30)) as held: - await held.query_raw(_LOCK_SQL, TEAM) + await held.query_raw(_LOCK_SQL, team_id) task = asyncio.create_task(run_delete()) await lock_acquired.wait() await asyncio.sleep(0.2) @@ -196,7 +204,7 @@ async def test_member_delete_blocked_by_member_add_removes_from_the_fresh_roster # member_add wins the race: it adds `other_user` while holding the lock await held.litellm_teamtable.update( - where={"team_id": TEAM}, + where={"team_id": team_id}, data={"members_with_roles": winning_add_roster}, ) @@ -206,7 +214,7 @@ async def test_member_delete_blocked_by_member_add_removes_from_the_fresh_roster finally: proxy_server_module.prisma_client = original_prisma_client - team_row = await db.litellm_teamtable.find_unique(where={"team_id": TEAM}) + team_row = await db.litellm_teamtable.find_unique(where={"team_id": team_id}) raw_roster = team_row.members_with_roles parsed_roster = json.loads(raw_roster) if isinstance(raw_roster, str) else raw_roster remaining_ids = {m["user_id"] for m in parsed_roster} @@ -228,8 +236,9 @@ async def test_delete_blocked_by_member_add_sweeps_the_fresh_reference(): from litellm.proxy._types import LiteLLM_TeamTable from litellm.proxy.management_endpoints.team_endpoints import delete_team - async with _clean_db() as db: - await db.litellm_teamtable.create(data={"team_id": TEAM, "team_alias": TEAM, "members_with_roles": "[]"}) + team_id, user_id = _race_ids() + async with _clean_db(team_id, user_id) as db: + await db.litellm_teamtable.create(data={"team_id": team_id, "team_alias": team_id, "members_with_roles": "[]"}) async with _real_prisma_client() as prisma_client: proxy_logging_obj = prisma_client.proxy_logging_obj @@ -261,7 +270,7 @@ async def test_delete_blocked_by_member_add_sweeps_the_fresh_reference(): async def run_delete(): lock_acquired.set() return await delete_team( - data=DeleteTeamRequest(team_ids=[TEAM]), + data=DeleteTeamRequest(team_ids=[team_id]), http_request=MagicMock(), user_api_key_dict=_admin_auth(), litellm_changed_by="lit5544-admin", @@ -269,7 +278,7 @@ async def test_delete_blocked_by_member_add_sweeps_the_fresh_reference(): try: async with blocker.tx(timeout=timedelta(seconds=30)) as held: - await held.query_raw(_LOCK_SQL, TEAM) + await held.query_raw(_LOCK_SQL, team_id) task = asyncio.create_task(run_delete()) await lock_acquired.wait() await asyncio.sleep(0.3) @@ -277,16 +286,16 @@ async def test_delete_blocked_by_member_add_sweeps_the_fresh_reference(): # member_add wins the race: write the reference while holding the lock await held.litellm_usertable.upsert( - where={"user_id": USER}, + where={"user_id": user_id}, data={ - "create": {"user_id": USER, "teams": [TEAM]}, - "update": {"teams": {"push": [TEAM]}}, + "create": {"user_id": user_id, "teams": [team_id]}, + "update": {"teams": {"push": [team_id]}}, }, ) - await held.litellm_teammembership.create(data={"team_id": TEAM, "user_id": USER}) + await held.litellm_teammembership.create(data={"team_id": team_id, "user_id": user_id}) await held.litellm_teamtable.update( - where={"team_id": TEAM}, - data={"members_with_roles": '[{"user_id": "%s", "role": "user"}]' % USER}, + where={"team_id": team_id}, + data={"members_with_roles": '[{"user_id": "%s", "role": "user"}]' % user_id}, ) await asyncio.wait_for(task, timeout=30) @@ -295,13 +304,13 @@ async def test_delete_blocked_by_member_add_sweeps_the_fresh_reference(): finally: await restore() - team_row = await db.litellm_teamtable.find_unique(where={"team_id": TEAM}) + team_row = await db.litellm_teamtable.find_unique(where={"team_id": team_id}) assert team_row is None - user_row = await db.litellm_usertable.find_unique(where={"user_id": USER}) - assert user_row is not None and TEAM not in user_row.teams, ( + user_row = await db.litellm_usertable.find_unique(where={"user_id": user_id}) + assert user_row is not None and team_id not in user_row.teams, ( "delete_team's locked sweep must reap the reference member_add wrote just before losing the lock" ) - membership_row = await db.litellm_teammembership.find_first(where={"team_id": TEAM, "user_id": USER}) + membership_row = await db.litellm_teammembership.find_first(where={"team_id": team_id, "user_id": user_id}) assert membership_row is None diff --git a/tests/test_keys.py b/tests/test_keys.py index e39c715de03..7a5b2502cfd 100644 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -19,7 +19,7 @@ async def generate_team( headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"} if team_id is None: team_id = "litellm-dashboard" - data = {"team_id": team_id, "models": models} + data = {"team_id": team_id, **({"models": models} if models is not None else {})} async with session.post(url, headers=headers, json=data) as response: status = response.status @@ -810,6 +810,7 @@ async def test_key_model_list(model_access, model_access_level, model_endpoint): models=_models if model_access_level == "team" else None, team_id=team_id, ) + assert new_team["team_id"] == team_id key_gen = await generate_key( session=session, i=0, From bfc1dc73a16a098e7703c20ee2857eb28cf97d1f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:11:13 +0000 Subject: [PATCH 008/105] registry audit: add gemini 3.5 transcribe + omni 1.1 flash, xai grok-imagine image models and grok-4.20 aliases, mistral cache-read pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 319 +++++++++++++++++- model_prices_and_context_window.json | 319 +++++++++++++++++- 2 files changed, 628 insertions(+), 10 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7f5e41d45ce..4d7e36c90e1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -22876,6 +22876,34 @@ "supports_vision": true, "tpm": 800000 }, + "gemini/gemini-omni-1.1-flash": { + "input_cost_per_audio_token": 1.5e-06, + "input_cost_per_token": 1.5e-06, + "litellm_provider": "gemini", + "mode": "chat", + "output_cost_per_reasoning_token": 9e-06, + "output_cost_per_token": 9e-06, + "output_cost_per_video_token": 1.75e-05, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text", + "video" + ], + "supports_audio_input": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_video_input": true, + "supports_vision": true + }, "gemini/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, @@ -30683,6 +30711,7 @@ "supports_tool_choice": true }, "mistral/codestral-2508": { + "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, "litellm_provider": "mistral", "max_input_tokens": 128000, @@ -30697,6 +30726,7 @@ "supports_tool_choice": true }, "mistral/codestral-latest": { + "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, "litellm_provider": "mistral", "max_input_tokens": 128000, @@ -30704,11 +30734,11 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", - "supports_function_calling": true + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "mistral/codestral-mamba-latest": { "input_cost_per_token": 2.5e-07, @@ -31042,6 +31072,7 @@ "mode": "embedding" }, "mistral/codestral-embed": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, @@ -31049,6 +31080,7 @@ "mode": "embedding" }, "mistral/codestral-embed-2505": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, @@ -31098,6 +31130,7 @@ "supports_tool_choice": true }, "mistral/mistral-large-latest": { + "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 5e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31113,6 +31146,7 @@ "supports_vision": true }, "mistral/mistral-large-3": { + "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 5e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31128,6 +31162,7 @@ "supports_vision": true }, "mistral/mistral-large-2512": { + "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 5e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31198,6 +31233,7 @@ "supports_vision": true }, "mistral/mistral-medium-2604": { + "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31214,6 +31250,7 @@ "supports_vision": true }, "mistral/mistral-medium-latest": { + "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31246,6 +31283,7 @@ "supports_vision": true }, "mistral/mistral-medium-3-5": { + "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31275,6 +31313,7 @@ "supports_tool_choice": true }, "mistral/mistral-small-latest": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31285,9 +31324,9 @@ "source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_reasoning": true, "supports_vision": true }, "mistral/mistral-small-3-2-2506": { @@ -31307,6 +31346,7 @@ "supports_vision": true }, "mistral/ministral-3-3b-2512": { + "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, "litellm_provider": "mistral", "max_input_tokens": 131072, @@ -31322,6 +31362,7 @@ "supports_vision": true }, "mistral/ministral-3-8b-2512": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31337,6 +31378,7 @@ "supports_vision": true }, "mistral/ministral-3-14b-2512": { + "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31352,6 +31394,7 @@ "supports_vision": true }, "mistral/ministral-8b-2512": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31367,6 +31410,7 @@ "supports_vision": true }, "mistral/ministral-8b-latest": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -44080,6 +44124,69 @@ "supports_prompt_caching": true, "supports_response_schema": true }, + "xai/grok-4.20": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-reasoning-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "supports_prompt_caching": true, + "supports_response_schema": true + }, "xai/grok-4.20-beta-0309-non-reasoning": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.25e-06, @@ -44281,6 +44388,125 @@ "supports_vision": true, "deprecation_date": "2026-05-15" }, + "xai/grok-imagine-image": { + "input_cost_per_image": 0.002, + "litellm_provider": "xai", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/images/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "xai/grok-imagine-image-2026-03-02": { + "input_cost_per_image": 0.002, + "litellm_provider": "xai", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/images/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "xai/grok-imagine-image-quality": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/images/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "xai/grok-imagine-image-quality-20260403": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/images/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "xai/grok-imagine-image-quality-latest": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/images/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "xai/grok-imagine-image-pro": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/images/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "xai/grok-imagine-image-2.0": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/images/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, "xai/grok-vision-beta": { "input_cost_per_image": 5e-06, "input_cost_per_token": 5e-06, @@ -50957,6 +51183,46 @@ "cache_read_input_token_cost_above_200k_tokens": 4e-07, "supports_response_schema": true }, + "xai/grok-4.20-non-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "supports_response_schema": true + }, + "xai/grok-4.20-non-reasoning-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "supports_response_schema": true + }, "xai/grok-4.20-multi-agent-0309": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.25e-06, @@ -50978,6 +51244,48 @@ "cache_read_input_token_cost_above_200k_tokens": 4e-07, "supports_response_schema": true }, + "xai/grok-4.20-multi-agent": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "supports_response_schema": true + }, + "xai/grok-4.20-multi-agent-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "supports_response_schema": true + }, "xai/grok-build-0.1": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1e-06, @@ -51161,6 +51469,7 @@ "web_search_billing_unit": "per_query" }, "mistral/mistral-small-2603": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7f5e41d45ce..4d7e36c90e1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -22876,6 +22876,34 @@ "supports_vision": true, "tpm": 800000 }, + "gemini/gemini-omni-1.1-flash": { + "input_cost_per_audio_token": 1.5e-06, + "input_cost_per_token": 1.5e-06, + "litellm_provider": "gemini", + "mode": "chat", + "output_cost_per_reasoning_token": 9e-06, + "output_cost_per_token": 9e-06, + "output_cost_per_video_token": 1.75e-05, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text", + "video" + ], + "supports_audio_input": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_video_input": true, + "supports_vision": true + }, "gemini/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, @@ -30683,6 +30711,7 @@ "supports_tool_choice": true }, "mistral/codestral-2508": { + "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, "litellm_provider": "mistral", "max_input_tokens": 128000, @@ -30697,6 +30726,7 @@ "supports_tool_choice": true }, "mistral/codestral-latest": { + "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, "litellm_provider": "mistral", "max_input_tokens": 128000, @@ -30704,11 +30734,11 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", - "supports_function_calling": true + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "mistral/codestral-mamba-latest": { "input_cost_per_token": 2.5e-07, @@ -31042,6 +31072,7 @@ "mode": "embedding" }, "mistral/codestral-embed": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, @@ -31049,6 +31080,7 @@ "mode": "embedding" }, "mistral/codestral-embed-2505": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, @@ -31098,6 +31130,7 @@ "supports_tool_choice": true }, "mistral/mistral-large-latest": { + "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 5e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31113,6 +31146,7 @@ "supports_vision": true }, "mistral/mistral-large-3": { + "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 5e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31128,6 +31162,7 @@ "supports_vision": true }, "mistral/mistral-large-2512": { + "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 5e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31198,6 +31233,7 @@ "supports_vision": true }, "mistral/mistral-medium-2604": { + "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31214,6 +31250,7 @@ "supports_vision": true }, "mistral/mistral-medium-latest": { + "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31246,6 +31283,7 @@ "supports_vision": true }, "mistral/mistral-medium-3-5": { + "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31275,6 +31313,7 @@ "supports_tool_choice": true }, "mistral/mistral-small-latest": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31285,9 +31324,9 @@ "source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_reasoning": true, "supports_vision": true }, "mistral/mistral-small-3-2-2506": { @@ -31307,6 +31346,7 @@ "supports_vision": true }, "mistral/ministral-3-3b-2512": { + "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, "litellm_provider": "mistral", "max_input_tokens": 131072, @@ -31322,6 +31362,7 @@ "supports_vision": true }, "mistral/ministral-3-8b-2512": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31337,6 +31378,7 @@ "supports_vision": true }, "mistral/ministral-3-14b-2512": { + "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31352,6 +31394,7 @@ "supports_vision": true }, "mistral/ministral-8b-2512": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -31367,6 +31410,7 @@ "supports_vision": true }, "mistral/ministral-8b-latest": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, @@ -44080,6 +44124,69 @@ "supports_prompt_caching": true, "supports_response_schema": true }, + "xai/grok-4.20": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-reasoning-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "supports_prompt_caching": true, + "supports_response_schema": true + }, "xai/grok-4.20-beta-0309-non-reasoning": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.25e-06, @@ -44281,6 +44388,125 @@ "supports_vision": true, "deprecation_date": "2026-05-15" }, + "xai/grok-imagine-image": { + "input_cost_per_image": 0.002, + "litellm_provider": "xai", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/images/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "xai/grok-imagine-image-2026-03-02": { + "input_cost_per_image": 0.002, + "litellm_provider": "xai", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/images/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "xai/grok-imagine-image-quality": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/images/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "xai/grok-imagine-image-quality-20260403": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/images/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "xai/grok-imagine-image-quality-latest": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/images/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "xai/grok-imagine-image-pro": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/images/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "xai/grok-imagine-image-2.0": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/images/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, "xai/grok-vision-beta": { "input_cost_per_image": 5e-06, "input_cost_per_token": 5e-06, @@ -50957,6 +51183,46 @@ "cache_read_input_token_cost_above_200k_tokens": 4e-07, "supports_response_schema": true }, + "xai/grok-4.20-non-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "supports_response_schema": true + }, + "xai/grok-4.20-non-reasoning-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "supports_response_schema": true + }, "xai/grok-4.20-multi-agent-0309": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.25e-06, @@ -50978,6 +51244,48 @@ "cache_read_input_token_cost_above_200k_tokens": 4e-07, "supports_response_schema": true }, + "xai/grok-4.20-multi-agent": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "supports_response_schema": true + }, + "xai/grok-4.20-multi-agent-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "supports_response_schema": true + }, "xai/grok-build-0.1": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1e-06, @@ -51161,6 +51469,7 @@ "web_search_billing_unit": "per_query" }, "mistral/mistral-small-2603": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 262144, From 90073864ece70a50155dfd405315f6b7dba87287 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:21:21 +0000 Subject: [PATCH 009/105] fix(registry): add tpm/rpm to new gemini entries per repo convention Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 12 +++++++----- model_prices_and_context_window.json | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4d7e36c90e1..fc10b61b7cd 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -22884,6 +22884,7 @@ "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, + "rpm": 2000, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions" @@ -22902,7 +22903,8 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "tpm": 800000 }, "gemini/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, @@ -51655,6 +51657,7 @@ "litellm_provider": "gemini", "mode": "audio_transcription", "output_cost_per_token": 1.2e-05, + "rpm": 10, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/audio/transcriptions" @@ -51667,8 +51670,7 @@ "text" ], "supports_audio_input": true, - "tpm": 800000, - "rpm": 2000 + "tpm": 250000 }, "gemini/gemini-3.5-transcribe-live": { "input_cost_per_audio_token": 3.5e-06, @@ -51676,6 +51678,7 @@ "litellm_provider": "gemini", "mode": "audio_transcription", "output_cost_per_token": 2.1e-05, + "rpm": 10, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/realtime" @@ -51687,8 +51690,7 @@ "text" ], "supports_audio_input": true, - "tpm": 250000, - "rpm": 10 + "tpm": 250000 }, "perplexity/pplx-embed-context-v1-0.6b": { "input_cost_per_token": 8e-09, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4d7e36c90e1..fc10b61b7cd 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -22884,6 +22884,7 @@ "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, + "rpm": 2000, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions" @@ -22902,7 +22903,8 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "tpm": 800000 }, "gemini/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, @@ -51655,6 +51657,7 @@ "litellm_provider": "gemini", "mode": "audio_transcription", "output_cost_per_token": 1.2e-05, + "rpm": 10, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/audio/transcriptions" @@ -51667,8 +51670,7 @@ "text" ], "supports_audio_input": true, - "tpm": 800000, - "rpm": 2000 + "tpm": 250000 }, "gemini/gemini-3.5-transcribe-live": { "input_cost_per_audio_token": 3.5e-06, @@ -51676,6 +51678,7 @@ "litellm_provider": "gemini", "mode": "audio_transcription", "output_cost_per_token": 2.1e-05, + "rpm": 10, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/realtime" @@ -51687,8 +51690,7 @@ "text" ], "supports_audio_input": true, - "tpm": 250000, - "rpm": 10 + "tpm": 250000 }, "perplexity/pplx-embed-context-v1-0.6b": { "input_cost_per_token": 8e-09, From afe61aa0c8bafb6ffb1e4b6a902f96ed8bbd6964 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:50:00 +0000 Subject: [PATCH 010/105] fix(registry): price xai grok-imagine generated images via input_cost_per_image for default calculator Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 21 +++++++------------ model_prices_and_context_window.json | 21 +++++++------------ 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b21e8a60599..78863dc5cca 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -54171,10 +54171,9 @@ "supports_response_schema": true }, "xai/grok-imagine-image": { - "input_cost_per_image": 0.002, + "input_cost_per_image": 0.02, "litellm_provider": "xai", "mode": "image_generation", - "output_cost_per_image": 0.02, "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ "/v1/images/generations" @@ -54188,10 +54187,9 @@ ] }, "xai/grok-imagine-image-2026-03-02": { - "input_cost_per_image": 0.002, + "input_cost_per_image": 0.02, "litellm_provider": "xai", "mode": "image_generation", - "output_cost_per_image": 0.02, "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ "/v1/images/generations" @@ -54205,10 +54203,9 @@ ] }, "xai/grok-imagine-image-quality": { - "input_cost_per_image": 0.01, + "input_cost_per_image": 0.05, "litellm_provider": "xai", "mode": "image_generation", - "output_cost_per_image": 0.05, "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ "/v1/images/generations" @@ -54222,10 +54219,9 @@ ] }, "xai/grok-imagine-image-quality-20260403": { - "input_cost_per_image": 0.01, + "input_cost_per_image": 0.05, "litellm_provider": "xai", "mode": "image_generation", - "output_cost_per_image": 0.05, "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ "/v1/images/generations" @@ -54239,10 +54235,9 @@ ] }, "xai/grok-imagine-image-quality-latest": { - "input_cost_per_image": 0.01, + "input_cost_per_image": 0.05, "litellm_provider": "xai", "mode": "image_generation", - "output_cost_per_image": 0.05, "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ "/v1/images/generations" @@ -54256,10 +54251,9 @@ ] }, "xai/grok-imagine-image-pro": { - "input_cost_per_image": 0.01, + "input_cost_per_image": 0.05, "litellm_provider": "xai", "mode": "image_generation", - "output_cost_per_image": 0.05, "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ "/v1/images/generations" @@ -54273,10 +54267,9 @@ ] }, "xai/grok-imagine-image-2.0": { - "input_cost_per_image": 0.01, + "input_cost_per_image": 0.06, "litellm_provider": "xai", "mode": "image_generation", - "output_cost_per_image": 0.06, "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ "/v1/images/generations" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b21e8a60599..78863dc5cca 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -54171,10 +54171,9 @@ "supports_response_schema": true }, "xai/grok-imagine-image": { - "input_cost_per_image": 0.002, + "input_cost_per_image": 0.02, "litellm_provider": "xai", "mode": "image_generation", - "output_cost_per_image": 0.02, "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ "/v1/images/generations" @@ -54188,10 +54187,9 @@ ] }, "xai/grok-imagine-image-2026-03-02": { - "input_cost_per_image": 0.002, + "input_cost_per_image": 0.02, "litellm_provider": "xai", "mode": "image_generation", - "output_cost_per_image": 0.02, "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ "/v1/images/generations" @@ -54205,10 +54203,9 @@ ] }, "xai/grok-imagine-image-quality": { - "input_cost_per_image": 0.01, + "input_cost_per_image": 0.05, "litellm_provider": "xai", "mode": "image_generation", - "output_cost_per_image": 0.05, "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ "/v1/images/generations" @@ -54222,10 +54219,9 @@ ] }, "xai/grok-imagine-image-quality-20260403": { - "input_cost_per_image": 0.01, + "input_cost_per_image": 0.05, "litellm_provider": "xai", "mode": "image_generation", - "output_cost_per_image": 0.05, "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ "/v1/images/generations" @@ -54239,10 +54235,9 @@ ] }, "xai/grok-imagine-image-quality-latest": { - "input_cost_per_image": 0.01, + "input_cost_per_image": 0.05, "litellm_provider": "xai", "mode": "image_generation", - "output_cost_per_image": 0.05, "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ "/v1/images/generations" @@ -54256,10 +54251,9 @@ ] }, "xai/grok-imagine-image-pro": { - "input_cost_per_image": 0.01, + "input_cost_per_image": 0.05, "litellm_provider": "xai", "mode": "image_generation", - "output_cost_per_image": 0.05, "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ "/v1/images/generations" @@ -54273,10 +54267,9 @@ ] }, "xai/grok-imagine-image-2.0": { - "input_cost_per_image": 0.01, + "input_cost_per_image": 0.06, "litellm_provider": "xai", "mode": "image_generation", - "output_cost_per_image": 0.06, "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ "/v1/images/generations" From cbac167a4a0f97cc42e0bb14faa6b4b7f4107bbe Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 27 Aug 2026 14:51:40 -0700 Subject: [PATCH 011/105] fix(ui): keep the usage filter visible when the caller's scope is empty The tag usage filter was removed from the DOM whenever the tag list came back empty, so an internal user whose traffic all runs through team keys saw a blank Tag Usage panel with no filter and no explanation. Their scope is legitimately empty, but the page gave them no way to tell that apart from a broken or gated feature. Render the filter whenever the entity list has resolved, disabling it and swapping in an entity-specific empty message when there are no options. A still-unresolved list keeps hiding the control as before. --- .../EntityUsage/EntityUsage.test.tsx | 31 +++++++++++++++- .../components/EntityUsage/EntityUsage.tsx | 2 +- .../UsageExportHeader.test.tsx | 37 +++++++++++++++++++ .../EntityUsageExport/UsageExportHeader.tsx | 10 ++++- 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx index 666172947d1..5bb48a78437 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx @@ -65,10 +65,19 @@ vi.mock("@/components/EntityUsageExport/EntityUsageExportModal", () => ({ })); vi.mock("@/components/EntityUsageExport", () => ({ - UsageExportHeader: ({ filterLabel, filterSlot }: { filterLabel?: string; filterSlot?: ReactNode }) => ( + UsageExportHeader: ({ + filterLabel, + filterSlot, + showFilters, + }: { + filterLabel?: string; + filterSlot?: ReactNode; + showFilters?: boolean; + }) => (
Usage Export Header {filterLabel} + {`show-filters:${showFilters === true}`} {filterSlot}
), @@ -739,6 +748,26 @@ describe("EntityUsage", () => { }); }); + it("should still request the filter when the caller's tag scope is empty", async () => { + render(); + + await waitFor(() => { + expect(mockTagDailyActivityCall).toHaveBeenCalled(); + }); + + expect(screen.getByText("show-filters:true")).toBeInTheDocument(); + }); + + it("should not request the filter while the entity list is still unresolved", async () => { + render(); + + await waitFor(() => { + expect(mockTagDailyActivityCall).toHaveBeenCalled(); + }); + + expect(screen.getByText("show-filters:false")).toBeInTheDocument(); + }); + it("should display Agent Activity tab for team entity type", async () => { render(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx index 9501fa7a9a1..ef3943e5b71 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx @@ -661,7 +661,7 @@ const EntityUsage: React.FC = ({ dateValue={dateValue} entityType={entityType} spendData={spendData} - showFilters={filterSlot === undefined && entityList !== null && entityList.length > 0} + showFilters={filterSlot === undefined && entityList !== null} filterSlot={filterSlot} filterLabel={getFilterLabel(entityType)} filterPlaceholder={getFilterPlaceholder(entityType)} diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx index 27985c3db28..6c824459c08 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx @@ -84,4 +84,41 @@ describe("UsageExportHeader", () => { expect(screen.getByTestId("custom-filter")).toBeInTheDocument(); expect(screen.queryByRole("combobox")).not.toBeInTheDocument(); }); + + it("should keep the filter visible and disabled with an explanation when the caller has no options", () => { + renderWithProviders( + , + ); + + expect(screen.getByText("Filter by tag")).toBeInTheDocument(); + const input = screen.getByPlaceholderText("No tags with usage in this range"); + expect(input).toBeDisabled(); + expect(screen.queryByPlaceholderText("Select tag to filter...")).not.toBeInTheDocument(); + }); + + it("should leave the filter enabled with its normal placeholder when options exist", () => { + renderWithProviders( + , + ); + + const input = screen.getByPlaceholderText("Select tag to filter..."); + expect(input).toBeEnabled(); + expect(screen.queryByPlaceholderText("No tags with usage in this range")).not.toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.tsx index e694f905d8c..0d040e7c30b 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.tsx +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.tsx @@ -54,9 +54,11 @@ const UsageExportHeader: React.FC = ({ const anchor = useComboboxAnchor(); const [isExportModalOpen, setIsExportModalOpen] = useState(false); - const hasFilters = filterSlot != null || (showFilters && filterOptions.length > 0); + const hasFilters = filterSlot != null || showFilters; const optionValues = filterOptions.map((option) => option.value); const labelOf = (value: string) => filterOptions.find((option) => option.value === value)?.label ?? value; + const hasNoOptions = filterOptions.length === 0; + const emptyPlaceholder = `No ${entityType}s with usage in this range`; const filterList = ( @@ -74,6 +76,7 @@ const UsageExportHeader: React.FC = ({ const builtInFilter = ( onFiltersChange?.(next)} @@ -88,7 +91,10 @@ const UsageExportHeader: React.FC = ({ )) } - + {selectedFilters.length > 0 && } {filterList} From 13b5c80c90cc01ab3705576b57939ff230bf2d7a Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 27 Aug 2026 15:38:14 -0700 Subject: [PATCH 012/105] fix(ui): withhold the tag list until it resolves The usage page seeded its tag list as an empty array, so between first paint and the tag request landing the filter had a resolved-looking empty list and showed "No tags with usage in this range" for a scope nobody had measured yet. Seed it as null instead, which the filter already treats as unresolved and hides, so the empty state only appears once the list has actually come back. --- .../components/UsagePageView.test.tsx | 24 +++++++++++++++++++ .../_components/components/UsagePageView.tsx | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx index 118371aa9ac..f2dc56af9c5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx @@ -586,6 +586,30 @@ describe("UsagePage", () => { }); }); + it("should withhold the tag list until it resolves so no empty state is shown while loading", async () => { + let resolveTagList: (tags: Record) => void = () => {}; + mockTagListCall.mockReturnValue( + new Promise((resolve) => { + resolveTagList = resolve; + }) as ReturnType, + ); + + renderWithProviders(); + + act(() => { + fireEvent.change(screen.getByTestId("usage-view-select"), { target: { value: "tag" } }); + }); + + const entityUsage = await screen.findByTestId("entity-usage"); + expect(entityUsage).toHaveAttribute("data-entity-list", "null"); + + await act(async () => { + resolveTagList({}); + }); + + expect(screen.getByTestId("entity-usage")).toHaveAttribute("data-entity-list", "[]"); + }); + it("should show tag usage selector option for internal users", async () => { mockUseAuthorized.mockReturnValue({ isLoading: false, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx index c742b3af7e9..0d1e557a833 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx @@ -96,7 +96,7 @@ const UsagePage: React.FC = ({ teams, organizations }) => { to: initialToDate, }); - const [allTags, setAllTags] = useState([]); + const [allTags, setAllTags] = useState(null); const { data: customers = [] } = useCustomers(); const { data: agentsResponse } = useAgents(); const { data: currentUser } = useCurrentUser(); From beed32eb6050d306b55545d632340dececa347ec Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 27 Aug 2026 15:43:39 -0700 Subject: [PATCH 013/105] fix(ui): stamp the tag list with the range it answers Changing the date range left the previous range's tags in state until the new request landed, so the filter either offered tags that range no longer has or, when the old range was empty, stated "no tags" about a range nobody had measured yet. Stamp the fetched list with its range key and select it during render, the same way the request tiles above already guard against a superseded range. Clearing in the effect would be a render too late. --- .../components/UsagePageView.test.tsx | 36 +++++++++++++++++++ .../_components/components/UsagePageView.tsx | 17 ++++++--- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx index f2dc56af9c5..7d9bee735b4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx @@ -610,6 +610,42 @@ describe("UsagePage", () => { expect(screen.getByTestId("entity-usage")).toHaveAttribute("data-entity-list", "[]"); }); + it("should drop the previous range's tags as soon as the range changes", async () => { + mockTagListCall.mockResolvedValue({ "old-range-tag": { name: "old-range-tag" } } as never); + + renderWithProviders(); + + act(() => { + fireEvent.change(screen.getByTestId("usage-view-select"), { target: { value: "tag" } }); + }); + + await waitFor(() => { + expect(screen.getByTestId("entity-usage")).toHaveAttribute( + "data-entity-list", + JSON.stringify([{ label: "old-range-tag", value: "old-range-tag" }]), + ); + }); + + let resolveNewRange: (tags: Record) => void = () => {}; + mockTagListCall.mockReturnValue( + new Promise((resolve) => { + resolveNewRange = resolve; + }) as ReturnType, + ); + + act(() => { + fireEvent.click(screen.getByTestId("pick-a-different-range")); + }); + + expect(screen.getByTestId("entity-usage")).toHaveAttribute("data-entity-list", "null"); + + await act(async () => { + resolveNewRange({}); + }); + + expect(screen.getByTestId("entity-usage")).toHaveAttribute("data-entity-list", "[]"); + }); + it("should show tag usage selector option for internal users", async () => { mockUseAuthorized.mockReturnValue({ isLoading: false, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx index 0d1e557a833..19d7a752938 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx @@ -96,7 +96,7 @@ const UsagePage: React.FC = ({ teams, organizations }) => { to: initialToDate, }); - const [allTags, setAllTags] = useState(null); + const [fetchedTags, setFetchedTags] = useState | null>(null); const { data: customers = [] } = useCustomers(); const { data: agentsResponse } = useAgents(); const { data: currentUser } = useCurrentUser(); @@ -138,6 +138,12 @@ const UsagePage: React.FC = ({ teams, organizations }) => { const startTime = useMemo(() => (dateValue.from ? new Date(dateValue.from) : null), [dateValue.from]); const endTime = useMemo(() => (dateValue.to ? new Date(dateValue.to) : null), [dateValue.to]); + // Stamped and selected during render like the request tiles below: the tag + // filter reads "no tags" from an empty list, so a list left over from the + // previous range would state that about a range nobody has measured yet. + const currentTagRangeKey = fetchedRangeKey(startTime, endTime); + const allTags = selectForRange(fetchedTags, currentTagRangeKey); + useEffect(() => { if (!accessToken) return; let cancelled = false; @@ -145,12 +151,13 @@ const UsagePage: React.FC = ({ teams, organizations }) => { try { const tags = await tagListCall(accessToken, startTime, endTime); if (cancelled) return; - setAllTags( - Object.values(tags).map((tag: Tag) => ({ + setFetchedTags({ + rangeKey: currentTagRangeKey, + value: Object.values(tags).map((tag: Tag) => ({ label: tag.name, value: tag.name, })), - ); + }); } catch (e) { if (!cancelled) { console.error("Failed to fetch tag list", e); @@ -160,7 +167,7 @@ const UsagePage: React.FC = ({ teams, organizations }) => { return () => { cancelled = true; }; - }, [accessToken, startTime, endTime]); + }, [accessToken, startTime, endTime, currentTagRangeKey]); // Everything the request tiles read is stamped with the range it answers and // selected during render, rather than cleared in an effect. An effect runs From ceb2fa61c41b7c76eef8eadb3f267761c1c60783 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 27 Aug 2026 15:49:27 -0700 Subject: [PATCH 014/105] fix(ui): keep an outlived filter selection clearable, and defer the customer list Two loading/empty transitions the disabled empty state got wrong. A selection made in a range that had options survives a move to a range that has none, and it still scopes the data below, so disabling the combobox outright took away the only control that could clear it. Disable it only when there is nothing selected to clear. The customer list defaulted to an empty array while its query was in flight, so the filter announced a range with no customers before anything had been read. Leave it undefined until the query resolves, as the tag list now does. --- .../components/UsagePageView.test.tsx | 13 +++++++++++ .../_components/components/UsagePageView.tsx | 4 +++- .../UsageExportHeader.test.tsx | 22 +++++++++++++++++++ .../EntityUsageExport/UsageExportHeader.tsx | 5 ++++- 4 files changed, 42 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx index 7d9bee735b4..26d595f4d74 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx @@ -754,6 +754,19 @@ describe("UsagePage", () => { }); }); + it("should withhold the customer list while it is still loading", async () => { + mockUseCustomers.mockReturnValue({ data: undefined, isLoading: true, error: null } as any); + + renderWithProviders(); + + act(() => { + fireEvent.change(screen.getByTestId("usage-view-select"), { target: { value: "customer" } }); + }); + + const entityUsage = await screen.findByTestId("entity-usage"); + expect(entityUsage).toHaveAttribute("data-entity-list", "null"); + }); + it("should show agent usage view for admins", async () => { mockUseAgents.mockReturnValue({ data: { agents: mockAgents }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx index 19d7a752938..cbdfc8f39e6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx @@ -97,7 +97,9 @@ const UsagePage: React.FC = ({ teams, organizations }) => { }); const [fetchedTags, setFetchedTags] = useState | null>(null); - const { data: customers = [] } = useCustomers(); + // No [] default: an unresolved query must stay undefined so the customer + // filter reads as loading rather than as a range with no customers. + const { data: customers } = useCustomers(); const { data: agentsResponse } = useAgents(); const { data: currentUser } = useCurrentUser(); const isAdmin = all_admin_roles.includes(userRole || ""); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx index 6c824459c08..52fc7605d90 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx @@ -104,6 +104,28 @@ describe("UsageExportHeader", () => { expect(screen.queryByPlaceholderText("Select tag to filter...")).not.toBeInTheDocument(); }); + it("should stay usable when a carried-over selection outlives its options", async () => { + const user = userEvent.setup(); + const onFiltersChange = vi.fn(); + renderWithProviders( + , + ); + + expect(screen.getByPlaceholderText("No tags with usage in this range")).toBeEnabled(); + + await user.click(screen.getByRole("button", { name: "Clear Filter by tag" })); + expect(onFiltersChange).toHaveBeenCalledWith([]); + }); + it("should leave the filter enabled with its normal placeholder when options exist", () => { renderWithProviders( = ({ const labelOf = (value: string) => filterOptions.find((option) => option.value === value)?.label ?? value; const hasNoOptions = filterOptions.length === 0; const emptyPlaceholder = `No ${entityType}s with usage in this range`; + // A selection carried over from a range that did have options still scopes + // the data below, so the control has to stay usable long enough to clear it. + const isFilterDisabled = hasNoOptions && selectedFilters.length === 0; const filterList = ( @@ -76,7 +79,7 @@ const UsageExportHeader: React.FC = ({ const builtInFilter = ( onFiltersChange?.(next)} From 0b7852d646e6b8856043e69a4e097f64ebfc0047 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 27 Aug 2026 16:09:20 -0700 Subject: [PATCH 015/105] feat(ui): make provider logos readable in dark mode The dashboard's dark theme left a chunk of the bundled provider logos unreadable: 25 of them are pure black marks on a transparent background, so on a near-black surface they disappeared entirely, and another 11 are dark multicolor marks drawn for a white page. This adds the seam the rest of the work hangs off: a per-asset treatment manifest in logoTreatments.ts, and a Logo component that applies the treatment it names. Two treatments exist today. "invert" flattens a mark to solid white with brightness(0) invert(1), which is what the vendor's own white mark looks like for a pure-black transparent glyph. "plate" puts a white surface behind the mark so it reads exactly as it does on a light page. Both are dark-only, and only assets named in the manifest are touched, so light mode is unchanged and the other 96 bundled logos keep rendering byte for byte as they do today. The className an untreated logo receives is passed through verbatim rather than routed through cn(), so even the class string is unchanged. The split between invert and plate was measured per asset, not guessed: luminance, saturation and alpha coverage sampled off a canvas render. Two assets that look monochrome, aiml_api and repelloai, carry a light knockout inside dark artwork, so inversion would flatten the knockout into the mark and erase it. They get a plate instead, and a test pins that. Six assets whose artwork is an opaque dark box (aim_logo, aim_security, deepgram, jina, lakeraai, openmeter) are deliberately left untreated. A plate cannot show through an opaque image, so the only honest fix for them is a replacement asset. --- ui/litellm-dashboard/src/app/globals.css | 2 + .../components/molecules/logo/Logo.test.tsx | 34 +++++++++++ .../src/components/molecules/logo/Logo.tsx | 11 +++- .../src/lib/logoTreatments.test.ts | 56 +++++++++++++++++++ .../src/lib/logoTreatments.ts | 56 +++++++++++++++++++ 5 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 ui/litellm-dashboard/src/lib/logoTreatments.test.ts create mode 100644 ui/litellm-dashboard/src/lib/logoTreatments.ts diff --git a/ui/litellm-dashboard/src/app/globals.css b/ui/litellm-dashboard/src/app/globals.css index f389fa5df3d..87959ca0139 100644 --- a/ui/litellm-dashboard/src/app/globals.css +++ b/ui/litellm-dashboard/src/app/globals.css @@ -140,6 +140,7 @@ --sidebar-border: oklch(0.928 0.006 264.531); --sidebar-ring: oklch(0.707 0.022 261.325); --neutral-border: #dcddeb; + --logo-surface: oklch(1 0 0); } .dark { @@ -227,6 +228,7 @@ --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); --color-sidebar-border: var(--sidebar-border); --color-sidebar-ring: var(--sidebar-ring); + --color-logo-surface: var(--logo-surface); } @layer base { diff --git a/ui/litellm-dashboard/src/components/molecules/logo/Logo.test.tsx b/ui/litellm-dashboard/src/components/molecules/logo/Logo.test.tsx index 5e4da208f4e..62ce8e9ee12 100644 --- a/ui/litellm-dashboard/src/components/molecules/logo/Logo.test.tsx +++ b/ui/litellm-dashboard/src/components/molecules/logo/Logo.test.tsx @@ -52,6 +52,40 @@ describe("Logo", () => { warnSpy.mockRestore(); }); + it("leaves the caller's class list untouched for an asset that reads on dark", () => { + render(); + expect(screen.getByRole("img", { name: "Slack logo" })).toHaveClass("w-5 h-5 shrink-0", { exact: true }); + }); + + it("passes an untreated logo's classes through verbatim rather than normalizing them", () => { + render(); + expect(screen.getByRole("img", { name: "Slack logo" })).toHaveClass("w-4 w-5 h-5", { exact: true }); + }); + + it("forces a monochrome mark to white on dark without disturbing the caller's classes", () => { + render(); + const img = screen.getByRole("img", { name: "GitHub logo" }); + expect(img).toHaveClass("w-5", "h-5", "dark:[filter:brightness(0)_invert(1)]"); + expect(img).not.toHaveClass("dark:bg-logo-surface"); + }); + + it("plates a multicolor dark mark rather than inverting it", () => { + render(); + const img = screen.getByRole("img", { name: "Fireworks logo" }); + expect(img).toHaveClass("dark:bg-logo-surface", "dark:object-contain", "dark:p-0.5"); + expect(img).not.toHaveClass("dark:[filter:brightness(0)_invert(1)]"); + }); + + it("does not treat an external logo URL that collides with a bundled filename", () => { + render(); + expect(screen.getByRole("img", { name: "Ext logo" })).toHaveClass("w-5 h-5", { exact: true }); + }); + + it("applies the treatment to a provider logo resolved through the bundler", () => { + render(); + expect(screen.getByRole("img", { name: "openrouter logo" })).toHaveClass("dark:[filter:brightness(0)_invert(1)]"); + }); + it("retries with a new src after a previous src errored", () => { const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); const { rerender } = render(); diff --git a/ui/litellm-dashboard/src/components/molecules/logo/Logo.tsx b/ui/litellm-dashboard/src/components/molecules/logo/Logo.tsx index d388e3dc07d..66c10f1e222 100644 --- a/ui/litellm-dashboard/src/components/molecules/logo/Logo.tsx +++ b/ui/litellm-dashboard/src/components/molecules/logo/Logo.tsx @@ -1,12 +1,19 @@ import React, { useState } from "react"; import { getProviderLogoAndName } from "@/components/provider_info_helpers"; import { resolveLogoSrc } from "@/lib/assetPaths"; +import { cn } from "@/lib/cva.config"; +import { logoTreatmentFor, type LogoTreatment } from "@/lib/logoTreatments"; type LogoProps = { className?: string } & ( | { provider: string; src?: never; label?: string } | { provider?: never; src: string | null | undefined; label: string } ); +const DARK_TREATMENT_CLASS: Readonly> = { + invert: "dark:[filter:brightness(0)_invert(1)]", + plate: "dark:bg-logo-surface dark:object-contain dark:p-0.5", +}; + export const Logo: React.FC = ({ provider, src, label, className = "w-4 h-4" }) => { const [erroredSrc, setErroredSrc] = useState(null); const resolvedSrc = provider !== undefined ? getProviderLogoAndName(provider).logo : resolveLogoSrc(src) ?? ""; @@ -20,11 +27,13 @@ export const Logo: React.FC = ({ provider, src, label, className = "w ); } + const treatment = logoTreatmentFor(resolvedSrc); + return ( {`${name { console.warn(`Logo failed to load: ${resolvedSrc}`); setErroredSrc(resolvedSrc); diff --git a/ui/litellm-dashboard/src/lib/logoTreatments.test.ts b/ui/litellm-dashboard/src/lib/logoTreatments.test.ts new file mode 100644 index 00000000000..b0a2073be8d --- /dev/null +++ b/ui/litellm-dashboard/src/lib/logoTreatments.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import { logoTreatmentFor } from "./logoTreatments"; + +describe("logoTreatmentFor", () => { + it("marks a monochrome transparent mark for inversion", () => { + expect(logoTreatmentFor("/ui/assets/logos/github.svg")).toBe("invert"); + }); + + it("marks a multicolor dark mark for a plate instead of inversion", () => { + expect(logoTreatmentFor("/ui/assets/logos/fireworks.svg")).toBe("plate"); + }); + + it("plates a dark mark with a light knockout, which inversion would flatten away", () => { + expect(logoTreatmentFor("/ui/assets/logos/repelloai.png")).toBe("plate"); + expect(logoTreatmentFor("/ui/assets/logos/aiml_api.svg")).toBe("plate"); + }); + + it("leaves an asset that already reads on dark untreated", () => { + expect(logoTreatmentFor("/ui/assets/logos/slack.svg")).toBeUndefined(); + }); + + it("resolves through a bundler fingerprint in the filename", () => { + expect(logoTreatmentFor("/litellm-asset-prefix/_next/static/media/openrouter.1xk7748-_jixf.svg")).toBe("invert"); + }); + + it("resolves a bundled asset served under a proxy root path", () => { + expect(logoTreatmentFor("/litellm/ui/assets/logos/notion.svg")).toBe("invert"); + }); + + it("ignores a query string and fragment on the asset URL", () => { + expect(logoTreatmentFor("/ui/assets/logos/vercel.svg?v=2#icon")).toBe("invert"); + }); + + it("does not treat an external URL whose filename collides with a bundled asset", () => { + expect(logoTreatmentFor("https://cdn.example.com/github.svg")).toBeUndefined(); + }); + + it("does not treat a non-logo path whose filename collides with a bundled asset", () => { + expect(logoTreatmentFor("/uploads/user/github.svg")).toBeUndefined(); + }); + + it("leaves an opaque dark box untreated, since a plate behind it cannot show through", () => { + expect(logoTreatmentFor("/ui/assets/logos/lakeraai.jpeg")).toBeUndefined(); + }); + + it("returns undefined for empty and nullish input", () => { + expect(logoTreatmentFor(null)).toBeUndefined(); + expect(logoTreatmentFor(undefined)).toBeUndefined(); + expect(logoTreatmentFor("")).toBeUndefined(); + }); + + it("distinguishes assets that share a stem but differ by extension", () => { + expect(logoTreatmentFor("/ui/assets/logos/runway.png")).toBe("invert"); + expect(logoTreatmentFor("/ui/assets/logos/runway.svg")).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/lib/logoTreatments.ts b/ui/litellm-dashboard/src/lib/logoTreatments.ts new file mode 100644 index 00000000000..f8378800046 --- /dev/null +++ b/ui/litellm-dashboard/src/lib/logoTreatments.ts @@ -0,0 +1,56 @@ +export type LogoTreatment = "invert" | "plate"; + +const BUNDLED_LOGO_PATH = /(?:\/assets\/logos\/|\/_next\/static\/media\/)/; + +const TREATMENT_BY_ASSET: Readonly> = { + "baseten.svg": "invert", + "cursor.svg": "invert", + "enkrypt_ai.avif": "invert", + "friendli.svg": "invert", + "github.svg": "invert", + "github_copilot.svg": "invert", + "lago.svg": "invert", + "lambda.svg": "invert", + "langflow.svg": "invert", + "lmstudio.svg": "invert", + "moonshot.svg": "invert", + "nebius.svg": "invert", + "notion.svg": "invert", + "ollama.svg": "invert", + "openrouter.svg": "invert", + "promptguard.svg": "invert", + "recraft.svg": "invert", + "replicate.svg": "invert", + "runway.png": "invert", + "scx_ai.svg": "invert", + "secret_detect.png": "invert", + "topaz.svg": "invert", + "v0.svg": "invert", + "vercel.svg": "invert", + "watsonx.svg": "invert", + "aiml_api.svg": "plate", + "akto.svg": "plate", + "aws.svg": "plate", + "deepkeep.svg": "plate", + "fireworks.svg": "plate", + "llm_guard.png": "plate", + "pangea.png": "plate", + "repelloai.png": "plate", + "sambanova.svg": "plate", + "sentry.svg": "plate", + "valkey.svg": "plate", +}; + +const basenameOf = (src: string): string | undefined => src.split(/[?#]/)[0].split("/").pop() || undefined; + +const withoutBundlerHash = (basename: string): string | undefined => { + const parts = basename.split("."); + return parts.length < 2 ? undefined : `${parts[0]}.${parts[parts.length - 1]}`; +}; + +export const logoTreatmentFor = (src: string | null | undefined): LogoTreatment | undefined => { + if (!src || !BUNDLED_LOGO_PATH.test(src)) return undefined; + const basename = basenameOf(src); + const key = basename === undefined ? undefined : withoutBundlerHash(basename); + return key === undefined ? undefined : TREATMENT_BY_ASSET[key]; +}; From 51959feb89822eb7ec60b80294f91f60936811e5 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 27 Aug 2026 17:19:54 -0700 Subject: [PATCH 016/105] fix(ui): never treat a remote logo URL as a bundled asset The bundled-path check was an unanchored substring match, so a user-supplied logo URL that happened to carry /assets/logos/ or /_next/static/media/ in its path, and whose filename collided with one of the 36 manifest entries, would pick up a dark-mode treatment meant only for assets we ship. assetPaths already draws this line for resolveLogoSrc, which returns an external src untouched. Export that predicate instead of writing a second one, and require a treated src to clear it. The existing test only covered a remote URL with a bare filename, which passed either way. The new ones fail without the guard. --- .../src/components/molecules/logo/Logo.test.tsx | 5 +++++ ui/litellm-dashboard/src/lib/assetPaths.ts | 4 +++- ui/litellm-dashboard/src/lib/logoTreatments.test.ts | 11 +++++++++++ ui/litellm-dashboard/src/lib/logoTreatments.ts | 4 +++- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/molecules/logo/Logo.test.tsx b/ui/litellm-dashboard/src/components/molecules/logo/Logo.test.tsx index 62ce8e9ee12..a8e7b4e41d4 100644 --- a/ui/litellm-dashboard/src/components/molecules/logo/Logo.test.tsx +++ b/ui/litellm-dashboard/src/components/molecules/logo/Logo.test.tsx @@ -81,6 +81,11 @@ describe("Logo", () => { expect(screen.getByRole("img", { name: "Ext logo" })).toHaveClass("w-5 h-5", { exact: true }); }); + it("does not treat a user-supplied logo URL that mimics the bundled asset path", () => { + render(); + expect(screen.getByRole("img", { name: "Ext logo" })).toHaveClass("w-5 h-5", { exact: true }); + }); + it("applies the treatment to a provider logo resolved through the bundler", () => { render(); expect(screen.getByRole("img", { name: "openrouter logo" })).toHaveClass("dark:[filter:brightness(0)_invert(1)]"); diff --git a/ui/litellm-dashboard/src/lib/assetPaths.ts b/ui/litellm-dashboard/src/lib/assetPaths.ts index abce69127cb..af208e7d78c 100644 --- a/ui/litellm-dashboard/src/lib/assetPaths.ts +++ b/ui/litellm-dashboard/src/lib/assetPaths.ts @@ -3,6 +3,8 @@ import { normalizeRootPath } from "@/lib/http/resolveApiBase"; const EXTERNAL_SRC = /^(https?:|data:|blob:|\/\/)/i; +export const isExternalAssetSrc = (value: string): boolean => EXTERNAL_SRC.test(value); + /** * Prefix a root-relative asset path (e.g. "/ui/assets/logos/openai.svg") with the * proxy's server root path so it resolves when the UI is mounted under a sub-path @@ -22,7 +24,7 @@ export const withServerRoot = (path: string, root: string): string => { */ export const resolveLogoSrc = (value: string | null | undefined, root: string = serverRootPath): string | undefined => { if (!value) return undefined; - if (EXTERNAL_SRC.test(value)) return value; + if (isExternalAssetSrc(value)) return value; if (value.includes("/_next/static/")) return value; const prefix = normalizeRootPath(root); if (prefix && (value === prefix || value.startsWith(`${prefix}/`))) return value; diff --git a/ui/litellm-dashboard/src/lib/logoTreatments.test.ts b/ui/litellm-dashboard/src/lib/logoTreatments.test.ts index b0a2073be8d..08ea283a7a3 100644 --- a/ui/litellm-dashboard/src/lib/logoTreatments.test.ts +++ b/ui/litellm-dashboard/src/lib/logoTreatments.test.ts @@ -35,6 +35,17 @@ describe("logoTreatmentFor", () => { expect(logoTreatmentFor("https://cdn.example.com/github.svg")).toBeUndefined(); }); + it("does not treat a remote URL that also carries a bundled-looking path", () => { + expect(logoTreatmentFor("https://cdn.example.com/assets/logos/github.svg")).toBeUndefined(); + expect(logoTreatmentFor("http://cdn.example.com/_next/static/media/github.abc123.svg")).toBeUndefined(); + expect(logoTreatmentFor("//cdn.example.com/assets/logos/github.svg")).toBeUndefined(); + expect(logoTreatmentFor("HTTPS://CDN.EXAMPLE.COM/assets/logos/github.svg")).toBeUndefined(); + }); + + it("does not treat a data URL that happens to contain a bundled-looking path", () => { + expect(logoTreatmentFor("data:image/svg+xml,/assets/logos/github.svg")).toBeUndefined(); + }); + it("does not treat a non-logo path whose filename collides with a bundled asset", () => { expect(logoTreatmentFor("/uploads/user/github.svg")).toBeUndefined(); }); diff --git a/ui/litellm-dashboard/src/lib/logoTreatments.ts b/ui/litellm-dashboard/src/lib/logoTreatments.ts index f8378800046..7a5a7892d17 100644 --- a/ui/litellm-dashboard/src/lib/logoTreatments.ts +++ b/ui/litellm-dashboard/src/lib/logoTreatments.ts @@ -1,3 +1,5 @@ +import { isExternalAssetSrc } from "@/lib/assetPaths"; + export type LogoTreatment = "invert" | "plate"; const BUNDLED_LOGO_PATH = /(?:\/assets\/logos\/|\/_next\/static\/media\/)/; @@ -49,7 +51,7 @@ const withoutBundlerHash = (basename: string): string | undefined => { }; export const logoTreatmentFor = (src: string | null | undefined): LogoTreatment | undefined => { - if (!src || !BUNDLED_LOGO_PATH.test(src)) return undefined; + if (!src || isExternalAssetSrc(src) || !BUNDLED_LOGO_PATH.test(src)) return undefined; const basename = basenameOf(src); const key = basename === undefined ? undefined : withoutBundlerHash(basename); return key === undefined ? undefined : TREATMENT_BY_ASSET[key]; From e5dcc6873ec060d57038dd4093ab3b803b229a4a Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 27 Aug 2026 18:05:04 -0700 Subject: [PATCH 017/105] fix(ui): make the theme toggle switch on one click and stop Docs looking dimmer than Blog The top bar's theme control needed a click on the sun/moon, then a menu, then a choice, to do something every other product does in one click. It is now a plain button that flips between light and dark, with the beta marker moved into the label of the click that turns dark on. An explicit "system" choice is gone, but next-themes still follows the OS for anyone who has it stored and has not clicked yet. Docs and Blog also drifted apart in the gateway header: Blog rendered through the shared product-link class while Docs was a muted ghost button one size down, so Docs read as dimmer and sat 4px shorter. Both now go through a shared DocsLink component, which is also what the legacy navbar uses, so the pair cannot drift again. --- .../src/components/DashboardHeader.test.tsx | 11 ++++ .../src/components/DashboardHeader.tsx | 12 +--- .../Navbar/DocsLink/DocsLink.test.tsx | 27 ++++++++ .../components/Navbar/DocsLink/DocsLink.tsx | 15 +++++ .../ThemeToggle/ThemeToggle.test.tsx | 64 +++++++++---------- .../components/ThemeToggle/ThemeToggle.tsx | 58 ++++------------- .../src/components/navbar.tsx | 15 +---- 7 files changed, 101 insertions(+), 101 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/Navbar/DocsLink/DocsLink.test.tsx create mode 100644 ui/litellm-dashboard/src/components/Navbar/DocsLink/DocsLink.tsx diff --git a/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx b/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx index 6c656382deb..6a06e1ba612 100644 --- a/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx +++ b/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { act, fireEvent, render, screen } from "@testing-library/react"; import { DashboardHeader } from "./DashboardHeader"; +import { NAV_PRODUCT_LINK_CLASS } from "@/components/Navbar/navProductLinkClass"; const { mockUsePluginMode, mockUseUISettings, state } = vi.hoisted(() => { const state = { @@ -55,6 +56,16 @@ describe("DashboardHeader breadcrumb", () => { expect(screen.queryByText("Observability")).not.toBeInTheDocument(); }); + it("styles Docs with the shared product-link class instead of a muted toolbar button", () => { + render(); + + const docs = screen.getByRole("link", { name: "Docs" }); + for (const cls of NAV_PRODUCT_LINK_CLASS.trim().split(/\s+/)) { + expect(docs).toHaveClass(cls); + } + expect(docs).not.toHaveClass("text-muted-foreground"); + }); + it("renders the tools divider centered rather than stretched to the top of the row", () => { const { container } = render(); diff --git a/ui/litellm-dashboard/src/components/DashboardHeader.tsx b/ui/litellm-dashboard/src/components/DashboardHeader.tsx index 31f2a0b715a..fe824ce074d 100644 --- a/ui/litellm-dashboard/src/components/DashboardHeader.tsx +++ b/ui/litellm-dashboard/src/components/DashboardHeader.tsx @@ -1,6 +1,5 @@ "use client"; -import { Button } from "@/components/ui/button"; import { Breadcrumb, BreadcrumbItem, @@ -11,6 +10,7 @@ import { import { ToolbarSeparator } from "@/components/shared/ToolbarSeparator"; import { getBreadcrumb } from "@/components/leftnav"; import { BlogDropdown } from "@/components/Navbar/BlogDropdown/BlogDropdown"; +import { DocsLink } from "@/components/Navbar/DocsLink/DocsLink"; import { CommunityEngagementButtons } from "@/components/Navbar/CommunityEngagementButtons/CommunityEngagementButtons"; import { NotificationsBell } from "@/components/Navbar/NotificationsBell/NotificationsBell"; import ViewSwitcher from "@/components/Navbar/ViewSwitcher"; @@ -62,15 +62,7 @@ export function DashboardHeader({ page }: DashboardHeaderProps) { )} - + {!hideCommunityLinks && } diff --git a/ui/litellm-dashboard/src/components/Navbar/DocsLink/DocsLink.test.tsx b/ui/litellm-dashboard/src/components/Navbar/DocsLink/DocsLink.test.tsx new file mode 100644 index 00000000000..d5a155cf503 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Navbar/DocsLink/DocsLink.test.tsx @@ -0,0 +1,27 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { NAV_PRODUCT_LINK_CLASS } from "@/components/Navbar/navProductLinkClass"; +import { DocsLink } from "./DocsLink"; + +const sharedClasses = NAV_PRODUCT_LINK_CLASS.trim().split(/\s+/); + +describe("DocsLink", () => { + it("opens the docs in a new tab without leaking the opener", () => { + render(); + + const link = screen.getByRole("link", { name: "Docs" }); + expect(link).toHaveAttribute("href", "https://docs.litellm.ai/docs/"); + expect(link).toHaveAttribute("target", "_blank"); + expect(link).toHaveAttribute("rel", "noopener noreferrer"); + }); + + it("carries the same product-link styling as the Blog trigger, so the two never drift apart", () => { + render(); + + const link = screen.getByRole("link", { name: "Docs" }); + for (const cls of sharedClasses) { + expect(link).toHaveClass(cls); + } + expect(link).not.toHaveClass("text-muted-foreground"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Navbar/DocsLink/DocsLink.tsx b/ui/litellm-dashboard/src/components/Navbar/DocsLink/DocsLink.tsx new file mode 100644 index 00000000000..9176e1f9c38 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Navbar/DocsLink/DocsLink.tsx @@ -0,0 +1,15 @@ +import { NAV_PRODUCT_LINK_CLASS } from "@/components/Navbar/navProductLinkClass"; +import { ChevronDown } from "lucide-react"; +import React from "react"; + +export const DOCS_URL = "https://docs.litellm.ai/docs/"; + +export const DocsLink: React.FC = () => ( + + Docs + {/* Docs is a single outbound link; the hidden chevron keeps its box identical to the Blog dropdown trigger. */} + + +); + +export default DocsLink; diff --git a/ui/litellm-dashboard/src/components/ThemeToggle/ThemeToggle.test.tsx b/ui/litellm-dashboard/src/components/ThemeToggle/ThemeToggle.test.tsx index 41efedcd43a..c5dda3d3f7c 100644 --- a/ui/litellm-dashboard/src/components/ThemeToggle/ThemeToggle.test.tsx +++ b/ui/litellm-dashboard/src/components/ThemeToggle/ThemeToggle.test.tsx @@ -11,12 +11,7 @@ const renderToggle = () => , ); -const openMenu = async () => { - await userEvent.click(screen.getByRole("button", { name: "Theme" })); - await screen.findByRole("menu"); -}; - -const pick = async (label: string | RegExp) => userEvent.click(screen.getByRole("menuitemradio", { name: label })); +const toggle = () => screen.getByRole("button", { name: /Switch to (light|dark) mode/ }); beforeEach(() => { localStorage.clear(); @@ -28,50 +23,49 @@ afterAll(() => { }); describe("ThemeToggle", () => { - it("starts on light rather than following the system preference", async () => { + it("switches to dark on a single click, with no menu in between", async () => { renderToggle(); - await openMenu(); - expect(screen.getByRole("menuitemradio", { name: "Light" })).toBeChecked(); - expect(screen.getByRole("menuitemradio", { name: /^Dark/ })).not.toBeChecked(); - expect(screen.getByRole("menuitemradio", { name: "System" })).not.toBeChecked(); - }); - - it("puts the dark class on the document and remembers the choice", async () => { - renderToggle(); - await openMenu(); - - await pick(/^Dark/); + await userEvent.click(toggle()); + expect(screen.queryByRole("menu")).not.toBeInTheDocument(); expect(document.documentElement).toHaveClass("dark"); expect(localStorage.getItem("theme")).toBe("dark"); }); - it("hands control back to the system preference when asked", async () => { + it("switches back to light on the next click", async () => { renderToggle(); - await openMenu(); - await pick(/^Dark/); + await userEvent.click(toggle()); - await pick("System"); + await userEvent.click(toggle()); + + expect(document.documentElement).not.toHaveClass("dark"); + expect(localStorage.getItem("theme")).toBe("light"); + }); + + it("names the mode the click will switch to, so the button says what it does", async () => { + renderToggle(); + expect(screen.getByRole("button", { name: "Switch to dark mode (beta)" })).toBeInTheDocument(); + + await userEvent.click(toggle()); + + expect(await screen.findByRole("button", { name: "Switch to light mode" })).toBeInTheDocument(); + }); + + it("leaves a stored system preference following the OS until the user clicks", () => { + localStorage.setItem("theme", "system"); + + renderToggle(); expect(localStorage.getItem("theme")).toBe("system"); - expect(document.documentElement).not.toHaveClass("dark"); }); - it("marks dark as beta in the menu, and leaves the other choices unmarked", async () => { + it("keeps the beta marker out of the toolbar label once dark is on", async () => { renderToggle(); - await openMenu(); - expect(screen.getByRole("menuitemradio", { name: /^Dark/ })).toHaveTextContent("Beta"); - expect(screen.getByRole("menuitemradio", { name: "Light" })).not.toHaveTextContent("Beta"); - expect(screen.getByRole("menuitemradio", { name: "System" })).not.toHaveTextContent("Beta"); - }); + await userEvent.click(toggle()); - it("keeps the beta marker inside the menu rather than in the toolbar", async () => { - renderToggle(); - await openMenu(); - await pick(/^Dark/); - - expect(screen.getByRole("button", { name: "Theme" })).not.toHaveTextContent("Beta"); + expect(toggle()).not.toHaveTextContent("Beta"); + expect(toggle()).toHaveAccessibleName("Switch to light mode"); }); }); diff --git a/ui/litellm-dashboard/src/components/ThemeToggle/ThemeToggle.tsx b/ui/litellm-dashboard/src/components/ThemeToggle/ThemeToggle.tsx index 3fbb3d2eb5b..2f941dbc2d2 100644 --- a/ui/litellm-dashboard/src/components/ThemeToggle/ThemeToggle.tsx +++ b/ui/litellm-dashboard/src/components/ThemeToggle/ThemeToggle.tsx @@ -1,57 +1,27 @@ "use client"; -import { Monitor, Moon, Sun } from "lucide-react"; +import { Moon, Sun } from "lucide-react"; import { useTheme } from "next-themes"; import React from "react"; -import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuRadioGroup, - DropdownMenuRadioItem, - DropdownMenuTrigger, -} from "@/components/ui/dropdown-menu"; - -const THEMES = [ - { value: "system", label: "System", Icon: Monitor, beta: false }, - { value: "light", label: "Light", Icon: Sun, beta: false }, - { value: "dark", label: "Dark", Icon: Moon, beta: true }, -] as const; const ThemeToggle: React.FC = () => { - const { theme, setTheme, resolvedTheme } = useTheme(); + const { setTheme, resolvedTheme } = useTheme(); + const isDark = resolvedTheme === "dark"; + const label = isDark ? "Switch to light mode" : "Switch to dark mode (beta)"; return ( - - - } - > - {resolvedTheme === "dark" ? : } - - - - {THEMES.map(({ value, label, Icon, beta }) => ( - - - {label} - {beta && ( - - Beta - - )} - - ))} - - - + ); }; diff --git a/ui/litellm-dashboard/src/components/navbar.tsx b/ui/litellm-dashboard/src/components/navbar.tsx index e81c67cc958..88cd125ea15 100644 --- a/ui/litellm-dashboard/src/components/navbar.tsx +++ b/ui/litellm-dashboard/src/components/navbar.tsx @@ -9,12 +9,12 @@ import { clearTokenCookies } from "@/utils/cookieUtils"; import { clearStoredReturnUrl, getLoginUrl } from "@/utils/returnUrlUtils"; import useProxySettings from "@/app/(dashboard)/hooks/proxySettings/useProxySettings"; import { Badge } from "@/components/ui/badge"; -import { ChevronDown, PanelLeftClose, PanelLeftOpen } from "lucide-react"; +import { PanelLeftClose, PanelLeftOpen } from "lucide-react"; import Link from "next/link"; import React from "react"; import { BlogDropdown } from "./Navbar/BlogDropdown/BlogDropdown"; +import { DocsLink } from "./Navbar/DocsLink/DocsLink"; import { CommunityEngagementButtons } from "./Navbar/CommunityEngagementButtons/CommunityEngagementButtons"; -import { NAV_PRODUCT_LINK_CLASS } from "./Navbar/navProductLinkClass"; import { cn } from "@/lib/cva.config"; import { NotificationsBell } from "./Navbar/NotificationsBell/NotificationsBell"; import UserDropdown from "./Navbar/UserDropdown/UserDropdown"; @@ -143,16 +143,7 @@ const Navbar: React.FC = ({ aria-label="Product documentation" className={`flex min-w-0 items-center gap-2 ${showWorkerSwitch ? "border-l border-border pl-4" : ""}`} > - - Docs - {/* Layout parity with Blog chevron — intentional single-level link */} - - + From ca21cf577304ebd9f1f170ae35d04b4e3152db63 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:31:15 -0700 Subject: [PATCH 018/105] feat(a2a): semantic search over the agent registry via GET /v1/agents?query and an agent_search MCP tool --- litellm/__init__.py | 1 + .../mcp_server/rest_endpoints.py | 18 +- .../proxy/_experimental/mcp_server/server.py | 13 +- .../_experimental/mcp_server/tool_search.py | 151 +++++++--- litellm/proxy/_lazy_openapi_snapshot.json | 90 +++++- litellm/proxy/agent_endpoints/agent_search.py | 184 ++++++++++++ .../auth/agent_permission_handler.py | 16 + litellm/proxy/agent_endpoints/endpoints.py | 92 ++++-- litellm/types/agents.py | 1 + .../mcp_server/test_mcp_tool_search.py | 128 +++++++- .../agent_endpoints/test_agent_search.py | 281 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 12 + 12 files changed, 910 insertions(+), 77 deletions(-) create mode 100644 litellm/proxy/agent_endpoints/agent_search.py create mode 100644 tests/test_litellm/proxy/agent_endpoints/test_agent_search.py diff --git a/litellm/__init__.py b/litellm/__init__.py index ec2960c196e..179cac47663 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -487,6 +487,7 @@ public_mcp_servers: Optional[List[str]] = None public_mcp_hub_strict_whitelist: bool = True public_model_groups: Optional[List[str]] = None public_agent_groups: Optional[List[str]] = None +agent_search_embedding_model: Optional[str] = None # Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]]) # New format: { "displayName": { "url": "...", "index": 0 } } # Old format: { "displayName": "url" } (for backward compatibility) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 3a8fd6de5e5..3efb6429326 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -168,8 +168,11 @@ if MCP_AVAILABLE: MCPRequestHandler, ) from litellm.proxy._experimental.mcp_server.tool_search import ( + AGENT_SEARCH_TOOL_NAME, + DEFAULT_AGENT_SEARCH_TOP_K, MCP_TOOL_SEARCH_TOOL_NAME, coerce_top_k, + handle_agent_search, handle_mcp_tool_call, handle_mcp_tool_search, ) @@ -182,6 +185,14 @@ if MCP_AVAILABLE: detail={"error": "forbidden", "message": f"{tool_name} requires mcp_tool_search_enabled on the key"}, ) tool_arguments: Final = data.get("arguments") or {} + if tool_name == AGENT_SEARCH_TOOL_NAME: + return await handle_agent_search( + query=str(tool_arguments.get("query", "")), + top_k=coerce_top_k( + tool_arguments.get("top_k", DEFAULT_AGENT_SEARCH_TOP_K), default=DEFAULT_AGENT_SEARCH_TOP_K + ), + user_api_key_dict=user_api_key_dict, + ) rest_client_ip: Final = IPAddressUtils.get_mcp_client_ip(request) ( virtual_mcp_auth_header, @@ -939,12 +950,9 @@ if MCP_AVAILABLE: tool_name: Final[str | None] = data.get("name") tool_arguments: Final[dict[str, object]] = data.get("arguments") or {} - from litellm.proxy._experimental.mcp_server.tool_search import ( - MCP_TOOL_CALL_TOOL_NAME, - MCP_TOOL_SEARCH_TOOL_NAME, - ) + from litellm.proxy._experimental.mcp_server.tool_search import VIRTUAL_TOOL_NAMES - if tool_name in (MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME): + if tool_name in VIRTUAL_TOOL_NAMES: return await _handle_virtual_mcp_tool(request, data, tool_name, user_api_key_dict) # Validate required parameters early diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index c6b2ac489bb..989b08b929a 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -912,14 +912,17 @@ if MCP_AVAILABLE: the caller falls through to normal tool routing. """ from litellm.proxy._experimental.mcp_server.tool_search import ( - MCP_TOOL_CALL_TOOL_NAME, + AGENT_SEARCH_TOOL_NAME, + DEFAULT_AGENT_SEARCH_TOP_K, MCP_TOOL_SEARCH_TOOL_NAME, + VIRTUAL_TOOL_NAMES, coerce_top_k, + handle_agent_search, handle_mcp_tool_call, handle_mcp_tool_search, ) - if name not in (MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME): + if name not in VIRTUAL_TOOL_NAMES: return None if not getattr( @@ -952,6 +955,12 @@ if MCP_AVAILABLE: ) assert user_api_key_auth is not None # guaranteed by the flag check above + if name == AGENT_SEARCH_TOOL_NAME: + return await handle_agent_search( + query=str(args.get("query", "")), + top_k=coerce_top_k(args.get("top_k", DEFAULT_AGENT_SEARCH_TOP_K), default=DEFAULT_AGENT_SEARCH_TOP_K), + user_api_key_dict=user_api_key_auth, + ) virtual_logging_obj: Final = await _build_virtual_call_logging_obj( name=name, arguments=args, diff --git a/litellm/proxy/_experimental/mcp_server/tool_search.py b/litellm/proxy/_experimental/mcp_server/tool_search.py index 3b0dd2071ae..a33ab4de3b7 100644 --- a/litellm/proxy/_experimental/mcp_server/tool_search.py +++ b/litellm/proxy/_experimental/mcp_server/tool_search.py @@ -1,8 +1,14 @@ from __future__ import annotations import json +from collections.abc import Mapping from datetime import datetime -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, TypedDict, assert_never + +from typing_extensions import ReadOnly, Required + +import litellm +from litellm.proxy.agent_endpoints.agent_search import DEFAULT_AGENT_SEARCH_TOP_K if TYPE_CHECKING: from mcp.types import CallToolResult @@ -12,6 +18,8 @@ if TYPE_CHECKING: MCP_TOOL_SEARCH_TOOL_NAME: Final[str] = "mcp_tool_search" MCP_TOOL_CALL_TOOL_NAME: Final[str] = "mcp_tool_call" +AGENT_SEARCH_TOOL_NAME: Final[str] = "agent_search" +VIRTUAL_TOOL_NAMES: Final = frozenset((MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME, AGENT_SEARCH_TOOL_NAME)) def coerce_top_k(value: Any, default: int = 5) -> int: @@ -34,46 +42,111 @@ def search_tools(query: str, tools: list[dict[str, Any]], top_k: int = 5) -> lis return [tool for _, tool in sorted(scored, key=lambda x: x[0], reverse=True)[:top_k]] -def get_virtual_tool_definitions() -> list[dict[str, Any]]: - return [ - { - "name": MCP_TOOL_SEARCH_TOOL_NAME, - "description": "Search for MCP tools by keyword. Returns top matching tools with names, descriptions, and input schemas.", - "inputSchema": { - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "Keywords to search for in tool names and descriptions.", - }, - "top_k": { - "type": "integer", - "description": "Maximum number of results to return.", - "default": 5, - }, - }, - "required": ["query"], +class _ToolParamSchema(TypedDict, total=False): + type: Required[ReadOnly[str]] + description: Required[ReadOnly[str]] + default: ReadOnly[int] + + +class _ToolInputSchema(TypedDict): + type: ReadOnly[str] + properties: ReadOnly[Mapping[str, _ToolParamSchema]] + required: ReadOnly[tuple[str, ...]] + + +class VirtualToolDefinition(TypedDict): + name: ReadOnly[str] + description: ReadOnly[str] + inputSchema: ReadOnly[_ToolInputSchema] + + +_MCP_TOOL_SEARCH_DEFINITION: Final[VirtualToolDefinition] = { + "name": MCP_TOOL_SEARCH_TOOL_NAME, + "description": "Search for MCP tools by keyword. Returns top matching tools with names, descriptions, and input schemas.", + "inputSchema": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Keywords to search for in tool names and descriptions."}, + "top_k": {"type": "integer", "description": "Maximum number of results to return.", "default": 5}, + }, + "required": ("query",), + }, +} + +_MCP_TOOL_CALL_DEFINITION: Final[VirtualToolDefinition] = { + "name": MCP_TOOL_CALL_TOOL_NAME, + "description": "Call an MCP tool by name with the given arguments.", + "inputSchema": { + "type": "object", + "properties": { + "tool_name": {"type": "string", "description": "The exact name of the MCP tool to call."}, + "arguments": {"type": "object", "description": "Arguments to pass to the tool."}, + }, + "required": ("tool_name",), + }, +} + +_AGENT_SEARCH_DEFINITION: Final[VirtualToolDefinition] = { + "name": AGENT_SEARCH_TOOL_NAME, + "description": "Find A2A agents by describing the task in natural language. Returns the best matching agents you can access, ranked by semantic similarity, each with its agent_id, name, description, skills, and score.", + "inputSchema": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "The task the agent should be able to do, in natural language."}, + "top_k": { + "type": "integer", + "description": "Maximum number of agents to return.", + "default": DEFAULT_AGENT_SEARCH_TOP_K, }, }, - { - "name": MCP_TOOL_CALL_TOOL_NAME, - "description": "Call an MCP tool by name with the given arguments.", - "inputSchema": { - "type": "object", - "properties": { - "tool_name": { - "type": "string", - "description": "The exact name of the MCP tool to call.", - }, - "arguments": { - "type": "object", - "description": "Arguments to pass to the tool.", - }, - }, - "required": ["tool_name"], - }, - }, - ] + "required": ("query",), + }, +} + + +def get_virtual_tool_definitions() -> tuple[VirtualToolDefinition, ...]: + return (_MCP_TOOL_SEARCH_DEFINITION, _MCP_TOOL_CALL_DEFINITION, _AGENT_SEARCH_DEFINITION) + + +def _text_tool_result(text: str, is_error: bool) -> CallToolResult: + from mcp.types import CallToolResult, TextContent + + return CallToolResult( + content=[TextContent(type="text", text=text)], # mutable-ok: CallToolResult accepts only list content + isError=is_error, + ) + + +async def handle_agent_search(query: str, top_k: int, user_api_key_dict: UserAPIKeyAuth) -> CallToolResult: + from litellm.proxy.agent_endpoints.agent_search import ( + AgentSearchEmbeddingFailed, + AgentSearchHits, + AgentSearchNotConfigured, + agent_search_result, + global_agent_search_index, + search_agents, + ) + from litellm.proxy.agent_endpoints.auth.agent_permission_handler import accessible_agents + from litellm.proxy.common_utils.rbac_utils import check_feature_access_for_user + from litellm.proxy.proxy_server import llm_router + + await check_feature_access_for_user(user_api_key_dict, "agents") + outcome: Final = await search_agents( + query=query, + agents=await accessible_agents(user_api_key_dict), + top_k=max(top_k, 1), + router=llm_router, + embedding_model=litellm.agent_search_embedding_model, + index=global_agent_search_index, + ) + match outcome: + case AgentSearchHits(hits): + results: Final = tuple(agent_search_result(hit).model_dump() for hit in hits) + return _text_tool_result(json.dumps(results), is_error=False) + case AgentSearchNotConfigured(reason) | AgentSearchEmbeddingFailed(reason): + return _text_tool_result(reason, is_error=True) + case _: + assert_never(outcome) async def handle_mcp_tool_search( diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index e30750ef565..9ee0d3fcf21 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -2377,6 +2377,17 @@ ], "title": "Rpm Limit" }, + "search_score": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Search Score" + }, "session_rpm_limit": { "anyOf": [ { @@ -3404,7 +3415,7 @@ }, "/v1/agents": { "get": { - "description": "Example usage:\n```\ncurl -X GET \"http://localhost:4000/v1/agents\" -H \"Content-Type: application/json\" -H \"Authorization: Bearer your-key\" ```\n\nPass `?health_check=true` to filter out agents whose URL is unreachable:\n```\ncurl -X GET \"http://localhost:4000/v1/agents?health_check=true\" -H \"Content-Type: application/json\" -H \"Authorization: Bearer your-key\" ```\n\nReturns: List[AgentResponse]", + "description": "Example usage:\n```\ncurl -X GET \"http://localhost:4000/v1/agents\" -H \"Content-Type: application/json\" -H \"Authorization: Bearer your-key\" ```\n\nPass `?health_check=true` to filter out agents whose URL is unreachable:\n```\ncurl -X GET \"http://localhost:4000/v1/agents?health_check=true\" -H \"Content-Type: application/json\" -H \"Authorization: Bearer your-key\" ```\n\nPass `?query=` to get the best matching agents ranked by semantic similarity:\n```\ncurl -X GET \"http://localhost:4000/v1/agents?query=translate+a+PDF+document&top_k=5\" -H \"Content-Type: application/json\" -H \"Authorization: Bearer your-key\" ```\n\nReturns: List[AgentResponse]", "operationId": "get_agents_v1_agents_get", "parameters": [ { @@ -3418,6 +3429,39 @@ "title": "Health Check", "type": "boolean" } + }, + { + "description": "Describe the task in natural language to rank the agents you can reach by semantic similarity over their name, description, and skills. Each result carries a search_score. Requires litellm_settings.agent_search_embedding_model.", + "in": "query", + "name": "query", + "required": false, + "schema": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Describe the task in natural language to rank the agents you can reach by semantic similarity over their name, description, and skills. Each result carries a search_score. Requires litellm_settings.agent_search_embedding_model.", + "title": "Query" + } + }, + { + "description": "With query: the maximum number of ranked agents to return.", + "in": "query", + "name": "top_k", + "required": false, + "schema": { + "default": 5, + "description": "With query: the maximum number of ranked agents to return.", + "maximum": 100, + "minimum": 1, + "title": "Top K", + "type": "integer" + } } ], "responses": { @@ -15038,6 +15082,17 @@ } ], "title": "Upstream Resource" + }, + "upstream_token_header": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Upstream Token Header" } }, "title": "MCPCredentials", @@ -17518,6 +17573,17 @@ } ], "title": "Upstream Resource" + }, + "upstream_token_header": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Upstream Token Header" } }, "title": "MCPCredentials", @@ -20352,6 +20418,17 @@ } ], "title": "Upstream Resource" + }, + "upstream_token_header": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Upstream Token Header" } }, "title": "MCPCredentials", @@ -23699,6 +23776,17 @@ } ], "title": "Upstream Resource" + }, + "upstream_token_header": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Upstream Token Header" } }, "title": "MCPCredentials", diff --git a/litellm/proxy/agent_endpoints/agent_search.py b/litellm/proxy/agent_endpoints/agent_search.py new file mode 100644 index 00000000000..a8fc57eb229 --- /dev/null +++ b/litellm/proxy/agent_endpoints/agent_search.py @@ -0,0 +1,184 @@ +"""Semantic ranking over the in-memory A2A agent registry, shared by GET /v1/agents?query= and the agent_search MCP tool.""" + +from __future__ import annotations + +import math +from collections.abc import Awaitable, Mapping, Sequence +from dataclasses import dataclass +from itertools import chain +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Protocol, TypeAlias + +from openai import OpenAIError +from pydantic import BaseModel, ConfigDict, ValidationError + +from litellm.exceptions import BudgetExceededError +from litellm.types.agents import AgentResponse + +if TYPE_CHECKING: + from litellm.router import Router + +DEFAULT_AGENT_SEARCH_TOP_K: Final = 5 + +Vector: TypeAlias = tuple[float, ...] + + +class Embedder(Protocol): + def __call__(self, texts: Sequence[str]) -> Awaitable[Sequence[Vector]]: ... + + +@dataclass(frozen=True, slots=True) +class AgentSearchHit: + agent: AgentResponse + score: float + + +@dataclass(frozen=True, slots=True) +class AgentSearchHits: + hits: tuple[AgentSearchHit, ...] + + +@dataclass(frozen=True, slots=True) +class AgentSearchNotConfigured: + reason: str + + +@dataclass(frozen=True, slots=True) +class AgentSearchEmbeddingFailed: + reason: str + + +AgentSearchOutcome: TypeAlias = AgentSearchHits | AgentSearchNotConfigured | AgentSearchEmbeddingFailed + + +class _SearchableSkill(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + name: str = "" + description: str = "" + tags: tuple[str, ...] = () + + +class _SearchableCard(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + description: str = "" + skills: tuple[_SearchableSkill, ...] = () + + +class _EmbeddingItem(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + embedding: tuple[float, ...] + + +class _EmbeddingData(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + data: tuple[_EmbeddingItem, ...] + + +class AgentSearchResult(BaseModel): + model_config = ConfigDict(frozen=True) + + agent_id: str + agent_name: str + description: str + skills: tuple[_SearchableSkill, ...] + score: float + + +def _searchable_card(agent: AgentResponse) -> _SearchableCard: + try: + return _SearchableCard.model_validate(agent.agent_card_params) + except ValidationError: + return _SearchableCard() + + +def _skill_text(skill: _SearchableSkill) -> str: + return " ".join(part for part in (skill.name, skill.description, " ".join(skill.tags)) if part) + + +def agent_search_text(agent: AgentResponse) -> str: + card: Final = _searchable_card(agent) + skill_lines: Final = tuple(_skill_text(skill) for skill in card.skills) + return "\n".join(part for part in (agent.agent_name, card.description, *skill_lines) if part) + + +def agent_search_result(hit: AgentSearchHit) -> AgentSearchResult: + card: Final = _searchable_card(hit.agent) + return AgentSearchResult( + agent_id=hit.agent.agent_id, + agent_name=hit.agent.agent_name, + description=card.description, + skills=card.skills, + score=hit.score, + ) + + +def cosine_similarity(left: Vector, right: Vector) -> float: + dot: Final = sum(a * b for a, b in zip(left, right, strict=True)) + norms: Final = math.sqrt(sum(a * a for a in left)) * math.sqrt(sum(b * b for b in right)) + return dot / norms if norms else 0.0 + + +def router_embedder(router: Router, embedding_model: str) -> Embedder: + async def embed(texts: Sequence[str]) -> Sequence[Vector]: + batch: Final = list(texts) # mutable-ok: Router.aembedding accepts only str | list input + response: Final = await router.aembedding(model=embedding_model, input=batch) + return tuple(item.embedding for item in _EmbeddingData.model_validate(response.model_dump()).data) + + return embed + + +class AgentSearchIndex: + """Caches one vector per distinct agent text, so repeat searches only embed the query.""" + + def __init__(self) -> None: + self._vectors: Mapping[str, Vector] = MappingProxyType({}) + + async def search( + self, query: str, agents: Sequence[AgentResponse], top_k: int, embed: Embedder + ) -> AgentSearchHits | AgentSearchEmbeddingFailed: + if not agents: + return AgentSearchHits(hits=()) + texts: Final = tuple(agent_search_text(agent) for agent in agents) + missing: Final = tuple(dict.fromkeys(text for text in texts if text not in self._vectors)) + try: + vectors: Final = await embed((query, *missing)) + except (OpenAIError, ValueError, BudgetExceededError) as exc: + return AgentSearchEmbeddingFailed(reason=f"embedding the search query failed: {exc}") + if len(vectors) != len(missing) + 1: + return AgentSearchEmbeddingFailed( + reason=f"embedding model returned {len(vectors)} vectors for {len(missing) + 1} inputs" + ) + self._vectors = MappingProxyType(dict(chain(self._vectors.items(), zip(missing, vectors[1:], strict=True)))) + ranked: Final = sorted( + ( + AgentSearchHit(agent=agent, score=cosine_similarity(vectors[0], self._vectors[text])) + for agent, text in zip(agents, texts, strict=True) + ), + key=lambda hit: hit.score, + reverse=True, + ) + return AgentSearchHits(hits=tuple(ranked[:top_k])) + + +global_agent_search_index: Final = AgentSearchIndex() + + +async def search_agents( + query: str, + agents: Sequence[AgentResponse], + top_k: int, + router: Router | None, + embedding_model: str | None, + index: AgentSearchIndex, +) -> AgentSearchOutcome: + if embedding_model is None: + return AgentSearchNotConfigured( + reason="agent search needs litellm_settings.agent_search_embedding_model set to an embedding model from model_list" + ) + if router is None: + return AgentSearchNotConfigured(reason="agent search needs a model_list so the embedding model can be called") + return await index.search(query, agents, top_k, router_embedder(router, embedding_model)) diff --git a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py index 81586b4eef3..d0ac94d3710 100644 --- a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py +++ b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py @@ -13,9 +13,11 @@ from litellm.proxy._types import ( UI_TEAM_ID, LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, + LitellmUserRoles, UserAPIKeyAuth, ) from litellm.repositories.table_repositories import AgentsRepository +from litellm.types.agents import AgentResponse @dataclass(frozen=True, slots=True) @@ -439,3 +441,17 @@ class AgentRequestHandler: except Exception as e: verbose_logger.warning("Failed to get agent access groups for team: %s", e) return [] + + +async def accessible_agents(user_api_key_auth: UserAPIKeyAuth) -> tuple[AgentResponse, ...]: + """Every registry agent for proxy admins, else the agents the key's and team's grants reach.""" + from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry + + all_agents: Final = global_agent_registry.get_agent_list() + if user_api_key_auth.user_role in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN.value): + return all_agents + match await AgentRequestHandler.resolve_agent_access(user_api_key_auth=user_api_key_auth): + case UnrestrictedAgentAccess(): + return all_agents + case RestrictedAgentAccess(allowed_agent_ids): + return tuple(agent for agent in all_agents if agent.agent_id in allowed_agent_ids) diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py index d348bc01153..cfb0597cc2a 100644 --- a/litellm/proxy/agent_endpoints/endpoints.py +++ b/litellm/proxy/agent_endpoints/endpoints.py @@ -12,10 +12,11 @@ import asyncio import os import uuid from collections.abc import Mapping, Sequence -from typing import Final, TypedDict +from types import MappingProxyType +from typing import Annotated, Final, TypedDict, assert_never from fastapi import APIRouter, Depends, HTTPException, Query, Request -from typing_extensions import Required +from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_proxy_logger @@ -32,6 +33,15 @@ from litellm.proxy.a2a.agent_card import ( merge_agent_card, normalize_protocol_version, ) +from litellm.proxy.agent_endpoints.agent_search import ( + DEFAULT_AGENT_SEARCH_TOP_K, + AgentSearchEmbeddingFailed, + AgentSearchHits, + AgentSearchNotConfigured, + global_agent_search_index, + search_agents, +) +from litellm.proxy.agent_endpoints.auth.agent_permission_handler import accessible_agents from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.rbac_utils import check_feature_access_for_user from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity @@ -211,6 +221,38 @@ async def _check_agent_url_health( } +class _AgentSearchErrorDetail(TypedDict): + error: ReadOnly[str] + message: ReadOnly[str] + + +def _agent_search_error(status_code: int, error: str, message: str) -> HTTPException: + detail: Final[_AgentSearchErrorDetail] = {"error": error, "message": message} + return HTTPException(status_code=status_code, detail=detail) + + +async def _rank_agents_by_query(query: str, agents: Sequence[AgentResponse], top_k: int) -> tuple[AgentResponse, ...]: + from litellm.proxy.proxy_server import llm_router + + outcome: Final = await search_agents( + query=query, + agents=agents, + top_k=top_k, + router=llm_router, + embedding_model=litellm.agent_search_embedding_model, + index=global_agent_search_index, + ) + match outcome: + case AgentSearchHits(hits): + return tuple(hit.agent.model_copy(update=MappingProxyType({"search_score": hit.score})) for hit in hits) + case AgentSearchNotConfigured(reason): + raise _agent_search_error(400, "agent_search_not_configured", reason) + case AgentSearchEmbeddingFailed(reason): + raise _agent_search_error(503, "agent_search_unavailable", reason) + case _: + assert_never(outcome) + + @router.get( "/v1/agents", tags=["[beta] A2A Agents"], @@ -223,6 +265,17 @@ async def get_agents( False, description="When true, performs a GET request to each agent's URL. Agents with reachable URLs (HTTP status < 500) and agents without a URL are returned; unreachable agents are filtered out.", ), + query: Annotated[ + str | None, + Query( + min_length=1, + description="Describe the task in natural language to rank the agents you can reach by semantic similarity over their name, description, and skills. Each result carries a search_score. Requires litellm_settings.agent_search_embedding_model.", + ), + ] = None, + top_k: Annotated[ + int, + Query(ge=1, le=100, description="With query: the maximum number of ranked agents to return."), + ] = DEFAULT_AGENT_SEARCH_TOP_K, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # Used for auth ): """ @@ -240,37 +293,22 @@ async def get_agents( -H "Authorization: Bearer your-key" \ ``` + Pass `?query=` to get the best matching agents ranked by semantic similarity: + ``` + curl -X GET "http://localhost:4000/v1/agents?query=translate+a+PDF+document&top_k=5" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-key" \ + ``` + Returns: List[AgentResponse] """ await check_feature_access_for_user(user_api_key_dict, "agents") from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry - from litellm.proxy.agent_endpoints.auth.agent_permission_handler import ( - AgentRequestHandler, - RestrictedAgentAccess, - UnrestrictedAgentAccess, - ) try: - returned_agents: Sequence[AgentResponse] = () - - # Admin users get all agents - if ( - user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN - or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value - ): - returned_agents = global_agent_registry.get_agent_list() - else: - # Get allowed agents from object_permission (key/team level) - agent_access: Final = await AgentRequestHandler.resolve_agent_access(user_api_key_auth=user_api_key_dict) - all_agents: Final = global_agent_registry.get_agent_list() - - match agent_access: - case UnrestrictedAgentAccess(): - returned_agents = all_agents - case RestrictedAgentAccess(allowed_agent_ids): - returned_agents = [agent for agent in all_agents if agent.agent_id in allowed_agent_ids] + returned_agents: Sequence[AgentResponse] = await accessible_agents(user_api_key_dict) # Fetch current spend from DB for all returned agents from litellm.proxy.proxy_server import prisma_client @@ -336,7 +374,9 @@ async def get_agents( healthy_ids: Final = {result["agent_id"] for result in health_results if result["healthy"]} returned_agents = [agent for agent in agents_with_url if agent.agent_id in healthy_ids] + agents_without_url - return returned_agents + if query is None: + return returned_agents + return await _rank_agents_by_query(query, returned_agents, top_k) except HTTPException: raise except Exception as e: diff --git a/litellm/types/agents.py b/litellm/types/agents.py index 7e499dde642..c85507b77c1 100644 --- a/litellm/types/agents.py +++ b/litellm/types/agents.py @@ -225,6 +225,7 @@ class AgentResponse(BaseModel): static_headers: dict[str, str] | None = None extra_headers: list[str] | None = None keys: list[AgentKeySummary] | None = None + search_score: float | None = None created_at: datetime | None = None updated_at: datetime | None = None created_by: str | None = None diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py index 0e442102e53..dfe1e224c1f 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py @@ -18,6 +18,7 @@ import pytest from litellm.models.object_permission import LiteLLM_ObjectPermissionTable from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing from litellm.proxy._experimental.mcp_server.tool_search import ( + AGENT_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME, MCP_TOOL_SEARCH_TOOL_NAME, coerce_top_k, @@ -114,8 +115,15 @@ class TestSearchTools: class TestGetVirtualToolDefinitions: - def test_returns_two_tools(self) -> None: - assert len(get_virtual_tool_definitions()) == 2 + def test_returns_three_tools(self) -> None: + assert len(get_virtual_tool_definitions()) == 3 + + def test_agent_search_schema_requires_query(self) -> None: + tools = get_virtual_tool_definitions() + agent_tool = next(t for t in tools if t["name"] == AGENT_SEARCH_TOOL_NAME) + props = agent_tool["inputSchema"]["properties"] + assert set(props) == {"query", "top_k"} + assert agent_tool["inputSchema"]["required"] == ("query",) def test_has_mcp_tool_search(self) -> None: names = [t["name"] for t in get_virtual_tool_definitions()] @@ -130,7 +138,7 @@ class TestGetVirtualToolDefinitions: search_tool = next(t for t in tools if t["name"] == MCP_TOOL_SEARCH_TOOL_NAME) props = search_tool["inputSchema"]["properties"] assert "query" in props - assert search_tool["inputSchema"]["required"] == ["query"] + assert search_tool["inputSchema"]["required"] == ("query",) def test_mcp_tool_call_schema_has_tool_name_and_arguments(self) -> None: tools = get_virtual_tool_definitions() @@ -153,6 +161,7 @@ class TestGetVirtualToolDefinitions: assert {t.name for t in built} == { MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME, + AGENT_SEARCH_TOOL_NAME, } @@ -187,7 +196,7 @@ class TestListToolRestApiWithToolSearch: assert result["error"] is None tool_names = [t["name"] for t in result["tools"]] - assert set(tool_names) == {MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME} + assert set(tool_names) == {MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME, AGENT_SEARCH_TOOL_NAME} @pytest.mark.asyncio async def test_returns_full_catalog_when_flag_disabled(self) -> None: @@ -517,6 +526,81 @@ class TestCallToolRestApiVirtualTools: mock_list.assert_awaited_once() assert mock_list.await_args.kwargs["client_ip"] == "203.0.113.7" + @pytest.mark.asyncio + async def test_agent_search_call_ranks_accessible_agents(self) -> None: + from litellm.proxy.agent_endpoints.agent_search import AgentSearchHit, AgentSearchHits + from litellm.types.agents import AgentResponse + + user_api_key_dict = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) + request = self._make_request( + {"name": AGENT_SEARCH_TOOL_NAME, "arguments": {"query": "translate a document", "top_k": "1"}} + ) + translator = AgentResponse( + agent_id="translator", + agent_name="document-translator", + agent_card_params={"description": "Translates files", "skills": [{"id": "t", "name": "Translate"}]}, + ) + with ( + patch( # test-quality-ok: the tool resolves agent access through proxy_server globals, no injection seam + "litellm.proxy.agent_endpoints.auth.agent_permission_handler.accessible_agents", + new_callable=AsyncMock, + return_value=(translator,), + ), + patch( # test-quality-ok: the embedding router only resolves via proxy_server globals, no injection seam + "litellm.proxy.agent_endpoints.agent_search.search_agents", + new_callable=AsyncMock, + return_value=AgentSearchHits(hits=(AgentSearchHit(agent=translator, score=0.91),)), + ) as mock_search, + ): + result = await self._get_call_fn()(request=request, user_api_key_dict=user_api_key_dict) + + assert result.isError is False + assert json.loads(result.content[0].text) == [ + { + "agent_id": "translator", + "agent_name": "document-translator", + "description": "Translates files", + "skills": [{"name": "Translate", "description": "", "tags": []}], + "score": 0.91, + } + ] + assert mock_search.await_args.kwargs["query"] == "translate a document" + assert mock_search.await_args.kwargs["top_k"] == 1 + assert mock_search.await_args.kwargs["agents"] == (translator,) + + @pytest.mark.asyncio + async def test_agent_search_call_reports_missing_embedding_model_as_tool_error(self) -> None: + from litellm.proxy.agent_endpoints.agent_search import AgentSearchNotConfigured + + user_api_key_dict = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) + request = self._make_request({"name": AGENT_SEARCH_TOOL_NAME, "arguments": {"query": "anything"}}) + with ( + patch( # test-quality-ok: the tool resolves agent access through proxy_server globals, no injection seam + "litellm.proxy.agent_endpoints.auth.agent_permission_handler.accessible_agents", + new_callable=AsyncMock, + return_value=(), + ), + patch( # test-quality-ok: the embedding router only resolves via proxy_server globals, no injection seam + "litellm.proxy.agent_endpoints.agent_search.search_agents", + new_callable=AsyncMock, + return_value=AgentSearchNotConfigured(reason="set agent_search_embedding_model"), + ), + ): + result = await self._get_call_fn()(request=request, user_api_key_dict=user_api_key_dict) + + assert result.isError is True + assert result.content[0].text == "set agent_search_embedding_model" + + @pytest.mark.asyncio + async def test_agent_search_requires_flag_enabled(self) -> None: + from fastapi import HTTPException + + user_api_key_dict = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=False)) + request = self._make_request({"name": AGENT_SEARCH_TOOL_NAME, "arguments": {"query": "anything"}}) + with pytest.raises(HTTPException) as exc_info: + await self._get_call_fn()(request=request, user_api_key_dict=user_api_key_dict) + assert exc_info.value.status_code == 403 + @pytest.mark.asyncio async def test_mcp_tool_search_requires_flag_enabled(self) -> None: from fastapi import HTTPException @@ -592,6 +676,41 @@ class TestDispatchVirtualMcpTool: assert mock_search.await_args.kwargs["query"] == "q" assert mock_search.await_args.kwargs["top_k"] == 3 + @pytest.mark.asyncio + async def test_routes_agent_search_to_its_handler(self) -> None: + from litellm.proxy._experimental.mcp_server import server as srv + + uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) + with patch( # test-quality-ok: dispatch routing is the subject; the handler is faked like its siblings here + "litellm.proxy._experimental.mcp_server.tool_search.handle_agent_search", + new_callable=AsyncMock, + return_value="AGENT_RESULT", + ) as mock_agent_search: + result = await srv._dispatch_virtual_mcp_tool( + name=AGENT_SEARCH_TOOL_NAME, + arguments={"query": "translate a document", "top_k": "2"}, + user_api_key_auth=uak, + client_ip=None, + ) + + assert result == "AGENT_RESULT" + assert mock_agent_search.await_args.kwargs == { + "query": "translate a document", + "top_k": 2, + "user_api_key_dict": uak, + } + + @pytest.mark.asyncio + async def test_agent_search_rejected_when_flag_disabled(self) -> None: + from litellm.proxy._experimental.mcp_server.server import _dispatch_virtual_mcp_tool + + uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=False)) + result = await _dispatch_virtual_mcp_tool( + name=AGENT_SEARCH_TOOL_NAME, arguments={"query": "x"}, user_api_key_auth=uak, client_ip=None + ) + assert result is not None + assert result.isError is True + @pytest.mark.asyncio async def test_routes_call_with_client_ip(self) -> None: from litellm.proxy._experimental.mcp_server import server as srv @@ -850,6 +969,7 @@ class TestHandleListToolsVirtual: assert {t.name for t in tools} == { MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME, + AGENT_SEARCH_TOOL_NAME, } diff --git a/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py b/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py new file mode 100644 index 00000000000..f6313eade9e --- /dev/null +++ b/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py @@ -0,0 +1,281 @@ +from collections.abc import Sequence +from types import MappingProxyType +from typing import Final +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from openai import APIConnectionError + +import litellm +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.agent_endpoints.agent_search import ( + AgentSearchEmbeddingFailed, + AgentSearchHits, + AgentSearchIndex, + AgentSearchNotConfigured, + Vector, + agent_search_text, + cosine_similarity, + search_agents, +) +from litellm.proxy.agent_endpoints.auth.agent_permission_handler import RestrictedAgentAccess +from litellm.proxy.agent_endpoints.endpoints import router, user_api_key_auth +from litellm.types.agents import AgentResponse + +TRANSLATOR: Final = AgentResponse( + agent_id="translator", + agent_name="document-translator", + agent_card_params={ + "name": "Document Translator", + "description": "Converts files from one language into another", + "skills": [ + { + "id": "t", + "name": "Translate a file", + "description": "Produce the document in the target language", + "tags": ["localization", "documents"], + } + ], + }, +) +SQL_ANALYST: Final = AgentResponse( + agent_id="sql", + agent_name="warehouse-sql-analyst", + agent_card_params={ + "name": "Warehouse SQL Analyst", + "description": "Runs SQL against the inventory database", + "skills": [], + }, +) +TRIP_PLANNER: Final = AgentResponse( + agent_id="trip", + agent_name="trip-planner", + agent_card_params={"name": "Trip Planner", "description": "Books flights and hotels"}, +) +AGENTS: Final = (TRANSLATOR, SQL_ANALYST, TRIP_PLANNER) + +VECTORS: Final = MappingProxyType( + { + "language translation": (1.0, 0.0, 0.0), + agent_search_text(TRANSLATOR): (0.9, 0.1, 0.0), + agent_search_text(SQL_ANALYST): (0.0, 1.0, 0.0), + agent_search_text(TRIP_PLANNER): (0.3, 0.0, 1.0), + } +) + + +class FakeEmbedder: + def __init__(self) -> None: + self.calls: list[tuple[str, ...]] = [] # mutable-ok: test spy recording embed inputs + + async def __call__(self, texts: Sequence[str]) -> Sequence[Vector]: + self.calls.append(tuple(texts)) + return tuple(VECTORS[text] for text in texts) + + +class TestAgentSearchText: + def test_joins_name_description_and_skills_with_tags(self) -> None: + assert agent_search_text(TRANSLATOR) == ( + "document-translator\n" + "Converts files from one language into another\n" + "Translate a file Produce the document in the target language localization documents" + ) + + def test_missing_card_fields_fall_back_to_the_name(self) -> None: + assert agent_search_text(AgentResponse(agent_id="x", agent_name="bare", agent_card_params={})) == "bare" + + def test_malformed_skills_do_not_break_the_text(self) -> None: + agent = AgentResponse( + agent_id="x", agent_name="odd", agent_card_params={"skills": "not-a-list", "description": "d"} + ) + assert agent_search_text(agent) == "odd" + + +class TestCosineSimilarity: + def test_identical_direction_scores_one(self) -> None: + assert cosine_similarity((2.0, 0.0), (1.0, 0.0)) == pytest.approx(1.0) + + def test_orthogonal_scores_zero(self) -> None: + assert cosine_similarity((1.0, 0.0), (0.0, 1.0)) == pytest.approx(0.0) + + def test_zero_vector_scores_zero_instead_of_dividing(self) -> None: + assert cosine_similarity((0.0, 0.0), (1.0, 0.0)) == 0.0 + + +class TestAgentSearchIndex: + @pytest.mark.asyncio + async def test_ranks_by_similarity_and_truncates_to_top_k(self) -> None: + outcome = await AgentSearchIndex().search("language translation", AGENTS, top_k=2, embed=FakeEmbedder()) + assert isinstance(outcome, AgentSearchHits) + assert [hit.agent.agent_id for hit in outcome.hits] == ["translator", "trip"] + assert outcome.hits[0].score > outcome.hits[1].score + + @pytest.mark.asyncio + async def test_second_search_only_embeds_the_query(self) -> None: + index = AgentSearchIndex() + embedder = FakeEmbedder() + await index.search("language translation", AGENTS, top_k=5, embed=embedder) + await index.search("language translation", AGENTS, top_k=5, embed=embedder) + assert len(embedder.calls[0]) == 1 + len(AGENTS) + assert embedder.calls[1] == ("language translation",) + + @pytest.mark.asyncio + async def test_empty_registry_returns_no_hits_without_embedding(self) -> None: + embedder = FakeEmbedder() + outcome = await AgentSearchIndex().search("anything", (), top_k=5, embed=embedder) + assert outcome == AgentSearchHits(hits=()) + assert embedder.calls == [] + + @pytest.mark.asyncio + async def test_provider_error_becomes_embedding_failed(self) -> None: + async def failing(texts: Sequence[str]) -> Sequence[Vector]: + raise APIConnectionError(request=MagicMock()) + + outcome = await AgentSearchIndex().search("q", AGENTS, top_k=5, embed=failing) + assert isinstance(outcome, AgentSearchEmbeddingFailed) + assert "embedding the search query failed" in outcome.reason + + @pytest.mark.asyncio + async def test_wrong_vector_count_becomes_embedding_failed(self) -> None: + async def short(texts: Sequence[str]) -> Sequence[Vector]: + return ((1.0, 0.0, 0.0),) + + outcome = await AgentSearchIndex().search("q", AGENTS, top_k=5, embed=short) + assert isinstance(outcome, AgentSearchEmbeddingFailed) + + +class TestSearchAgents: + @pytest.mark.asyncio + async def test_no_embedding_model_is_not_configured(self) -> None: + outcome = await search_agents( + "q", AGENTS, 5, router=MagicMock(), embedding_model=None, index=AgentSearchIndex() + ) + assert isinstance(outcome, AgentSearchNotConfigured) + assert "agent_search_embedding_model" in outcome.reason + + @pytest.mark.asyncio + async def test_no_router_is_not_configured(self) -> None: + outcome = await search_agents("q", AGENTS, 5, router=None, embedding_model="m", index=AgentSearchIndex()) + assert isinstance(outcome, AgentSearchNotConfigured) + + @pytest.mark.asyncio + async def test_router_embeddings_are_read_from_the_response(self) -> None: + router = MagicMock() + router.aembedding = AsyncMock( + side_effect=lambda model, input: litellm.EmbeddingResponse( + model=model, + data=[{"object": "embedding", "index": i, "embedding": list(VECTORS[t])} for i, t in enumerate(input)], + ) + ) + outcome = await search_agents( + "language translation", + AGENTS, + 1, + router=router, + embedding_model="text-embedding-3-small", + index=AgentSearchIndex(), + ) + assert isinstance(outcome, AgentSearchHits) + assert [hit.agent.agent_id for hit in outcome.hits] == ["translator"] + assert router.aembedding.await_args.kwargs["model"] == "text-embedding-3-small" + + +def _client(role: LitellmUserRoles) -> TestClient: + app = FastAPI() + app.include_router(router) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="u", user_role=role) + return TestClient(app) + + +@pytest.fixture +def registry(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + from litellm.proxy.agent_endpoints import agent_registry as registry_module + + mock_registry = MagicMock() + mock_registry.get_agent_list = MagicMock(return_value=AGENTS) + mock_registry.ids_for_agent = MagicMock(side_effect=lambda agent_id: frozenset({agent_id})) + monkeypatch.setattr(registry_module, "global_agent_registry", mock_registry) + monkeypatch.setattr("litellm.proxy.agent_endpoints.endpoints.global_agent_search_index", AgentSearchIndex()) + return mock_registry + + +@pytest.fixture +def embedding_router(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + router = MagicMock() + router.aembedding = AsyncMock( + side_effect=lambda model, input: litellm.EmbeddingResponse( + model=model, + data=[{"object": "embedding", "index": i, "embedding": list(VECTORS[t])} for i, t in enumerate(input)], + ) + ) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) + monkeypatch.setattr(litellm, "agent_search_embedding_model", "text-embedding-3-small") + return router + + +@pytest.fixture +def no_db(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + + +class TestGetAgentsQuery: + def test_query_ranks_and_scores_and_truncates( + self, registry: MagicMock, embedding_router: MagicMock, no_db: None + ) -> None: + response = _client(LitellmUserRoles.PROXY_ADMIN).get( + "/v1/agents", params={"query": "language translation", "top_k": 2}, headers={"Authorization": "Bearer k"} + ) + assert response.status_code == 200 + body = response.json() + assert [agent["agent_id"] for agent in body] == ["translator", "trip"] + assert body[0]["search_score"] > body[1]["search_score"] + + def test_without_query_the_list_is_unchanged_and_unscored( + self, registry: MagicMock, embedding_router: MagicMock, no_db: None + ) -> None: + response = _client(LitellmUserRoles.PROXY_ADMIN).get("/v1/agents", headers={"Authorization": "Bearer k"}) + assert response.status_code == 200 + assert [agent["agent_id"] for agent in response.json()] == ["translator", "sql", "trip"] + assert all(agent["search_score"] is None for agent in response.json()) + embedding_router.aembedding.assert_not_awaited() + + def test_restricted_key_only_ranks_its_own_agents( + self, registry: MagicMock, embedding_router: MagicMock, no_db: None, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr( + "litellm.proxy.agent_endpoints.auth.agent_permission_handler.AgentRequestHandler.resolve_agent_access", + AsyncMock(return_value=RestrictedAgentAccess(frozenset({"sql"}))), + ) + response = _client(LitellmUserRoles.INTERNAL_USER).get( + "/v1/agents", params={"query": "language translation"}, headers={"Authorization": "Bearer k"} + ) + assert response.status_code == 200 + assert [agent["agent_id"] for agent in response.json()] == ["sql"] + + def test_missing_embedding_model_is_a_400( + self, registry: MagicMock, embedding_router: MagicMock, no_db: None, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "agent_search_embedding_model", None) + response = _client(LitellmUserRoles.PROXY_ADMIN).get( + "/v1/agents", params={"query": "anything"}, headers={"Authorization": "Bearer k"} + ) + assert response.status_code == 400 + assert response.json()["detail"]["error"] == "agent_search_not_configured" + + def test_embedding_provider_failure_is_a_503( + self, registry: MagicMock, embedding_router: MagicMock, no_db: None + ) -> None: + embedding_router.aembedding = AsyncMock(side_effect=APIConnectionError(request=MagicMock())) + response = _client(LitellmUserRoles.PROXY_ADMIN).get( + "/v1/agents", params={"query": "anything"}, headers={"Authorization": "Bearer k"} + ) + assert response.status_code == 503 + assert response.json()["detail"]["error"] == "agent_search_unavailable" + + def test_top_k_is_validated(self, registry: MagicMock, embedding_router: MagicMock, no_db: None) -> None: + response = _client(LitellmUserRoles.PROXY_ADMIN).get( + "/v1/agents", params={"query": "anything", "top_k": 0}, headers={"Authorization": "Bearer k"} + ) + assert response.status_code == 422 diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index c124cc2e9c8..8c4c26c0c54 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -16575,6 +16575,10 @@ export interface paths { * ``` * curl -X GET "http://localhost:4000/v1/agents?health_check=true" -H "Content-Type: application/json" -H "Authorization: Bearer your-key" ``` * + * Pass `?query=` to get the best matching agents ranked by semantic similarity: + * ``` + * curl -X GET "http://localhost:4000/v1/agents?query=translate+a+PDF+document&top_k=5" -H "Content-Type: application/json" -H "Authorization: Bearer your-key" ``` + * * Returns: List[AgentResponse] */ get: operations["get_agents_v1_agents_get"]; @@ -22487,6 +22491,8 @@ export interface components { } | null; /** Rpm Limit */ rpm_limit?: number | null; + /** Search Score */ + search_score?: number | null; /** Session Rpm Limit */ session_rpm_limit?: number | null; /** Session Tpm Limit */ @@ -30208,6 +30214,8 @@ export interface components { token_exchange_profile?: string | null; /** Upstream Resource */ upstream_resource?: string | null; + /** Upstream Token Header */ + upstream_token_header?: string | null; }; /** * MCPEnvVar @@ -58515,6 +58523,10 @@ export interface operations { query?: { /** @description When true, performs a GET request to each agent's URL. Agents with reachable URLs (HTTP status < 500) and agents without a URL are returned; unreachable agents are filtered out. */ health_check?: boolean; + /** @description Describe the task in natural language to rank the agents you can reach by semantic similarity over their name, description, and skills. Each result carries a search_score. Requires litellm_settings.agent_search_embedding_model. */ + query?: string | null; + /** @description With query: the maximum number of ranked agents to return. */ + top_k?: number; }; header?: never; path?: never; From 6de53732ee9e543dfc23a728cad0e4bdb9e36b4c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:44:17 -0700 Subject: [PATCH 019/105] fix(mcp): keep the virtual tool required lists as JSON arrays so /mcp/ tools/call validates --- .../proxy/_experimental/mcp_server/tool_search.py | 14 +++++++++----- .../mcp_server/test_mcp_tool_search.py | 15 +++++++++++++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/tool_search.py b/litellm/proxy/_experimental/mcp_server/tool_search.py index a33ab4de3b7..a5259e127fa 100644 --- a/litellm/proxy/_experimental/mcp_server/tool_search.py +++ b/litellm/proxy/_experimental/mcp_server/tool_search.py @@ -1,7 +1,7 @@ from __future__ import annotations import json -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final, TypedDict, assert_never @@ -51,7 +51,7 @@ class _ToolParamSchema(TypedDict, total=False): class _ToolInputSchema(TypedDict): type: ReadOnly[str] properties: ReadOnly[Mapping[str, _ToolParamSchema]] - required: ReadOnly[tuple[str, ...]] + required: ReadOnly[Sequence[str]] class VirtualToolDefinition(TypedDict): @@ -60,6 +60,10 @@ class VirtualToolDefinition(TypedDict): inputSchema: ReadOnly[_ToolInputSchema] +def _json_array(*items: str) -> Sequence[str]: + return list(items) # mutable-ok: jsonschema's metaschema only accepts a JSON array for required + + _MCP_TOOL_SEARCH_DEFINITION: Final[VirtualToolDefinition] = { "name": MCP_TOOL_SEARCH_TOOL_NAME, "description": "Search for MCP tools by keyword. Returns top matching tools with names, descriptions, and input schemas.", @@ -69,7 +73,7 @@ _MCP_TOOL_SEARCH_DEFINITION: Final[VirtualToolDefinition] = { "query": {"type": "string", "description": "Keywords to search for in tool names and descriptions."}, "top_k": {"type": "integer", "description": "Maximum number of results to return.", "default": 5}, }, - "required": ("query",), + "required": _json_array("query"), }, } @@ -82,7 +86,7 @@ _MCP_TOOL_CALL_DEFINITION: Final[VirtualToolDefinition] = { "tool_name": {"type": "string", "description": "The exact name of the MCP tool to call."}, "arguments": {"type": "object", "description": "Arguments to pass to the tool."}, }, - "required": ("tool_name",), + "required": _json_array("tool_name"), }, } @@ -99,7 +103,7 @@ _AGENT_SEARCH_DEFINITION: Final[VirtualToolDefinition] = { "default": DEFAULT_AGENT_SEARCH_TOP_K, }, }, - "required": ("query",), + "required": _json_array("query"), }, } diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py index dfe1e224c1f..bf0bd3c795e 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py @@ -123,7 +123,7 @@ class TestGetVirtualToolDefinitions: agent_tool = next(t for t in tools if t["name"] == AGENT_SEARCH_TOOL_NAME) props = agent_tool["inputSchema"]["properties"] assert set(props) == {"query", "top_k"} - assert agent_tool["inputSchema"]["required"] == ("query",) + assert agent_tool["inputSchema"]["required"] == ["query"] def test_has_mcp_tool_search(self) -> None: names = [t["name"] for t in get_virtual_tool_definitions()] @@ -138,7 +138,7 @@ class TestGetVirtualToolDefinitions: search_tool = next(t for t in tools if t["name"] == MCP_TOOL_SEARCH_TOOL_NAME) props = search_tool["inputSchema"]["properties"] assert "query" in props - assert search_tool["inputSchema"]["required"] == ("query",) + assert search_tool["inputSchema"]["required"] == ["query"] def test_mcp_tool_call_schema_has_tool_name_and_arguments(self) -> None: tools = get_virtual_tool_definitions() @@ -148,6 +148,17 @@ class TestGetVirtualToolDefinitions: assert "arguments" in props assert "tool_name" in call_tool["inputSchema"]["required"] + def test_input_schemas_validate_arguments_like_the_mcp_server_does(self) -> None: + from jsonschema import ValidationError, validate + from mcp.types import Tool + + for definition in get_virtual_tool_definitions(): + tool = Tool.model_validate(definition) + required_arguments = {name: "x" for name in tool.inputSchema["required"]} + validate(instance=required_arguments, schema=tool.inputSchema) + with pytest.raises(ValidationError): + validate(instance={}, schema=tool.inputSchema) + def test_all_tools_have_description(self) -> None: for tool in get_virtual_tool_definitions(): assert tool.get("description"), f"{tool['name']} missing description" From e5c3df2da2de304fd662707f07b0c09f597bb097 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 27 Aug 2026 16:44:30 -0700 Subject: [PATCH 020/105] fix(gpt-5): resolve temperature support from the model's default reasoning effort A gpt-5 model accepts a non-default temperature only while its effective reasoning effort resolves to "none". litellm had no representation of the effort a model applies when the request omits reasoning_effort, so it substituted supports_none_reasoning_effort, which is a different fact. Every model that supports "none" without defaulting to it therefore had temperature forwarded and rejected upstream, and because the carve-out returned before the drop_params branch, drop_params: true could not save it. Declare the fact instead. A new cost-map key, default_reasoning_effort, states the effort the provider applies when the request omits one, and one shared predicate resolves the effective effort from it: an explicit reasoning_effort wins, otherwise the declared default, otherwise the catalogue decides. That last step matters because the cost map is fetched from the published branch at import time, so it can be OLDER than the code reading it. On such a map every model looks undeclared, and reading that as "reasoning is active" would strip temperature from the 39 gpt-5.1/5.2/5.4 entries that accept it, a regression caused by data lag rather than by anything about the model. So an absent declaration is only meaningful once the catalogue carries the key at all; a map that predates the feature keeps the answer litellm gave before it existed, and the conservative answer applies from the moment the data lands. The top_p/logprobs/top_logprobs gate carried the same assumption spelled differently and now shares the predicate, as does the Responses API, which reimplemented the rule and is what the default /v1/messages bridge routes openai models through. Azure normalises its routing names in one resolver that every capability lookup goes through, which replaces its bespoke per-lookup rewrite. Declared on the 37 gpt-5.1/5.2/5.4 entries measured to accept temperature=0 today, so their behaviour is unchanged. The 23 gpt-5.5/5.6 entries that reject it stay undeclared and are fixed once the catalogue carries the key. Resolves LIT-3797 Resolves LIT-5028 --- ci_cd/generate_model_prices_schema.py | 9 ++ .../llms/azure/chat/gpt_5_transformation.py | 20 +-- .../llms/openai/chat/gpt_5_transformation.py | 80 ++++++++++-- .../llms/openai/responses/transformation.py | 24 +++- ...odel_prices_and_context_window_backup.json | 55 ++++++-- litellm/types/utils.py | 1 + litellm/utils.py | 41 ++++++ model_prices_and_context_window.json | 55 ++++++-- model_prices_and_context_window.schema.json | 12 ++ .../chat/test_azure_gpt5_transformation.py | 37 ++++++ .../test_openai_responses_transformation.py | 33 +++++ .../llms/openai/test_gpt5_transformation.py | 122 +++++++++++++++++- tests/test_litellm/test_utils.py | 28 ++++ 13 files changed, 473 insertions(+), 44 deletions(-) diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 5e1c4b0dcd9..57cc742d5c4 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -220,6 +220,15 @@ def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]: "description": "Highest reasoning effort the Bedrock output_config accepts for this model.", "enum": ["low", "medium", "high", "max", "xhigh"], }, + "default_reasoning_effort": { + "type": "string", + "description": ( + "Reasoning effort the provider applies when the request omits reasoning_effort. " + "Gates whether a non-default temperature or the top_p/logprobs sampling params are " + "accepted, which hold only when the effort resolves to 'none'." + ), + "enum": ["none", "minimal", "low", "medium", "high", "xhigh"], + }, "comment": STRING, "audio_transcription_config": STRING, } diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_5_transformation.py index d7584083327..6fdd277a04f 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_5_transformation.py @@ -19,19 +19,19 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): GPT5_SERIES_ROUTE = "gpt5_series/" @classmethod - def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool: - """Override to handle gpt5_series/ prefix used for Azure routing. + def _model_map_lookup_name(cls, model: str) -> str: + """Normalise an Azure routing name to its cost-map key. - The parent class calls ``_supports_factory(model, custom_llm_provider=None)`` - which fails to resolve ``gpt5_series/gpt-5.1`` to the correct Azure model - entry. Strip the prefix and prepend ``azure/`` so the lookup finds - ``azure/gpt-5.1`` in model_prices_and_context_window.json. + Neither ``gpt5_series/gpt-5.1`` nor a bare ``gpt-5.1`` is a key in + model_prices_and_context_window.json; ``azure/gpt-5.1`` is. Overriding the shared + resolver rather than one lookup means the supports, explicitly-disabled and + default-effort answers all read the same entry. """ if model.startswith(cls.GPT5_SERIES_ROUTE): - model = "azure/" + model[len(cls.GPT5_SERIES_ROUTE) :] - elif not model.startswith("azure/"): - model = "azure/" + model - return super()._supports_reasoning_effort_level(model, level) + return "azure/" + model[len(cls.GPT5_SERIES_ROUTE) :] + if model.startswith("azure/"): + return model + return "azure/" + model @classmethod def is_model_gpt_5_model(cls, model: str) -> bool: diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 3a65e4a9426..0223be300b0 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -6,11 +6,28 @@ import litellm from litellm.utils import ( _is_explicitly_disabled_factory, _supports_factory, + declared_value_factory, ) from .gpt_transformation import OpenAIGPTConfig +def _catalogue_declares_default_effort() -> bool: + """Whether the loaded cost map carries default_reasoning_effort for ANY entry. + + The map is fetched from the published branch at import time, so it can be OLDER than the + code reading it. On such a map every model looks undeclared, and treating that as "reasoning + is active" would silently strip temperature from the gpt-5.1/5.2/5.4 deployments that accept + it - a regression caused purely by data lag rather than by anything about the model. + + So the absence of the key is only meaningful once the catalogue is known to carry it at all. + A map that has never heard of the key predates the feature, and the honest answer there is + the one litellm gave before it existed. Scanning costs ~80us on the largest published map and + only on the fallback path, which is noise beside the request it precedes. + """ + return any(isinstance(entry, dict) and "default_reasoning_effort" in entry for entry in litellm.model_cost.values()) + + def _normalize_reasoning_effort_for_chat_completion( value: str | dict | None, ) -> str | None: @@ -114,6 +131,17 @@ class OpenAIGPT5Config(OpenAIGPTConfig): except (ValueError, IndexError): return False + @classmethod + def _model_map_lookup_name(cls, model: str) -> str: + """The name this model is looked up by in the cost map. + + Identity here, because an OpenAI model name is already its map key. Azure overrides + it: its routing prefixes are not map keys, so every capability lookup has to + normalise the name the same way, and doing that in ONE place is what keeps the + supports/disabled/default answers from disagreeing about which entry they read. + """ + return model + @classmethod def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool: """Check if the model supports a specific reasoning_effort level. @@ -123,11 +151,40 @@ class OpenAIGPT5Config(OpenAIGPTConfig): Returns False for unknown models (safe fallback). """ return _supports_factory( - model=model, + model=cls._model_map_lookup_name(model), custom_llm_provider=None, key=f"supports_{level}_reasoning_effort", ) + @classmethod + def effort_resolves_to_none(cls, model: str, effective_effort: str | None) -> bool: + """Whether this request's reasoning effort ends up as "none", which is the single + condition under which the provider accepts a non-default temperature or the + top_p/logprobs sampling params. + + An explicit reasoning_effort answers outright. When the request omits it the answer + is the model's DEFAULT effort, which only the map can state: supporting "none" is a + different fact from defaulting to it, and reading the former as the latter is what + forwarded temperature=0 to every gpt-5.5/5.6 deployment. + + An undeclared default resolves to False. The map not saying is not the model + saying no, so the gate takes the conservative branch: a param the provider would + have rejected gets dropped or refused with an actionable error, and a model + released before its map entry declares a default needs no code change to be safe. + """ + if effective_effort is not None: + return effective_effort == "none" + declared: Final = declared_value_factory( + model=cls._model_map_lookup_name(model), + custom_llm_provider=None, + key="default_reasoning_effort", + ) + if declared is not None: + return declared == "none" + if not _catalogue_declares_default_effort(): + return cls._supports_reasoning_effort_level(model, "none") + return False + @classmethod def _is_reasoning_effort_level_explicitly_disabled(cls, model: str, level: str) -> bool: """Return True only when the model map explicitly sets the capability to False. @@ -140,7 +197,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): Use this for opt-out checks where unknown models should be allowed through. """ return _is_explicitly_disabled_factory( - model=model, + model=cls._model_map_lookup_name(model), custom_llm_provider=None, key=f"supports_{level}_reasoning_effort", ) @@ -260,15 +317,16 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if supports_none: sampling_params: Final = ["logprobs", "top_logprobs", "top_p"] has_sampling: Final = any(p in non_default_params for p in sampling_params) - if has_sampling and effective_effort not in (None, "none"): + if has_sampling and not self.effort_resolves_to_none(model, effective_effort): if litellm.drop_params or drop_params: for p in sampling_params: non_default_params.pop(p, None) else: raise litellm.utils.UnsupportedParamsError( message=( - "gpt-5.1/5.2/5.4 only support logprobs, top_p, top_logprobs when " - f"reasoning_effort='none'. Current reasoning_effort='{effective_effort}'. " + f"{model} only supports logprobs, top_p, top_logprobs when reasoning_effort " + "resolves to 'none', either set explicitly on the request or declared as the " + f"model's default_reasoning_effort. Current reasoning_effort={effective_effort!r}. " "To drop unsupported params set `litellm.drop_params = True`" ), status_code=400, @@ -277,17 +335,19 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if "temperature" in non_default_params: temperature_value: Final[float | None] = non_default_params.pop("temperature") if temperature_value is not None: - # models supporting reasoning_effort="none" also support flexible temperature - if supports_none and (effective_effort == "none" or effective_effort is None) or temperature_value == 1: + # a non-default temperature rides on the effort resolving to "none", not on + # the model merely supporting it + if (supports_none and self.effort_resolves_to_none(model, effective_effort)) or temperature_value == 1: optional_params["temperature"] = temperature_value elif litellm.drop_params or drop_params: pass else: raise litellm.utils.UnsupportedParamsError( message=( - f"gpt-5 models (including gpt-5-codex) don't support temperature={temperature_value}. " - "Only temperature=1 is supported. " - "For gpt-5.1, temperature is supported when reasoning_effort='none' (or not specified, as it defaults to 'none'). " + f"{model} doesn't support temperature={temperature_value} while reasoning is " + "active. Only temperature=1 is supported unless reasoning_effort resolves to " + "'none', either set explicitly on the request or declared as the model's " + "default_reasoning_effort. " "To drop unsupported params set `litellm.drop_params = True`" ), status_code=400, diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index b2a69564908..2fa44cfc2e3 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -61,6 +61,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): key="supports_none_reasoning_effort", ) + @staticmethod + def _effort_resolves_to_none(model: str, effort: str | None) -> bool: + """Whether this request's reasoning effort ends up as "none", the one condition + under which a non-default temperature is accepted. + + Delegates to the chat-completions gpt-5 config so both surfaces answer from one + rule: the Responses API reaches the same models over a different wire, and a second + copy of the rule here is what let this surface keep forwarding temperature after the + chat surface stopped. + """ + from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config + + return OpenAIGPT5Config.effort_resolves_to_none(model, effort) + @staticmethod def _enforce_min_max_output_tokens(max_output_tokens: "int | None") -> "int | None": """Raise sub-minimum max_output_tokens up to the OpenAI Responses API minimum. @@ -116,17 +130,17 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): reasoning: Final = params.get("reasoning") or {} effort: Final = reasoning.get("effort") if isinstance(reasoning, dict) else None supports_none: Final = self._supports_reasoning_effort_none(model=model) - if supports_none and (effort == "none" or effort is None): + if supports_none and self._effort_resolves_to_none(model, effort): pass # flexible temperature allowed elif drop_params or litellm.drop_params: params.pop("temperature", None) else: raise litellm.UnsupportedParamsError( message=( - f"gpt-5 models don't support temperature={temperature}. " - "Only temperature=1 is supported. " - "For models like gpt-5.1/5.4, temperature is supported " - "when reasoning.effort='none' (or not specified). " + f"{model} doesn't support temperature={temperature} while reasoning is " + "active. Only temperature=1 is supported unless reasoning.effort resolves " + "to 'none', either set explicitly on the request or declared as the " + "model's default_reasoning_effort. " "To drop unsupported params set `litellm.drop_params = True`" ), status_code=400, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 77572f69b8b..895cc6c884f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3409,6 +3409,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -3456,6 +3457,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -3589,6 +3591,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, @@ -3630,6 +3633,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, @@ -3671,6 +3675,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, @@ -3712,6 +3717,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, @@ -3937,7 +3943,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/eu/gpt-5.1-chat": { "cache_read_input_token_cost": 1.4e-07, @@ -3972,7 +3979,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/eu/gpt-5.1-codex": { "deprecation_date": "2027-05-15", @@ -4247,7 +4255,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/global/gpt-5.1-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -4282,7 +4291,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/global/gpt-5.1-codex": { "deprecation_date": "2027-05-15", @@ -5367,6 +5377,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_minimal_reasoning_effort": true }, "azure/gpt-5.1-chat-2025-11-13": { @@ -5404,7 +5415,8 @@ "supports_system_messages": true, "supports_tool_choice": false, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/gpt-5.1-codex-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, @@ -5833,7 +5845,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/gpt-5.1-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -5868,7 +5881,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/gpt-5.1-codex": { "deprecation_date": "2027-05-15", @@ -6315,6 +6329,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -6354,6 +6369,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -6393,6 +6409,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -6438,6 +6455,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -6477,6 +6495,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -6516,6 +6535,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -7663,6 +7683,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true }, "azure/gpt-5.4-mini-2026-03-17": { @@ -7704,6 +7725,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true }, "azure/gpt-5.4-nano": { @@ -7745,6 +7767,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true }, "azure/gpt-5.4-nano-2026-03-17": { @@ -7786,6 +7809,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true }, "azure/gpt-image-1": { @@ -8856,7 +8880,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/us/gpt-5.1-chat": { "cache_read_input_token_cost": 1.4e-07, @@ -8891,7 +8916,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/us/gpt-5.1-codex": { "deprecation_date": "2027-05-15", @@ -26292,6 +26318,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": true }, @@ -26336,6 +26363,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": true }, @@ -26381,6 +26409,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": true }, @@ -26426,6 +26455,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -26471,6 +26501,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -27295,6 +27326,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -27343,6 +27375,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -27492,6 +27525,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, @@ -27543,6 +27577,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, @@ -27591,6 +27626,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, @@ -27639,6 +27675,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 95429e899c9..9b2f20c2259 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -165,6 +165,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_xhigh_reasoning_effort: bool | None supports_max_reasoning_effort: bool | None reasoning_effort_levels: ReadOnly[Sequence[str] | None] + default_reasoning_effort: ReadOnly[Literal["none", "minimal", "low", "medium", "high", "xhigh"] | None] supports_output_config: bool | None supports_image_size: bool | None bedrock_output_config_effort_ceiling: Literal["low", "medium", "high", "max", "xhigh"] | None diff --git a/litellm/utils.py b/litellm/utils.py index 520c40f67c0..37c7098813e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2596,6 +2596,46 @@ def _supports_factory(model: str, custom_llm_provider: str | None, key: str) -> return False +def declared_value_factory(model: str, custom_llm_provider: str | None, key: str) -> str | None: + """Return a string value the model map declares for *key*, or ``None`` when it says nothing. + + The string-valued sibling of :func:`_supports_factory` and + :func:`_is_explicitly_disabled_factory`, public where those two are not because it is read + from the provider configs rather than from this module, sharing their + ``get_llm_provider`` -> ``_get_model_info_helper`` chain and their unprefixed-twin + fallback (#20885), so a provider-prefixed entry that omits the key still answers + from the bare entry that carries it. + + ``None`` means "the map does not say", never "the map says no" - callers decide what + an unknown declaration implies, and for a capability gate that decision must be the + conservative one. + """ + try: + resolved: Final = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + resolved_model: Final = resolved[0] + resolved_provider: Final = resolved[1] + model_info: Final = _get_model_info_helper(model=resolved_model, custom_llm_provider=resolved_provider) + declared: Final = model_info.get(key) + if isinstance(declared, str): + return declared + bare_model_key: Final = _get_model_cost_key(resolved_model) + bare_entry: Final = litellm.model_cost.get(bare_model_key) if bare_model_key is not None else None + if isinstance(bare_entry, dict): + bare_declared: Final = bare_entry.get(key) + if isinstance(bare_declared, str): + return bare_declared + return None + except Exception as e: # noqa: BLE001 # an unreadable map entry means "not declared", never a failed call + verbose_logger.debug( + "Model not found or error in reading %s. You passed model=%s, custom_llm_provider=%s. Error: %s", + key, + model, + custom_llm_provider, + e, + ) + return None + + def _is_explicitly_disabled_factory(model: str, custom_llm_provider: str | None, key: str) -> bool: """Return True only when the model map explicitly sets *key* to ``False``. @@ -5890,6 +5930,7 @@ def _get_model_info_helper( supports_xhigh_reasoning_effort=_model_info.get("supports_xhigh_reasoning_effort", None), supports_max_reasoning_effort=_model_info.get("supports_max_reasoning_effort", None), reasoning_effort_levels=_model_info.get("reasoning_effort_levels", None), + default_reasoning_effort=_model_info.get("default_reasoning_effort", None), bedrock_output_config_effort_ceiling=_model_info.get("bedrock_output_config_effort_ceiling", None), bedrock_converse_supports_strict_tools=_model_info.get("bedrock_converse_supports_strict_tools", None), supports_computer_use=_model_info.get("supports_computer_use", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 77572f69b8b..895cc6c884f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3409,6 +3409,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -3456,6 +3457,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -3589,6 +3591,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, @@ -3630,6 +3633,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, @@ -3671,6 +3675,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, @@ -3712,6 +3717,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, @@ -3937,7 +3943,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/eu/gpt-5.1-chat": { "cache_read_input_token_cost": 1.4e-07, @@ -3972,7 +3979,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/eu/gpt-5.1-codex": { "deprecation_date": "2027-05-15", @@ -4247,7 +4255,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/global/gpt-5.1-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -4282,7 +4291,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/global/gpt-5.1-codex": { "deprecation_date": "2027-05-15", @@ -5367,6 +5377,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_minimal_reasoning_effort": true }, "azure/gpt-5.1-chat-2025-11-13": { @@ -5404,7 +5415,8 @@ "supports_system_messages": true, "supports_tool_choice": false, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/gpt-5.1-codex-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, @@ -5833,7 +5845,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/gpt-5.1-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -5868,7 +5881,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/gpt-5.1-codex": { "deprecation_date": "2027-05-15", @@ -6315,6 +6329,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -6354,6 +6369,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -6393,6 +6409,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -6438,6 +6455,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -6477,6 +6495,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -6516,6 +6535,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -7663,6 +7683,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true }, "azure/gpt-5.4-mini-2026-03-17": { @@ -7704,6 +7725,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true }, "azure/gpt-5.4-nano": { @@ -7745,6 +7767,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true }, "azure/gpt-5.4-nano-2026-03-17": { @@ -7786,6 +7809,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true }, "azure/gpt-image-1": { @@ -8856,7 +8880,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/us/gpt-5.1-chat": { "cache_read_input_token_cost": 1.4e-07, @@ -8891,7 +8916,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_none_reasoning_effort": true + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none" }, "azure/us/gpt-5.1-codex": { "deprecation_date": "2027-05-15", @@ -26292,6 +26318,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": true }, @@ -26336,6 +26363,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": true }, @@ -26381,6 +26409,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": true }, @@ -26426,6 +26455,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -26471,6 +26501,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -27295,6 +27326,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -27343,6 +27375,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -27492,6 +27525,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, @@ -27543,6 +27577,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, @@ -27591,6 +27626,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, @@ -27639,6 +27675,7 @@ "supports_vision": true, "supports_web_search": true, "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 6e837354c60..3f6d3b4f910 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -179,6 +179,18 @@ "comment": { "type": "string" }, + "default_reasoning_effort": { + "type": "string", + "description": "Reasoning effort the provider applies when the request omits reasoning_effort. Gates whether a non-default temperature or the top_p/logprobs sampling params are accepted, which hold only when the effort resolves to 'none'.", + "enum": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh" + ] + }, "deprecation_date": { "type": "string", "description": "Date the provider deprecates the model, YYYY-MM-DD.", diff --git a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py index 83562331b9a..e06cae97283 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py @@ -1,6 +1,7 @@ import pytest import litellm +from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config @@ -9,6 +10,15 @@ def config() -> AzureOpenAIGPT5Config: return AzureOpenAIGPT5Config() +@pytest.fixture(autouse=True) +def use_local_model_cost_map(monkeypatch: pytest.MonkeyPatch): + """Pin the bundled cost map: these gates read model-map capability keys, and the default + import path fetches the published map, which lags a key added in this repo.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url)) + litellm.add_known_models(model_cost_map=litellm.model_cost) + + def test_azure_gpt5_supports_reasoning_effort(config: AzureOpenAIGPT5Config): assert "reasoning_effort" in config.get_supported_openai_params(model="gpt-5") assert "reasoning_effort" in config.get_supported_openai_params( @@ -299,3 +309,30 @@ def test_azure_gpt5_1_does_not_support_logprobs(config: AzureOpenAIGPT5Config): supported_params = config.get_supported_openai_params(model="gpt-5.1") assert "logprobs" not in supported_params assert "top_logprobs" not in supported_params + + +class TestAzureResolvesTheDeclaredDefaultEffort: + """Azure reaches the same models under names that are not cost-map keys. Every capability + lookup therefore has to normalise the name identically, which is why the normalisation is + one overridden resolver rather than a rewrite inside a single lookup. + """ + + @pytest.mark.parametrize( + "model, temperature_survives", + [ + ("azure/gpt-5.1", True), + ("gpt5_series/gpt-5.1", True), + ("gpt-5.1", True), + ("azure/gpt-5.6-terra", False), + ("gpt5_series/gpt-5.6-terra", False), + ("azure/gpt-5.5", False), + ], + ) + def test_every_azure_name_shape_reads_the_same_entry(self, config, model, temperature_survives): + mapped = config.map_openai_params( + non_default_params={"temperature": 0}, + optional_params={}, + model=model, + drop_params=True, + ) + assert ("temperature" in mapped) is temperature_survives diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index c03c632363d..e314b94444b 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -1593,3 +1593,36 @@ class TestPromptCacheOptionsOnResponsesPath: "text": "hi", "prompt_cache_breakpoint": {"mode": "explicit"}, } + + +class TestResponsesSurfaceSharesTheEffortRule: + """The Responses API reaches the same gpt-5 models over a different wire, and the default + /v1/messages bridge for openai models routes through it. It carried its own copy of the + temperature rule, so fixing chat completions alone left this surface still forwarding + temperature to a model that rejects it. + """ + + @pytest.mark.parametrize( + "model, effort, temperature_survives", + [ + ("gpt-5.1", None, True), + ("gpt-5.4", None, True), + ("gpt-5.5", None, False), + ("gpt-5.6-terra", None, False), + ("gpt-5.6-sol", None, False), + ("gpt-5.6-terra", "none", True), + ("gpt-5.6-terra", "medium", False), + ], + ) + def test_temperature_follows_the_resolved_effort( + self, local_model_cost_map, model, effort, temperature_survives + ): + params = {"temperature": 0} + if effort is not None: + params["reasoning"] = {"effort": effort} + mapped = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=params, + model=model, + drop_params=True, + ) + assert ("temperature" in mapped) is temperature_survives diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index a35b75a6106..9c5bd34d59a 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -1,3 +1,5 @@ +import re + import pytest import litellm @@ -137,7 +139,7 @@ def test_gpt5_codex_temperature_error(config: OpenAIConfig): """Test that GPT-5-Codex raises error for unsupported temperature when drop_params=False.""" with pytest.raises( litellm.utils.UnsupportedParamsError, - match="gpt-5 models \\(including gpt-5-codex\\)", + match=re.escape("gpt-5-codex doesn't support temperature=0.7 while reasoning is active"), ): config.map_openai_params( non_default_params={"temperature": 0.7}, @@ -1385,3 +1387,121 @@ def test_gpt5_drops_xhigh_when_requested(config: OpenAIConfig): drop_params=True, ) assert "reasoning_effort" not in params + + +class TestDefaultReasoningEffortGatesSamplingParams: + """A non-default temperature rides on the effort RESOLVING to "none", which for a request + that omits reasoning_effort is the model's declared default_reasoning_effort - not on the + model merely supporting "none". gpt-5.5 and gpt-5.6 support it and do not default to it, + so reading one fact as the other forwarded temperature=0 and the provider rejected it. + + Every expectation below was measured against the live provider before being pinned here. + """ + + @pytest.mark.parametrize( + "model, effort, temperature_survives", + [ + # declares default_reasoning_effort="none": reasoning is off, sampling is free + ("gpt-5.1", None, True), + ("gpt-5.2", None, True), + ("gpt-5.4", None, True), + ("gpt-5.4-nano", None, True), + # declares no default: reasoning is active, so the provider takes only temperature=1 + ("gpt-5.5", None, False), + ("gpt-5.6", None, False), + ("gpt-5.6-terra", None, False), + ("gpt-5.6-sol", None, False), + # an explicit effort always wins over the declared default, both ways + ("gpt-5.6-terra", "none", True), + ("gpt-5.6-terra", "medium", False), + ("gpt-5.1", "medium", False), + ], + ) + def test_temperature_follows_the_resolved_effort(self, model, effort, temperature_survives): + params = {"temperature": 0} if effort is None else {"temperature": 0, "reasoning_effort": effort} + mapped = OpenAIGPT5Config().map_openai_params( + non_default_params=params, + optional_params={}, + model=model, + drop_params=True, + ) + assert ("temperature" in mapped) is temperature_survives + + @pytest.mark.parametrize("model, top_p_survives", [("gpt-5.1", True), ("gpt-5.6-terra", False)]) + def test_the_same_rule_gates_top_p(self, model, top_p_survives): + """top_p/logprobs are gated by the identical condition, so they were identically wrong.""" + mapped = OpenAIGPT5Config().map_openai_params( + non_default_params={"top_p": 0.5}, + optional_params={}, + model=model, + drop_params=True, + ) + assert ("top_p" in mapped) is top_p_survives + + def test_an_undeclared_model_is_refused_rather_than_forwarded(self): + """Without drop_params the caller gets an actionable 400 naming the remedy, instead of + the provider's own rejection arriving from an upstream it did not address.""" + with pytest.raises(litellm.utils.UnsupportedParamsError, match="default_reasoning_effort"): + OpenAIGPT5Config().map_openai_params( + non_default_params={"temperature": 0}, + optional_params={}, + model="gpt-5.6-terra", + drop_params=False, + ) + + +class TestACatalogueOlderThanTheCodeDoesNotStripTemperature: + """The cost map is fetched from the published branch at import time, so it can be OLDER than + the code reading it. On such a map every model looks undeclared, and reading that as + "reasoning is active" silently stripped temperature from the gpt-5.1/5.2/5.4 deployments that + accept it - a regression caused by data lag rather than by anything about the model. + + Absence of the key only means something once the catalogue is known to carry it at all. + """ + + @staticmethod + def _map_without_the_key(monkeypatch: pytest.MonkeyPatch) -> None: + stripped = { + name: {k: v for k, v in entry.items() if k != "default_reasoning_effort"} + if isinstance(entry, dict) + else entry + for name, entry in litellm.model_cost.items() + } + monkeypatch.setattr(litellm, "model_cost", stripped) + + @pytest.mark.parametrize("model", ["gpt-5.1", "gpt-5.2", "gpt-5.4", "gpt-5.4-nano"]) + def test_a_pre_feature_catalogue_keeps_the_answer_it_gave_before(self, monkeypatch, model): + """These models accept temperature=0, verified against the provider. On a map that predates + the key they must keep it, exactly as they did before this feature existed.""" + self._map_without_the_key(monkeypatch) + + mapped = OpenAIGPT5Config().map_openai_params( + non_default_params={"temperature": 0}, + optional_params={}, + model=model, + drop_params=True, + ) + assert mapped.get("temperature") == 0 + + @pytest.mark.parametrize("model", ["gpt-5.1", "gpt-5.4"]) + def test_the_same_holds_for_the_sampling_params(self, monkeypatch, model): + self._map_without_the_key(monkeypatch) + + mapped = OpenAIGPT5Config().map_openai_params( + non_default_params={"top_p": 0.5}, + optional_params={}, + model=model, + drop_params=True, + ) + assert mapped.get("top_p") == 0.5 + + def test_once_the_catalogue_declares_the_key_the_conservative_answer_returns(self): + """The bundled map DOES carry the key, so an undeclared model there is a real statement + that its default is not none, and temperature is dropped.""" + mapped = OpenAIGPT5Config().map_openai_params( + non_default_params={"temperature": 0}, + optional_params={}, + model="gpt-5.6-terra", + drop_params=True, + ) + assert "temperature" not in mapped diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index e9b2851f717..362e55680a3 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1018,6 +1018,10 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "type": "array", "items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]}, }, + "default_reasoning_effort": { + "type": "string", + "enum": ["none", "minimal", "low", "medium", "high", "xhigh"], + }, "supports_adaptive_thinking": {"type": "boolean"}, "supports_legacy_thinking": {"type": "boolean"}, "thinking_always_on": {"type": "boolean"}, @@ -5639,3 +5643,27 @@ def test_snapshot_exception_for_hook_preserves_suppress_context_flag() -> None: snapshot = _snapshot_exception_for_hook(e) assert snapshot.__suppress_context__ is False assert snapshot.__context__ is e.__context__ + + +class TestDefaultReasoningEffortHydration: + """`get_model_info` is the public shape every other capability key is readable through, so + the declared default has to survive hydration too, not only the raw-map fallback the + request-path gate happens to reach it by. + """ + + @pytest.mark.parametrize( + "model, provider", + [("gpt-5.1", "openai"), ("gpt-5.4", "openai"), ("azure/gpt-5.1", "azure")], + ) + def test_the_declared_default_survives_model_info_hydration(self, local_model_cost_map, model, provider): + from litellm.utils import _get_model_info_helper + + model_info = dict(_get_model_info_helper(model=model, custom_llm_provider=provider)) + assert model_info["default_reasoning_effort"] == "none" + + def test_a_model_that_declares_nothing_hydrates_to_none(self, local_model_cost_map): + """Absent means "the map does not say", which the gate reads as reasoning being active.""" + from litellm.utils import _get_model_info_helper + + model_info = dict(_get_model_info_helper(model="gpt-5.6-terra", custom_llm_provider="openai")) + assert model_info.get("default_reasoning_effort") is None From e9cc9c9bc3b06f652f35bf14e04305316a6035bd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:11:00 -0700 Subject: [PATCH 021/105] fix(a2a): attribute agent search embedding spend to the calling key --- .../_experimental/mcp_server/tool_search.py | 1 + litellm/proxy/agent_endpoints/agent_search.py | 19 ++++++++-- litellm/proxy/agent_endpoints/endpoints.py | 7 +++- .../mcp_server/test_mcp_tool_search.py | 1 + .../agent_endpoints/test_agent_search.py | 37 +++++++++++++++++-- 5 files changed, 56 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/tool_search.py b/litellm/proxy/_experimental/mcp_server/tool_search.py index a5259e127fa..f79765f6d01 100644 --- a/litellm/proxy/_experimental/mcp_server/tool_search.py +++ b/litellm/proxy/_experimental/mcp_server/tool_search.py @@ -142,6 +142,7 @@ async def handle_agent_search(query: str, top_k: int, user_api_key_dict: UserAPI router=llm_router, embedding_model=litellm.agent_search_embedding_model, index=global_agent_search_index, + user_api_key_dict=user_api_key_dict, ) match outcome: case AgentSearchHits(hits): diff --git a/litellm/proxy/agent_endpoints/agent_search.py b/litellm/proxy/agent_endpoints/agent_search.py index a8fc57eb229..896a22fbf9d 100644 --- a/litellm/proxy/agent_endpoints/agent_search.py +++ b/litellm/proxy/agent_endpoints/agent_search.py @@ -16,6 +16,7 @@ from litellm.exceptions import BudgetExceededError from litellm.types.agents import AgentResponse if TYPE_CHECKING: + from litellm.proxy._types import UserAPIKeyAuth from litellm.router import Router DEFAULT_AGENT_SEARCH_TOP_K: Final = 5 @@ -122,10 +123,21 @@ def cosine_similarity(left: Vector, right: Vector) -> float: return dot / norms if norms else 0.0 -def router_embedder(router: Router, embedding_model: str) -> Embedder: +def embedding_spend_metadata(user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + return { # mutable-ok: the router mutates the metadata dict it is handed + **LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict), + "user_api_key": user_api_key_dict.api_key, + } + + +def router_embedder(router: Router, embedding_model: str, user_api_key_dict: UserAPIKeyAuth) -> Embedder: async def embed(texts: Sequence[str]) -> Sequence[Vector]: batch: Final = list(texts) # mutable-ok: Router.aembedding accepts only str | list input - response: Final = await router.aembedding(model=embedding_model, input=batch) + response: Final = await router.aembedding( + model=embedding_model, input=batch, metadata=embedding_spend_metadata(user_api_key_dict) + ) return tuple(item.embedding for item in _EmbeddingData.model_validate(response.model_dump()).data) return embed @@ -174,6 +186,7 @@ async def search_agents( router: Router | None, embedding_model: str | None, index: AgentSearchIndex, + user_api_key_dict: UserAPIKeyAuth, ) -> AgentSearchOutcome: if embedding_model is None: return AgentSearchNotConfigured( @@ -181,4 +194,4 @@ async def search_agents( ) if router is None: return AgentSearchNotConfigured(reason="agent search needs a model_list so the embedding model can be called") - return await index.search(query, agents, top_k, router_embedder(router, embedding_model)) + return await index.search(query, agents, top_k, router_embedder(router, embedding_model, user_api_key_dict)) diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py index cfb0597cc2a..b6c41a17503 100644 --- a/litellm/proxy/agent_endpoints/endpoints.py +++ b/litellm/proxy/agent_endpoints/endpoints.py @@ -231,7 +231,9 @@ def _agent_search_error(status_code: int, error: str, message: str) -> HTTPExcep return HTTPException(status_code=status_code, detail=detail) -async def _rank_agents_by_query(query: str, agents: Sequence[AgentResponse], top_k: int) -> tuple[AgentResponse, ...]: +async def _rank_agents_by_query( + query: str, agents: Sequence[AgentResponse], top_k: int, user_api_key_dict: UserAPIKeyAuth +) -> tuple[AgentResponse, ...]: from litellm.proxy.proxy_server import llm_router outcome: Final = await search_agents( @@ -241,6 +243,7 @@ async def _rank_agents_by_query(query: str, agents: Sequence[AgentResponse], top router=llm_router, embedding_model=litellm.agent_search_embedding_model, index=global_agent_search_index, + user_api_key_dict=user_api_key_dict, ) match outcome: case AgentSearchHits(hits): @@ -376,7 +379,7 @@ async def get_agents( if query is None: return returned_agents - return await _rank_agents_by_query(query, returned_agents, top_k) + return await _rank_agents_by_query(query, returned_agents, top_k, user_api_key_dict) except HTTPException: raise except Exception as e: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py index bf0bd3c795e..16221f44efe 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py @@ -566,6 +566,7 @@ class TestCallToolRestApiVirtualTools: result = await self._get_call_fn()(request=request, user_api_key_dict=user_api_key_dict) assert result.isError is False + assert mock_search.await_args.kwargs["user_api_key_dict"] is user_api_key_dict assert json.loads(result.content[0].text) == [ { "agent_id": "translator", diff --git a/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py b/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py index f6313eade9e..4b674eb142b 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py @@ -24,6 +24,8 @@ from litellm.proxy.agent_endpoints.auth.agent_permission_handler import Restrict from litellm.proxy.agent_endpoints.endpoints import router, user_api_key_auth from litellm.types.agents import AgentResponse +CALLER: Final = UserAPIKeyAuth(api_key="hashed-caller-key", team_id="team-1", user_id="user-1") + TRANSLATOR: Final = AgentResponse( agent_id="translator", agent_name="document-translator", @@ -150,21 +152,23 @@ class TestSearchAgents: @pytest.mark.asyncio async def test_no_embedding_model_is_not_configured(self) -> None: outcome = await search_agents( - "q", AGENTS, 5, router=MagicMock(), embedding_model=None, index=AgentSearchIndex() + "q", AGENTS, 5, router=MagicMock(), embedding_model=None, index=AgentSearchIndex(), user_api_key_dict=CALLER ) assert isinstance(outcome, AgentSearchNotConfigured) assert "agent_search_embedding_model" in outcome.reason @pytest.mark.asyncio async def test_no_router_is_not_configured(self) -> None: - outcome = await search_agents("q", AGENTS, 5, router=None, embedding_model="m", index=AgentSearchIndex()) + outcome = await search_agents( + "q", AGENTS, 5, router=None, embedding_model="m", index=AgentSearchIndex(), user_api_key_dict=CALLER + ) assert isinstance(outcome, AgentSearchNotConfigured) @pytest.mark.asyncio async def test_router_embeddings_are_read_from_the_response(self) -> None: router = MagicMock() router.aembedding = AsyncMock( - side_effect=lambda model, input: litellm.EmbeddingResponse( + side_effect=lambda model, input, metadata: litellm.EmbeddingResponse( model=model, data=[{"object": "embedding", "index": i, "embedding": list(VECTORS[t])} for i, t in enumerate(input)], ) @@ -176,11 +180,35 @@ class TestSearchAgents: router=router, embedding_model="text-embedding-3-small", index=AgentSearchIndex(), + user_api_key_dict=CALLER, ) assert isinstance(outcome, AgentSearchHits) assert [hit.agent.agent_id for hit in outcome.hits] == ["translator"] assert router.aembedding.await_args.kwargs["model"] == "text-embedding-3-small" + @pytest.mark.asyncio + async def test_embedding_spend_is_attributed_to_the_calling_key(self) -> None: + router = MagicMock() + router.aembedding = AsyncMock( + side_effect=lambda model, input, metadata: litellm.EmbeddingResponse( + model=model, + data=[{"object": "embedding", "index": i, "embedding": list(VECTORS[t])} for i, t in enumerate(input)], + ) + ) + await search_agents( + "language translation", + AGENTS, + 1, + router=router, + embedding_model="text-embedding-3-small", + index=AgentSearchIndex(), + user_api_key_dict=CALLER, + ) + metadata = router.aembedding.await_args.kwargs["metadata"] + assert metadata["user_api_key"] == "hashed-caller-key" + assert metadata["user_api_key_team_id"] == "team-1" + assert metadata["user_api_key_user_id"] == "user-1" + def _client(role: LitellmUserRoles) -> TestClient: app = FastAPI() @@ -205,7 +233,7 @@ def registry(monkeypatch: pytest.MonkeyPatch) -> MagicMock: def embedding_router(monkeypatch: pytest.MonkeyPatch) -> MagicMock: router = MagicMock() router.aembedding = AsyncMock( - side_effect=lambda model, input: litellm.EmbeddingResponse( + side_effect=lambda model, input, metadata: litellm.EmbeddingResponse( model=model, data=[{"object": "embedding", "index": i, "embedding": list(VECTORS[t])} for i, t in enumerate(input)], ) @@ -231,6 +259,7 @@ class TestGetAgentsQuery: body = response.json() assert [agent["agent_id"] for agent in body] == ["translator", "trip"] assert body[0]["search_score"] > body[1]["search_score"] + assert embedding_router.aembedding.await_args.kwargs["metadata"]["user_api_key_user_id"] == "u" def test_without_query_the_list_is_unchanged_and_unscored( self, registry: MagicMock, embedding_router: MagicMock, no_db: None From 837bcba32d0093a5f6263c27d0dc32a29e7c0c7b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:31:45 -0700 Subject: [PATCH 022/105] fix(model_prices): add bedrock_mantle gpt-5.5/5.4 272K tiers, align sol with AWS invoice AWS bills a Bedrock GPT-5.5 or GPT-5.4 prompt past 272K tokens under the long-context usage types for the whole prompt, at 2x input, 2x cache read, and 1.5x output, and the cost map only had the flat rates, so a 300K prompt was logged at half of what the invoice charges. The map's promo rates for gpt-5.6-sol are 20% under the $5.50 input, $33.00 output, $0.55 cache read, and $6.88 cache write per million the invoice bills. Adds the *_above_272k_tokens fields to gpt-5.5 and gpt-5.4, moves sol's base and tier rates to the invoiced ones, replaces the test that pinned the flat behaviour with one that pins the invoiced numbers, and updates the sol pins in the mantle transformation tests --- ...odel_prices_and_context_window_backup.json | 22 ++++-- model_prices_and_context_window.json | 22 ++++-- .../llm_cost_calc/test_llm_cost_calc_utils.py | 73 +++++++++++++------ ...bedrock_mantle_responses_transformation.py | 4 +- 4 files changed, 82 insertions(+), 39 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9dd0382a185..aa33e0d5010 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -49829,14 +49829,14 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-5.6-sol": { - "input_cost_per_token": 4.4e-06, - "input_cost_per_token_above_272k_tokens": 8.8e-06, - "cache_creation_input_token_cost": 5.5e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.1e-05, - "cache_read_input_token_cost": 4.4e-07, - "cache_read_input_token_cost_above_272k_tokens": 8.8e-07, - "output_cost_per_token": 2.2e-05, - "output_cost_per_token_above_272k_tokens": 3.3e-05, + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -50113,8 +50113,11 @@ }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -50140,8 +50143,11 @@ }, "bedrock_mantle/openai.gpt-5.4": { "input_cost_per_token": 2.75e-06, + "input_cost_per_token_above_272k_tokens": 5.5e-06, "cache_read_input_token_cost": 2.75e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "output_cost_per_token": 1.65e-05, + "output_cost_per_token_above_272k_tokens": 2.475e-05, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9dd0382a185..aa33e0d5010 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -49829,14 +49829,14 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-5.6-sol": { - "input_cost_per_token": 4.4e-06, - "input_cost_per_token_above_272k_tokens": 8.8e-06, - "cache_creation_input_token_cost": 5.5e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.1e-05, - "cache_read_input_token_cost": 4.4e-07, - "cache_read_input_token_cost_above_272k_tokens": 8.8e-07, - "output_cost_per_token": 2.2e-05, - "output_cost_per_token_above_272k_tokens": 3.3e-05, + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -50113,8 +50113,11 @@ }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -50140,8 +50143,11 @@ }, "bedrock_mantle/openai.gpt-5.4": { "input_cost_per_token": 2.75e-06, + "input_cost_per_token_above_272k_tokens": 5.5e-06, "cache_read_input_token_cost": 2.75e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "output_cost_per_token": 1.65e-05, + "output_cost_per_token_above_272k_tokens": 2.475e-05, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index ce90719789a..8ccd8235739 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -532,40 +532,71 @@ def test_generic_cost_per_token_bedrock_mantle_gpt56_long_context(_local_model_c @pytest.mark.parametrize( - "model", + "model,input_rate,cache_read_rate,output_rate,long_input_rate,long_cache_read_rate,long_output_rate", [ - "bedrock_mantle/openai.gpt-5.5", - "bedrock_mantle/openai.gpt-5.4", + ("bedrock_mantle/openai.gpt-5.5", 5.5e-06, 5.5e-07, 3.3e-05, 1.1e-05, 1.1e-06, 4.95e-05), + ("bedrock_mantle/openai.gpt-5.4", 2.75e-06, 2.75e-07, 1.65e-05, 5.5e-06, 5.5e-07, 2.475e-05), + ("bedrock_mantle/openai.gpt-5.6-sol", 5.5e-06, 5.5e-07, 3.3e-05, 1.1e-05, 1.1e-06, 4.95e-05), ], ) -def test_generic_cost_per_token_bedrock_mantle_gpt55_gpt54_long_context_flat_rate(_local_model_cost_map, model): - """Bedrock serves gpt-5.5 and gpt-5.4 up to its enforced 1,050,000-token prompt maximum and documents - no long-context tier for them, so a prompt past 272K is billed at the flat per-token rates.""" +def test_generic_cost_per_token_bedrock_mantle_gpt5_matches_aws_invoiced_rates( + _local_model_cost_map, + model, + input_rate, + cache_read_rate, + output_rate, + long_input_rate, + long_cache_read_rate, + long_output_rate, +): + """AWS bills a Bedrock GPT-5.x prompt past 272K under its long-context usage types, the whole prompt at + 2x input, 2x cache read, and 1.5x output. The flat rates undercounted a 300K gpt-5.5 prompt by half and + sol's base rates sat 20% under the invoice.""" - model_cost_map = litellm.model_cost[model] - assert model_cost_map["max_input_tokens"] == 1050000 - assert [key for key in model_cost_map if "above_272k" in key] == [] - - served_prompt_tokens = 1030590 cached_tokens = 100000 completion_tokens = 1000 - usage = Usage( - prompt_tokens=served_prompt_tokens, + + invoiced_prompt_tokens = 300238 + long_usage = Usage( + prompt_tokens=invoiced_prompt_tokens, completion_tokens=completion_tokens, - total_tokens=served_prompt_tokens + completion_tokens, + total_tokens=invoiced_prompt_tokens + completion_tokens, prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), ) - prompt_cost, completion_cost = generic_cost_per_token( + long_prompt_cost, long_completion_cost = generic_cost_per_token( model=model, - usage=usage, + usage=long_usage, custom_llm_provider="bedrock_mantle", ) - assert round(prompt_cost, 10) == round( - model_cost_map["input_cost_per_token"] * (served_prompt_tokens - cached_tokens) - + model_cost_map["cache_read_input_token_cost"] * cached_tokens, - 10, + assert long_prompt_cost == pytest.approx( + long_input_rate * (invoiced_prompt_tokens - cached_tokens) + long_cache_read_rate * cached_tokens ) - assert round(completion_cost, 10) == round(model_cost_map["output_cost_per_token"] * completion_tokens, 10) + assert long_completion_cost == pytest.approx(long_output_rate * completion_tokens) + + threshold_prompt_tokens = 272000 + short_usage = Usage( + prompt_tokens=threshold_prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=threshold_prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), + ) + short_prompt_cost, short_completion_cost = generic_cost_per_token( + model=model, + usage=short_usage, + custom_llm_provider="bedrock_mantle", + ) + assert short_prompt_cost == pytest.approx( + input_rate * (threshold_prompt_tokens - cached_tokens) + cache_read_rate * cached_tokens + ) + assert short_completion_cost == pytest.approx(output_rate * completion_tokens) + + +def test_bedrock_mantle_gpt56_sol_cache_write_matches_aws_invoiced_rate(_local_model_cost_map): + """The invoice bills sol 30-minute cache writes at $6.88 per million tokens, 1.25x the $5.50 input rate.""" + + sol = litellm.model_cost["bedrock_mantle/openai.gpt-5.6-sol"] + assert sol["cache_creation_input_token_cost"] == pytest.approx(6.875e-06) + assert sol["cache_creation_input_token_cost_above_272k_tokens"] == pytest.approx(1.375e-05) def test_generic_cost_per_token_honors_non_standard_above_threshold(): diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index a8a20670d4e..0033f4467bb 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -1695,7 +1695,7 @@ class TestBedrockMantleResponsesPricing: @pytest.mark.parametrize( "model, input_cost, cache_creation_cost, cache_read_cost, output_cost", [ - ("openai.gpt-5.6-sol", 4.4e-06, 5.5e-06, 4.4e-07, 2.2e-05), + ("openai.gpt-5.6-sol", 5.5e-06, 6.875e-06, 5.5e-07, 3.3e-05), ("openai.gpt-5.6-terra", 2.2e-06, 2.75e-06, 2.2e-07, 1.32e-05), ("openai.gpt-5.6-luna", 2.2e-07, 2.75e-07, 2.2e-08, 1.32e-06), ], @@ -1718,7 +1718,7 @@ class TestBedrockMantleResponsesPricing: @pytest.mark.parametrize( "model, input_cost, output_cost", [ - ("openai.gpt-5.6-sol", 4.4e-06, 2.2e-05), + ("openai.gpt-5.6-sol", 5.5e-06, 3.3e-05), ("openai.gpt-5.6-terra", 2.2e-06, 1.32e-05), ("openai.gpt-5.6-luna", 2.2e-07, 1.32e-06), ], From 8e455897a41fd16b462c8f43f108c5463bf45dca Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:33:33 -0700 Subject: [PATCH 023/105] fix(a2a): key the agent search vector cache by embedding model and re-embed on dimension changes --- litellm/proxy/agent_endpoints/agent_search.py | 53 ++++++++++++++----- .../agent_endpoints/test_agent_search.py | 51 +++++++++++++++--- 2 files changed, 85 insertions(+), 19 deletions(-) diff --git a/litellm/proxy/agent_endpoints/agent_search.py b/litellm/proxy/agent_endpoints/agent_search.py index 896a22fbf9d..88373e26f51 100644 --- a/litellm/proxy/agent_endpoints/agent_search.py +++ b/litellm/proxy/agent_endpoints/agent_search.py @@ -143,31 +143,56 @@ def router_embedder(router: Router, embedding_model: str, user_api_key_dict: Use return embed +_NO_VECTORS: Final[Mapping[str, Vector]] = MappingProxyType({}) + + +async def _embed_all(embed: Embedder, texts: Sequence[str]) -> tuple[Vector, ...] | AgentSearchEmbeddingFailed: + try: + vectors: Final = tuple(await embed(texts)) + except (OpenAIError, ValueError, BudgetExceededError) as exc: + return AgentSearchEmbeddingFailed(reason=f"embedding the search query failed: {exc}") + if len(vectors) != len(texts): + return AgentSearchEmbeddingFailed( + reason=f"embedding model returned {len(vectors)} vectors for {len(texts)} inputs" + ) + return vectors + + class AgentSearchIndex: - """Caches one vector per distinct agent text, so repeat searches only embed the query.""" + """Caches one vector per distinct agent text per embedding model, so repeat searches only embed the query.""" def __init__(self) -> None: - self._vectors: Mapping[str, Vector] = MappingProxyType({}) + self._vectors: Mapping[str, Mapping[str, Vector]] = MappingProxyType({}) async def search( - self, query: str, agents: Sequence[AgentResponse], top_k: int, embed: Embedder + self, query: str, agents: Sequence[AgentResponse], top_k: int, embed: Embedder, embedding_model: str ) -> AgentSearchHits | AgentSearchEmbeddingFailed: if not agents: return AgentSearchHits(hits=()) texts: Final = tuple(agent_search_text(agent) for agent in agents) - missing: Final = tuple(dict.fromkeys(text for text in texts if text not in self._vectors)) - try: - vectors: Final = await embed((query, *missing)) - except (OpenAIError, ValueError, BudgetExceededError) as exc: - return AgentSearchEmbeddingFailed(reason=f"embedding the search query failed: {exc}") - if len(vectors) != len(missing) + 1: + cached: Final = self._vectors.get(embedding_model, _NO_VECTORS) + missing: Final = tuple(dict.fromkeys(text for text in texts if text not in cached)) + embedded: Final = await _embed_all(embed, (query, *missing)) + if isinstance(embedded, AgentSearchEmbeddingFailed): + return embedded + query_vector: Final = embedded[0] + stale: Final = tuple( + dict.fromkeys(text for text in texts if text in cached and len(cached[text]) != len(query_vector)) + ) + refreshed: Final = await _embed_all(embed, stale) if stale else () + if isinstance(refreshed, AgentSearchEmbeddingFailed): + return refreshed + vectors: Final = MappingProxyType( + dict(chain(cached.items(), zip(missing, embedded[1:], strict=True), zip(stale, refreshed, strict=True))) + ) + if any(len(vectors[text]) != len(query_vector) for text in texts): return AgentSearchEmbeddingFailed( - reason=f"embedding model returned {len(vectors)} vectors for {len(missing) + 1} inputs" + reason=f"embedding model {embedding_model} returned vectors of mixed dimensions" ) - self._vectors = MappingProxyType(dict(chain(self._vectors.items(), zip(missing, vectors[1:], strict=True)))) + self._vectors = MappingProxyType({**self._vectors, embedding_model: vectors}) ranked: Final = sorted( ( - AgentSearchHit(agent=agent, score=cosine_similarity(vectors[0], self._vectors[text])) + AgentSearchHit(agent=agent, score=cosine_similarity(query_vector, vectors[text])) for agent, text in zip(agents, texts, strict=True) ), key=lambda hit: hit.score, @@ -194,4 +219,6 @@ async def search_agents( ) if router is None: return AgentSearchNotConfigured(reason="agent search needs a model_list so the embedding model can be called") - return await index.search(query, agents, top_k, router_embedder(router, embedding_model, user_api_key_dict)) + return await index.search( + query, agents, top_k, router_embedder(router, embedding_model, user_api_key_dict), embedding_model + ) diff --git a/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py b/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py index 4b674eb142b..4699b45ec94 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py @@ -77,6 +77,16 @@ class FakeEmbedder: return tuple(VECTORS[text] for text in texts) +class FixedDimensionEmbedder: + def __init__(self, dimensions: int) -> None: + self.dimensions: Final = dimensions + self.calls: list[tuple[str, ...]] = [] # mutable-ok: test spy recording embed inputs + + async def __call__(self, texts: Sequence[str]) -> Sequence[Vector]: + self.calls.append(tuple(texts)) + return tuple((1.0,) * self.dimensions for _ in texts) + + class TestAgentSearchText: def test_joins_name_description_and_skills_with_tags(self) -> None: assert agent_search_text(TRANSLATOR) == ( @@ -109,7 +119,9 @@ class TestCosineSimilarity: class TestAgentSearchIndex: @pytest.mark.asyncio async def test_ranks_by_similarity_and_truncates_to_top_k(self) -> None: - outcome = await AgentSearchIndex().search("language translation", AGENTS, top_k=2, embed=FakeEmbedder()) + outcome = await AgentSearchIndex().search( + "language translation", AGENTS, top_k=2, embed=FakeEmbedder(), embedding_model="m" + ) assert isinstance(outcome, AgentSearchHits) assert [hit.agent.agent_id for hit in outcome.hits] == ["translator", "trip"] assert outcome.hits[0].score > outcome.hits[1].score @@ -118,15 +130,42 @@ class TestAgentSearchIndex: async def test_second_search_only_embeds_the_query(self) -> None: index = AgentSearchIndex() embedder = FakeEmbedder() - await index.search("language translation", AGENTS, top_k=5, embed=embedder) - await index.search("language translation", AGENTS, top_k=5, embed=embedder) + await index.search("language translation", AGENTS, top_k=5, embed=embedder, embedding_model="m") + await index.search("language translation", AGENTS, top_k=5, embed=embedder, embedding_model="m") assert len(embedder.calls[0]) == 1 + len(AGENTS) assert embedder.calls[1] == ("language translation",) + @pytest.mark.asyncio + async def test_switching_embedding_models_does_not_reuse_cached_vectors(self) -> None: + index = AgentSearchIndex() + await index.search("language translation", AGENTS, top_k=5, embed=FakeEmbedder(), embedding_model="small") + wide = FixedDimensionEmbedder(2) + outcome = await index.search("language translation", AGENTS, top_k=5, embed=wide, embedding_model="wide") + assert isinstance(outcome, AgentSearchHits) + assert len(wide.calls[0]) == 1 + len(AGENTS) + + @pytest.mark.asyncio + async def test_cached_vectors_of_another_dimension_are_re_embedded(self) -> None: + index = AgentSearchIndex() + await index.search("language translation", AGENTS, top_k=5, embed=FakeEmbedder(), embedding_model="m") + fallback = FixedDimensionEmbedder(2) + outcome = await index.search("language translation", AGENTS, top_k=5, embed=fallback, embedding_model="m") + assert isinstance(outcome, AgentSearchHits) + assert fallback.calls == [("language translation",), tuple(agent_search_text(agent) for agent in AGENTS)] + + @pytest.mark.asyncio + async def test_mixed_dimensions_in_one_batch_become_embedding_failed(self) -> None: + async def mixed(texts: Sequence[str]) -> Sequence[Vector]: + return ((1.0, 0.0), *((1.0, 0.0, 0.0) for _ in texts[1:])) + + outcome = await AgentSearchIndex().search("q", AGENTS, top_k=5, embed=mixed, embedding_model="m") + assert isinstance(outcome, AgentSearchEmbeddingFailed) + assert "mixed dimensions" in outcome.reason + @pytest.mark.asyncio async def test_empty_registry_returns_no_hits_without_embedding(self) -> None: embedder = FakeEmbedder() - outcome = await AgentSearchIndex().search("anything", (), top_k=5, embed=embedder) + outcome = await AgentSearchIndex().search("anything", (), top_k=5, embed=embedder, embedding_model="m") assert outcome == AgentSearchHits(hits=()) assert embedder.calls == [] @@ -135,7 +174,7 @@ class TestAgentSearchIndex: async def failing(texts: Sequence[str]) -> Sequence[Vector]: raise APIConnectionError(request=MagicMock()) - outcome = await AgentSearchIndex().search("q", AGENTS, top_k=5, embed=failing) + outcome = await AgentSearchIndex().search("q", AGENTS, top_k=5, embed=failing, embedding_model="m") assert isinstance(outcome, AgentSearchEmbeddingFailed) assert "embedding the search query failed" in outcome.reason @@ -144,7 +183,7 @@ class TestAgentSearchIndex: async def short(texts: Sequence[str]) -> Sequence[Vector]: return ((1.0, 0.0, 0.0),) - outcome = await AgentSearchIndex().search("q", AGENTS, top_k=5, embed=short) + outcome = await AgentSearchIndex().search("q", AGENTS, top_k=5, embed=short, embedding_model="m") assert isinstance(outcome, AgentSearchEmbeddingFailed) From db02cf81e5e8eddd28fbc9c704d9136fe8dd111f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:38:00 -0700 Subject: [PATCH 024/105] fix(a2a): re-embed the query with the agents in one call when cached vectors change dimension --- litellm/proxy/agent_endpoints/agent_search.py | 50 +++++++++++++------ .../agent_endpoints/test_agent_search.py | 5 +- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/agent_endpoints/agent_search.py b/litellm/proxy/agent_endpoints/agent_search.py index 88373e26f51..c249079ccac 100644 --- a/litellm/proxy/agent_endpoints/agent_search.py +++ b/litellm/proxy/agent_endpoints/agent_search.py @@ -158,6 +158,35 @@ async def _embed_all(embed: Embedder, texts: Sequence[str]) -> tuple[Vector, ... return vectors +@dataclass(frozen=True, slots=True) +class _Embedded: + query_vector: Vector + vectors: Mapping[str, Vector] + + +def _same_dimension(query_vector: Vector, vectors: Mapping[str, Vector], texts: Sequence[str]) -> bool: + return all(len(vectors[text]) == len(query_vector) for text in texts) + + +async def _embed_query_and_agents( + embed: Embedder, query: str, texts: Sequence[str], cached: Mapping[str, Vector] +) -> _Embedded | AgentSearchEmbeddingFailed: + missing: Final = tuple(dict.fromkeys(text for text in texts if text not in cached)) + embedded: Final = await _embed_all(embed, (query, *missing)) + if isinstance(embedded, AgentSearchEmbeddingFailed): + return embedded + vectors: Final = MappingProxyType(dict(chain(cached.items(), zip(missing, embedded[1:], strict=True)))) + if _same_dimension(embedded[0], vectors, texts): + return _Embedded(query_vector=embedded[0], vectors=vectors) + unique: Final = tuple(dict.fromkeys(texts)) + reembedded: Final = await _embed_all(embed, (query, *unique)) + if isinstance(reembedded, AgentSearchEmbeddingFailed): + return reembedded + return _Embedded( + query_vector=reembedded[0], vectors=MappingProxyType(dict(zip(unique, reembedded[1:], strict=True))) + ) + + class AgentSearchIndex: """Caches one vector per distinct agent text per embedding model, so repeat searches only embed the query.""" @@ -171,28 +200,19 @@ class AgentSearchIndex: return AgentSearchHits(hits=()) texts: Final = tuple(agent_search_text(agent) for agent in agents) cached: Final = self._vectors.get(embedding_model, _NO_VECTORS) - missing: Final = tuple(dict.fromkeys(text for text in texts if text not in cached)) - embedded: Final = await _embed_all(embed, (query, *missing)) + embedded: Final = await _embed_query_and_agents(embed, query, texts, cached) if isinstance(embedded, AgentSearchEmbeddingFailed): return embedded - query_vector: Final = embedded[0] - stale: Final = tuple( - dict.fromkeys(text for text in texts if text in cached and len(cached[text]) != len(query_vector)) - ) - refreshed: Final = await _embed_all(embed, stale) if stale else () - if isinstance(refreshed, AgentSearchEmbeddingFailed): - return refreshed - vectors: Final = MappingProxyType( - dict(chain(cached.items(), zip(missing, embedded[1:], strict=True), zip(stale, refreshed, strict=True))) - ) - if any(len(vectors[text]) != len(query_vector) for text in texts): + if not _same_dimension(embedded.query_vector, embedded.vectors, texts): return AgentSearchEmbeddingFailed( reason=f"embedding model {embedding_model} returned vectors of mixed dimensions" ) - self._vectors = MappingProxyType({**self._vectors, embedding_model: vectors}) + self._vectors = MappingProxyType( + {**self._vectors, embedding_model: MappingProxyType({**cached, **embedded.vectors})} + ) ranked: Final = sorted( ( - AgentSearchHit(agent=agent, score=cosine_similarity(query_vector, vectors[text])) + AgentSearchHit(agent=agent, score=cosine_similarity(embedded.query_vector, embedded.vectors[text])) for agent, text in zip(agents, texts, strict=True) ), key=lambda hit: hit.score, diff --git a/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py b/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py index 4699b45ec94..93f29c4a1d2 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py @@ -151,7 +151,10 @@ class TestAgentSearchIndex: fallback = FixedDimensionEmbedder(2) outcome = await index.search("language translation", AGENTS, top_k=5, embed=fallback, embedding_model="m") assert isinstance(outcome, AgentSearchHits) - assert fallback.calls == [("language translation",), tuple(agent_search_text(agent) for agent in AGENTS)] + assert fallback.calls == [ + ("language translation",), + ("language translation", *(agent_search_text(agent) for agent in AGENTS)), + ] @pytest.mark.asyncio async def test_mixed_dimensions_in_one_batch_become_embedding_failed(self) -> None: From 3002994c0e7f1b14c8be1cc2955fcb32929bdef4 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 27 Aug 2026 20:11:05 -0700 Subject: [PATCH 025/105] feat(ui): the model and wire layer for operator-defined auto-router tier sets (#38602) * feat(ui): the model and wire layer for operator-defined auto-router tier sets The data half of the custom tier set editor, with no visible UI change: the editor lands separately on top of it. One reader, activeTierRows, mints built-in rows with the canonical tier key as their id, so the fallback pointer, the plan-mode floor and the per-model params are row ids in both modes and nothing downstream branches on the mode. One restrictions table carries each forbidden setting beside the reason shown for it, so the greyed control and the omitted payload key cannot disagree. The tier-set writes live in applyTierSetAction, where the fallback re-point and the floor turn-off happen in one commit, unit-tested without a render. buildComplexityRouterConfig emits tiers, tier_definitions and fallback_tier from the rows, forces the LLM classifier, and strips what the backend rejects beside tier_definitions. A payload built without a custom tier set is byte-identical to what the form sends today. * fix(ui): resolve frontend-lint failures on the tier-set model layer * test(ui): drop a redundant explanatory comment per repo convention * fix(ui): keyword rules follow their tier row through every tier-set action --- .../add_model/ComplexityRouterConfig.tsx | 200 ++++++++-------- .../build_complexity_router_config.test.ts | 206 ++++++++++++++++- .../build_complexity_router_config.ts | 214 +++++++++++++++--- .../add_model/complexity_router_tiers.test.ts | 20 ++ .../add_model/complexity_router_tiers.ts | 27 ++- .../components/add_model/tier_rows.test.ts | 120 +++++++++- .../src/components/add_model/tier_rows.ts | 148 +++++++++++- .../add_model/tier_set_actions.test.ts | 171 ++++++++++++++ .../components/add_model/tier_set_actions.ts | 150 ++++++++++++ 9 files changed, 1100 insertions(+), 156 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/tier_set_actions.test.ts create mode 100644 ui/litellm-dashboard/src/components/add_model/tier_set_actions.ts diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index de6c9cd72fb..9894e26d552 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -24,7 +24,8 @@ import EscalationKeywords from "./EscalationKeywords"; import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules"; import SemanticKeywordMatching from "./SemanticKeywordMatching"; import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs"; -import { type TierRow, activeTierRows, resolveComplexityDefaultModel } from "./tier_rows"; +import { type CustomTierSet, type TierRow, activeTierRows, resolveComplexityDefaultModel } from "./tier_rows"; +export type { CustomTierSet, TierRow } from "./tier_rows"; export type { DimensionWeights, TierBoundaries, TokenThresholds }; @@ -133,7 +134,12 @@ export const heuristicScoringRoleFor = ( }; export const heuristicScoringRole = (value: ComplexityRouterConfigValue): HeuristicScoringRole => - heuristicScoringRoleFor(value.classifier_type, value.classifier_fallback); + value.custom_tier_set ? "never" : heuristicScoringRoleFor(value.classifier_type, value.classifier_fallback); + +// Derived, never written into the value, so undoing a tier edit reverts the form with nothing left behind. +export const effectiveClassifierType = ( + value: Pick, +): ClassifierType => (value.custom_tier_set ? "llm" : value.classifier_type); export type AdaptiveEligible = "all" | "classified_tier"; @@ -179,6 +185,7 @@ export interface ComplexityRouterConfigValue { * params object is held, not just reasoning_effort, so keys authored in config.yaml survive an * edit round-trip. */ + custom_tier_set?: CustomTierSet; tier_model_params?: TierModelParamsByTier; } @@ -242,6 +249,104 @@ export const DEFAULT_HEURISTIC_FIRST_MAX_TIER = "SIMPLE"; */ export const HEURISTIC_FIRST_MAX_TIER_KEYS = TIER_KEYS.slice(0, -1); +const AffinityControls: React.FC<{ + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; +}> = ({ value, onChange }) => ( + <> +
+ onChange({ ...value, deployment_affinity: deploymentAffinity })} + aria-label="Pin a session to one deployment per model group" + /> + Pin a session to one deployment per model group +
+ + Keeps a session on the same deployment within a group, so provider prompt caches stay warm. Turn off to + load-balance every turn. + +
+ onChange({ ...value, session_affinity: sessionAffinity })} + aria-label="Pin a session to its first model" + /> + Pin a session to its first model +
+ + Keeps a session on its first turn's model instead of re-classifying each turn. Also pins the deployment. + + +); + +const PlanModeOverrideControls: React.FC<{ + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; + planModeTierOptions: { value: string; label: string }[]; +}> = ({ value, onChange, planModeTierOptions }) => ( + <> +
+ + onChange({ + ...value, + plan_mode_min_tier: enabled ? planModeTierOptions.at(-1)?.value : undefined, + }) + } + aria-label="Route plan-mode requests to a minimum tier" + /> + Route plan-mode requests to a minimum tier +
+ + Requests from coding agents in plan mode (Claude Code, GitHub Copilot) route to at least this tier. The classifier + still wins when it picks higher, and the override only lasts while plan mode is active. + {planModeTierOptions.length === 0 && " Add models to a tier to enable this."} + + {value.plan_mode_min_tier !== undefined && ( +
+ +
+ )} + +); + +const ResponseFormatControls: React.FC<{ + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; +}> = ({ value, onChange }) => ( + <> +
+ onChange({ ...value, return_raw_model_name: returnRawModelName })} + aria-label="Return raw model name" + /> + Return raw model name +
+ + Return the resolved underlying model name in responses instead of the autorouter alias. + + +); + const ComplexityRouterConfig: React.FC = ({ modelInfo, value, @@ -459,104 +564,19 @@ const ComplexityRouterConfig: React.FC = ({ { key: "affinity", label: Advanced: Affinity, - children: ( - <> -
- - onChange({ ...value, deployment_affinity: deploymentAffinity }) - } - aria-label="Pin a session to one deployment per model group" - /> - Pin a session to one deployment per model group -
- - Keeps a session on the same deployment within a group, so provider prompt caches stay warm. Turn off - to load-balance every turn. - -
- onChange({ ...value, session_affinity: sessionAffinity })} - aria-label="Pin a session to its first model" - /> - Pin a session to its first model -
- - Keeps a session on its first turn's model instead of re-classifying each turn. Also pins the - deployment. - - - ), + children: , }, { key: "plan-mode", label: Advanced: Plan-Mode Override, children: ( - <> -
- - onChange({ - ...value, - plan_mode_min_tier: enabled ? planModeTierOptions.at(-1)?.value : undefined, - }) - } - aria-label="Route plan-mode requests to a minimum tier" - /> - Route plan-mode requests to a minimum tier -
- - Requests from coding agents in plan mode (Claude Code, GitHub Copilot) route to at least this tier. - The classifier still wins when it picks higher, and the override only lasts while plan mode is active. - {planModeTierOptions.length === 0 && " Add models to a tier to enable this."} - - {value.plan_mode_min_tier !== undefined && ( -
- -
- )} - + ), }, { key: "response", label: Advanced: Response Format, - children: ( - <> -
- - onChange({ ...value, return_raw_model_name: returnRawModelName }) - } - aria-label="Return raw model name" - /> - Return raw model name -
- - Return the resolved underlying model name in responses instead of the autorouter alias. - - - ), + children: , }, ...(onEscalationKeywordsChange ? [ diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 325122f755a..ea9ee1bb305 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -5,13 +5,14 @@ import { getKeywordTierRulesError, getClassifierModelError, getMissingTiersError, + hydrateCustomTierSet, getSemanticConfigError, getTierLabelsError, hydrateTierLabels, BuildComplexityRouterConfigParams, dryRunRejection, } from "./build_complexity_router_config"; -import { activeTierRows } from "./tier_rows"; +import { CUSTOM_TIER_RESTRICTIONS, activeTierRows } from "./tier_rows"; const tiers = { SIMPLE: ["gpt-4o-mini"], @@ -47,13 +48,14 @@ const baseParams: BuildComplexityRouterConfigParams = { describe("buildComplexityRouterConfig", () => { it("emits tiers, classifier_type, and escalation_keywords when nothing else is configured", () => { const config = buildComplexityRouterConfig(baseParams); - expect(config).toEqual({ + const expected = { tiers, classifier_type: "heuristic", session_affinity: false, deployment_affinity: true, escalation_keywords: ["LITELLM ESCALATE"], - }); + }; + expect(config).toEqual(expected); }); it("trims escalation keywords and drops blank entries", () => { @@ -215,13 +217,14 @@ describe("buildComplexityRouterConfig", () => { }); it("omits adaptive fields when adaptive is disabled even if weights linger in state", () => { - const config = buildComplexityRouterConfig({ + const params = { ...baseParams, adaptive: false, adaptiveWeights: { quality: 0.9, cost: 0.1 }, tierDistancePenalty: 2, - adaptiveEligible: "classified_tier", - }); + adaptiveEligible: "classified_tier" as const, + }; + const config = buildComplexityRouterConfig(params); expect(config.adaptive).toBeUndefined(); expect(config.adaptive_weights).toBeUndefined(); expect(config.tier_distance_penalty).toBeUndefined(); @@ -249,13 +252,14 @@ describe("buildComplexityRouterConfig", () => { }); it("includes tier_distance_penalty when adaptive is enabled with eligible='all'", () => { - const config = buildComplexityRouterConfig({ + const params = { ...baseParams, adaptive: true, adaptiveWeights: { quality: 0.6, cost: 0.4 }, tierDistancePenalty: 0.75, - adaptiveEligible: "all", - }); + adaptiveEligible: "all" as const, + }; + const config = buildComplexityRouterConfig(params); expect(config.adaptive).toBe(true); expect(config.adaptive_weights).toEqual({ quality: 0.6, cost: 0.4 }); expect(config.tier_distance_penalty).toBe(0.75); @@ -263,13 +267,14 @@ describe("buildComplexityRouterConfig", () => { }); it("omits tier_distance_penalty when eligible='classified_tier', since the penalty doesn't apply there", () => { - const config = buildComplexityRouterConfig({ + const params = { ...baseParams, adaptive: true, adaptiveWeights: { quality: 0.6, cost: 0.4 }, tierDistancePenalty: 0.75, - adaptiveEligible: "classified_tier", - }); + adaptiveEligible: "classified_tier" as const, + }; + const config = buildComplexityRouterConfig(params); expect(config.adaptive).toBe(true); expect(config.adaptive_eligible).toBe("classified_tier"); expect(config.tier_distance_penalty).toBeUndefined(); @@ -695,6 +700,19 @@ describe("getClassifierModelError", () => { ); }); + it("asks for a model under an edited tier set even while the stored type still reads heuristic", () => { + const customSet = { + tiers: [ + { id: "a", name: "CASUAL", definition: "d", models: ["m"] }, + { id: "b", name: "AUDIT", definition: "d", models: ["m"] }, + ], + fallback_tier_id: "a", + }; + expect(getClassifierModelError({ classifier_type: "heuristic", custom_tier_set: customSet })).toContain( + "an edited tier set routes with the LLM classifier", + ); + }); + it("stays quiet once a model is chosen", () => { expect( getClassifierModelError({ classifier_type: "llm", classifier_llm_config: { model: "m", timeout_ms: 3000 } }), @@ -761,6 +779,170 @@ describe("heuristic_first", () => { }); }); +describe("buildComplexityRouterConfig with an edited tier set", () => { + const customTierSet = { + tiers: [ + { id: "CASUAL", name: "CASUAL", definition: "small talk", models: ["gpt-4o-mini"] }, + { id: "sec", name: " SECURITY_REVIEW ", definition: " audits and vulnerability review ", models: ["o1-preview"] }, + ], + fallback_tier_id: "CASUAL", + }; + const build = (overrides: Partial = {}) => { + const params: BuildComplexityRouterConfigParams = { + ...baseParams, + customTierSet, + classifierType: "llm", + classifierLlmConfig: { model: "gpt-4o-mini", timeout_ms: 3000 }, + ...overrides, + }; + return buildComplexityRouterConfig(params); + }; + + it("writes tiers, tier_definitions and fallback_tier from the rows, trimmed to what the backend matches", () => { + const payload = build(); + expect(payload.tiers).toEqual({ CASUAL: ["gpt-4o-mini"], SECURITY_REVIEW: ["o1-preview"] }); + expect(payload.tier_definitions).toEqual([ + { name: "CASUAL", description: "small talk" }, + { name: "SECURITY_REVIEW", description: "audits and vulnerability review" }, + ]); + expect(payload.fallback_tier).toBe("CASUAL"); + }); + + it("forces the LLM classifier even when the form still holds heuristic, which the backend rejects", () => { + expect(build({ classifierType: "heuristic" }).classifier_type).toBe("llm"); + }); + + it("turns session pinning off rather than leaving a stale true the backend rejects", () => { + expect(build({ sessionAffinity: true }).session_affinity).toBe(false); + }); + + it("omits a definition on a built-in name, letting the backend rubric supply it", () => { + const payload = build({ + customTierSet: { + tiers: [{ id: "SIMPLE", name: "SIMPLE", definition: "", models: ["gpt-4o-mini"] }, customTierSet.tiers[1]], + fallback_tier_id: "SIMPLE", + }, + }); + expect(payload.tier_definitions?.[0]).toEqual({ name: "SIMPLE" }); + }); + + it("drops the replacement prompt and the rubric preset, which live inside classifier_llm_config", () => { + const payload = build({ + classifierLlmConfig: { + model: "gpt-4o-mini", + timeout_ms: 3000, + system_prompt: "replace the whole rubric", + classification_rubric: "agentic", + }, + }); + expect(payload.classifier_llm_config).toEqual({ model: "gpt-4o-mini", timeout_ms: 3000 }); + }); + + it.each( + Object.entries(CUSTOM_TIER_RESTRICTIONS).flatMap(([name, restriction]) => + restriction.omit.map((key) => [name, key] as const), + ), + )("drops %s's %s key, which an edited tier set never uses", (_name, key) => { + const loaded: Partial = { + tierLabels: { SIMPLE: "Cheap" }, + escalationKeywords: ["LITELLM ESCALATE"], + adaptive: true, + classifierFallback: "heuristic", + tierBoundaries: { simple_medium: 0.1, medium_complex: 0.25, complex_reasoning: 0.5 }, + tokenThresholds: { short: 1, long: 2 }, + dimensionWeights: { length: 1 }, + reasoningOverrideMinScore: 0.5, + heuristicFirstMaxTier: "SIMPLE", + customTechnicalKeywords: ["kubernetes"], + }; + const emittingType = key === "heuristic_first_max_tier" ? "heuristic_first" : "llm"; + expect(buildComplexityRouterConfig({ ...baseParams, ...loaded, classifierType: emittingType })).toHaveProperty(key); + expect(build(loaded)).not.toHaveProperty(key); + }); + + it("drops the local-scorer threshold left behind by a heuristic_first router, which the backend rejects here", () => { + const payload = build({ classifierType: "heuristic_first", heuristicFirstMaxTier: "SIMPLE" }); + expect(payload).not.toHaveProperty("heuristic_first_max_tier"); + expect(payload.classifier_type).toBe("llm"); + }); + + it("keeps the classifier context knobs the operator set, which the forced LLM classifier reads", () => { + const payload = build({ classifierType: "heuristic", classifierContextWindowSize: 5 }); + expect(payload.classifier_context_window_size).toBe(5); + expect(payload.classifier_llm_config).toEqual({ model: "gpt-4o-mini", timeout_ms: 3000 }); + }); + + it("carries the plan-mode floor as the row's name, not the row id the form holds", () => { + expect(build({ planModeMinTier: "sec" }).plan_mode_min_tier).toBe("SECURITY_REVIEW"); + }); + + it("leaves the floor off when its row is gone, the same rule hydration and the editor apply", () => { + expect(build({ planModeMinTier: "removed-row" })).not.toHaveProperty("plan_mode_min_tier"); + }); + + it("keys per-model params by tier name on the wire while the editor keys them by row id", () => { + const payload = build({ tierModelParams: { sec: { "o1-preview": { reasoning_effort: "high" } } } }); + expect(payload.tier_model_configs).toEqual({ + SECURITY_REVIEW: [{ model_name: "o1-preview", litellm_params: { reasoning_effort: "high" } }], + }); + }); + + it("leaves behind no params for a row that is no longer in the set", () => { + const payload = build({ tierModelParams: { removed: { "o1-preview": { reasoning_effort: "high" } } } }); + expect(payload).not.toHaveProperty("tier_model_configs"); + }); +}); + +describe("hydrateCustomTierSet", () => { + it("returns nothing when the stored config has no tier_definitions, keeping built-in routers built-in", () => { + expect(hydrateCustomTierSet({ tiers: { SIMPLE: ["a"] } })).toBeUndefined(); + }); + + it("round-trips a payload this form built, so an edit save cannot lose a key", () => { + const roundTripParams: BuildComplexityRouterConfigParams = { + ...baseParams, + classifierType: "llm", + classifierLlmConfig: { model: "gpt-4o-mini", timeout_ms: 3000 }, + customTierSet: { + tiers: [ + { id: "CASUAL", name: "CASUAL", definition: "small talk", models: ["gpt-4o-mini"] }, + { id: "sec", name: "SECURITY_REVIEW", definition: "audits", models: ["o1-preview"] }, + ], + fallback_tier_id: "sec", + }, + }; + const payload = buildComplexityRouterConfig(roundTripParams); + const hydrated = hydrateCustomTierSet(payload); + expect(hydrated?.tiers).toEqual([ + { id: "stored-0", name: "CASUAL", definition: "small talk", models: ["gpt-4o-mini"] }, + { id: "stored-1", name: "SECURITY_REVIEW", definition: "audits", models: ["o1-preview"] }, + ]); + expect(hydrated?.tiers.find((row) => row.id === hydrated.fallback_tier_id)?.name).toBe("SECURITY_REVIEW"); + }); + + it("mints the canonical key for a built-in name so every pointer into the set is a row id", () => { + const hydrated = hydrateCustomTierSet({ + tier_definitions: [{ name: "SIMPLE" }, { name: "AUDIT", description: "audits" }], + tiers: { SIMPLE: ["a"], AUDIT: ["b"] }, + fallback_tier: "AUDIT", + }); + expect(hydrated?.tiers.map((row) => row.id)).toEqual(["SIMPLE", "stored-1"]); + }); + + it("matches a hand-written config's tier pool case-insensitively rather than losing its models", () => { + const hydrated = hydrateCustomTierSet({ + tier_definitions: [ + { name: "audit", description: "x" }, + { name: "CASUAL", description: "y" }, + ], + tiers: { Audit: ["m1"], CASUAL: ["m2"] }, + fallback_tier: " audit ", + }); + expect(hydrated?.tiers[0].models).toEqual(["m1"]); + expect(hydrated?.fallback_tier_id).toBe(hydrated?.tiers[0].id); + }); +}); + describe("dryRunRejection", () => { it("blocks the save on a rejection whose message is missing, which the write would return as a raw 400", () => { expect(dryRunRejection({ valid: false })).toBe("The proxy rejected this auto-router configuration"); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index eb014092ec2..9ff256323d6 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -1,7 +1,21 @@ import { KeywordTierRule } from "./KeywordTierRules"; -import { type TierRow, activeTierName, tierRowById } from "./tier_rows"; +import { + type CustomTierSet, + type TierRow, + CUSTOM_TIER_OMITTED_KEYS, + activeTierName, + sameTierIdentity, + tierDefinitionsFromRows, + tierRowById, + tierRowByName, +} from "./tier_rows"; import { emptyKeywordTierRuleIndexes, serializeKeywordTierRules } from "./complexity_router_keywords"; -import { TierModelParams, TierModelParamsByTier, serializeTierModelConfigs } from "./complexity_router_tiers"; +import { + TierModelParams, + TierModelParamsByTier, + normalizeTierModels, + serializeTierModelConfigs, +} from "./complexity_router_tiers"; import { AdaptiveEligible, AdaptiveRouterWeights, @@ -13,6 +27,7 @@ import { ComplexityTiers, DimensionWeights, TIER_KEYS, + effectiveClassifierType, TIER_DESCRIPTIONS, TierBoundaries, TokenThresholds, @@ -78,6 +93,7 @@ const scorerKnobPayload = ({ export interface BuildComplexityRouterConfigParams { tiers: ComplexityTiers; + customTierSet?: CustomTierSet; defaultModel: string | undefined; planModeMinTier: string | undefined; tierLabels: ComplexityTierLabels | undefined; @@ -108,11 +124,26 @@ export interface BuildComplexityRouterConfigParams { tierModelParams?: TierModelParamsByTier; } +/** + * The message to surface when the dry-run rejects a save, or null to let it through. + * + * Gated on `valid` alone. The verdict's `valid` is derived from `error` server side today, but the + * two arrive as independent fields, so reading `error` as the gate would let a rejection whose + * message is missing or blank through to the write and back as a raw 400. A transport failure fails + * open as `{valid: true}`, which this passes, leaving the write gate authoritative. + */ export const dryRunRejection = (verdict: { valid: boolean; error?: string | null }): string | null => verdict.valid ? null : verdict.error?.trim() || "The proxy rejected this auto-router configuration"; +export interface TierDefinitionPayload { + name: string; + description?: string; +} + export interface ComplexityRouterConfigPayload { - tiers: ComplexityTiers; + tiers: ComplexityTiers | Record; + tier_definitions?: TierDefinitionPayload[]; + fallback_tier?: string; default_model?: string; plan_mode_min_tier?: string; tier_labels?: ComplexityTierLabels; @@ -180,7 +211,7 @@ export const getTierLabelsError = (tierLabels: ComplexityTierLabels | undefined) // Requires every active tier non-empty, so the create form can never reach the // resolveComplexityDefaultModel === undefined case. The edit modal allows a partially filled -// set, which is why it keeps its own !defaultModel guard after deriving. +// built-in set, which is why it keeps its own !defaultModel guard after deriving. export const getMissingTiersError = (rows: readonly TierRow[]): string | null => { const missing = rows.filter((row) => row.models.length === 0).map(activeTierName); if (missing.length === 0) return null; @@ -194,8 +225,9 @@ export const getPlanModeTierError = (planModeMinTier: string | undefined, rows: return `The plan-mode minimum tier (${floor ? activeTierName(floor) : planModeMinTier}) has no models. Add one or turn the override off.`; }; -// The tier is a free string since #37413, and _validate_keyword_rule_tiers matches it EXACTLY, so a -// rule naming a tier this router does not have is a raw 400 unless the gate catches it first. +// The orphan check compares exactly, not casefold: _validate_keyword_rule_tiers is exact +// membership, so a rule left pointing at a differently cased name would clear a gate the +// backend then rejects. export const getKeywordTierRulesError = ( keywordTierRules: KeywordTierRule[], rows: readonly TierRow[], @@ -209,14 +241,16 @@ export const getKeywordTierRulesError = ( return `Keyword rule(s) ${orphaned.join(", ")} route to a tier this router no longer has`; }; -// The submit gate and the submit handler both read this, so a disabled button and a refused submit -// cannot disagree about why. +// An edited tier set forces the LLM classifier, so the model requirement follows the EFFECTIVE type. +// Both forms' submit gates and their submit handlers read this one answer so they cannot drift. export const getClassifierModelError = ( - config: Pick, -): string | null => - usesLlmClassifier(config.classifier_type) && !config.classifier_llm_config?.model - ? "Please select a classifier model, or switch back to Heuristic" - : null; + config: Pick, +): string | null => { + if (!usesLlmClassifier(effectiveClassifierType(config)) || config.classifier_llm_config?.model) return null; + return config.custom_tier_set + ? "Please select a classifier model: an edited tier set routes with the LLM classifier" + : "Please select a classifier model, or switch back to Heuristic"; +}; export const getSemanticConfigError = ({ semanticMatchingEnabled, @@ -231,8 +265,115 @@ export const getSemanticConfigError = ({ return null; }; +export const customTierWireFields = ( + customTierSet: CustomTierSet, + classifierLlmConfig: ClassifierLLMConfig | undefined, + planModeMinTierId: string | undefined, +): Partial => { + const rows = customTierSet.tiers; + const fallback = tierRowById(rows, customTierSet.fallback_tier_id); + const floor = tierRowById(rows, planModeMinTierId); + return { + tiers: Object.fromEntries(rows.map((row) => [activeTierName(row), row.models])), + tier_definitions: tierDefinitionsFromRows(rows), + ...(fallback && { fallback_tier: activeTierName(fallback) }), + classifier_type: "llm", + // Rebuilt from the two fields an edited tier set allows. The backend rejects system_prompt and + // classification_rubric beside tier_definitions, and both live inside this object rather than at + // the top level the omit list covers. + ...(classifierLlmConfig && { + classifier_llm_config: { model: classifierLlmConfig.model, timeout_ms: classifierLlmConfig.timeout_ms }, + }), + session_affinity: false, + ...(floor && { plan_mode_min_tier: activeTierName(floor) }), + }; +}; + +// plan_mode_min_tier rides the strip list because the base payload carries it as a row id; +// customTierWireFields re-emits it as the row's name, and an unresolvable floor stays off. +const CUSTOM_TIER_STRIPPED_KEYS: readonly string[] = [...CUSTOM_TIER_OMITTED_KEYS, "plan_mode_min_tier"]; + +export const hydrateCustomTierSet = (parsedConfig: { + tier_definitions?: unknown; + fallback_tier?: unknown; + tiers?: unknown; +}): CustomTierSet | undefined => { + if (!Array.isArray(parsedConfig.tier_definitions) || parsedConfig.tier_definitions.length === 0) return undefined; + const storedTiers = + typeof parsedConfig.tiers === "object" && parsedConfig.tiers !== null && !Array.isArray(parsedConfig.tiers) + ? Object.entries(parsedConfig.tiers as Record) + : []; + const rows = parsedConfig.tier_definitions.flatMap((entry, index): TierRow[] => { + if (typeof entry !== "object" || entry === null) return []; + const { name, description } = entry as { name?: unknown; description?: unknown }; + if (typeof name !== "string" || !name.trim()) return []; + return [ + { + id: TIER_KEYS.find((tier) => sameTierIdentity(tier, name)) ?? `stored-${index}`, + name: name.trim(), + definition: typeof description === "string" ? description.trim() : "", + models: normalizeTierModels(storedTiers.find(([tier]) => sameTierIdentity(tier, name))?.[1]), + }, + ]; + }); + if (rows.length === 0) return undefined; + const storedFallback = typeof parsedConfig.fallback_tier === "string" ? parsedConfig.fallback_tier : ""; + return { tiers: rows, fallback_tier_id: tierRowByName(rows, storedFallback)?.id ?? "" }; +}; + +// Ids are session-ephemeral, so a stored floor hydrates by name; unresolvable means off, the same +// rule the editor and the wire apply. +export const hydratePlanModeMinTier = ( + stored: unknown, + customTierSet: CustomTierSet | undefined, +): string | undefined => { + if (typeof stored !== "string" || !stored.trim()) return undefined; + if (!customTierSet) return stored; + return tierRowByName(customTierSet.tiers, stored)?.id; +}; + +const classifierWireFields = ( + effectiveType: ClassifierType, + { + classifierLlmConfig, + classifierFallback, + heuristicFirstMaxTier, + classifierContextWindowSize, + classifierContextBudgetChars, + classifierContextIncludeAssistantTurns, + }: Pick< + BuildComplexityRouterConfigParams, + | "classifierLlmConfig" + | "classifierFallback" + | "heuristicFirstMaxTier" + | "classifierContextWindowSize" + | "classifierContextBudgetChars" + | "classifierContextIncludeAssistantTurns" + >, +): Partial => ({ + ...(usesLlmClassifier(effectiveType) && + classifierLlmConfig && { classifier_llm_config: normalizeClassifierLlmConfig(classifierLlmConfig) }), + ...(usesLlmClassifier(effectiveType) && + classifierFallback !== undefined && { classifier_fallback: classifierFallback }), + ...(effectiveType === "heuristic_first" && + heuristicFirstMaxTier?.trim() && { heuristic_first_max_tier: heuristicFirstMaxTier }), + ...(usesLlmClassifier(effectiveType) && + classifierContextWindowSize !== undefined && { + classifier_context_window_size: classifierContextWindowSize, + }), + ...(usesLlmClassifier(effectiveType) && + classifierContextBudgetChars !== undefined && { + classifier_context_budget_chars: classifierContextBudgetChars, + }), + ...(usesLlmClassifier(effectiveType) && + classifierContextIncludeAssistantTurns !== undefined && { + classifier_context_include_assistant_turns: classifierContextIncludeAssistantTurns, + }), +}); + export const buildComplexityRouterConfig = ({ tiers, + customTierSet, defaultModel, planModeMinTier, tierLabels, @@ -262,7 +403,12 @@ export const buildComplexityRouterConfig = ({ reasoningOverrideMinScore, tierModelParams, }: BuildComplexityRouterConfigParams): ComplexityRouterConfigPayload => { - const serializedTierModelConfigs = serializeTierModelConfigs(tiers, tierModelParams); + const serializedTierModelConfigs = customTierSet + ? serializeTierModelConfigs( + Object.fromEntries(customTierSet.tiers.map((row) => [activeTierName(row), row.models])), + Object.fromEntries(customTierSet.tiers.map((row) => [activeTierName(row), tierModelParams?.[row.id] ?? {}])), + ) + : serializeTierModelConfigs(tiers, tierModelParams); const cleanedEscalationKeywords = escalationKeywords.map((keyword) => keyword.trim()).filter(Boolean); const cleanedKeywordTierRules = serializeKeywordTierRules(keywordTierRules); const cleanedTierLabels = serializeTierLabels(tierLabels); @@ -275,32 +421,26 @@ export const buildComplexityRouterConfig = ({ reasoningOverrideMinScore, }; const scorerKnobs = scorerKnobPayload(scorerInputs); + const classifierInputs = { + classifierLlmConfig, + classifierFallback, + heuristicFirstMaxTier, + classifierContextWindowSize, + classifierContextBudgetChars, + classifierContextIncludeAssistantTurns, + }; + // An edited tier set forces the LLM classifier, so llm-only inputs must survive a classifier_type + // the form never rewrote. The UI gates the same controls on this, not on the raw value. + const effectiveType: ClassifierType = customTierSet ? "llm" : classifierType; - return { + const payload: ComplexityRouterConfigPayload = { tiers, ...(serializedTierModelConfigs && { tier_model_configs: serializedTierModelConfigs }), ...(defaultModel?.trim() && { default_model: defaultModel }), ...(planModeMinTier?.trim() && { plan_mode_min_tier: planModeMinTier }), ...(cleanedTierLabels && { tier_labels: cleanedTierLabels }), classifier_type: classifierType, - ...(usesLlmClassifier(classifierType) && - classifierLlmConfig && { classifier_llm_config: normalizeClassifierLlmConfig(classifierLlmConfig) }), - ...(usesLlmClassifier(classifierType) && - classifierFallback !== undefined && { classifier_fallback: classifierFallback }), - ...(classifierType === "heuristic_first" && - heuristicFirstMaxTier?.trim() && { heuristic_first_max_tier: heuristicFirstMaxTier }), - ...(usesLlmClassifier(classifierType) && - classifierContextWindowSize !== undefined && { - classifier_context_window_size: classifierContextWindowSize, - }), - ...(usesLlmClassifier(classifierType) && - classifierContextBudgetChars !== undefined && { - classifier_context_budget_chars: classifierContextBudgetChars, - }), - ...(usesLlmClassifier(classifierType) && - classifierContextIncludeAssistantTurns !== undefined && { - classifier_context_include_assistant_turns: classifierContextIncludeAssistantTurns, - }), + ...classifierWireFields(effectiveType, classifierInputs), session_affinity: sessionAffinity, deployment_affinity: deploymentAffinity, ...(customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords }), @@ -320,4 +460,12 @@ export const buildComplexityRouterConfig = ({ ...(returnRawModelName && { return_raw_model_name: true }), ...scorerKnobs, }; + if (!customTierSet) return payload; + const kept = Object.fromEntries( + Object.entries(payload).filter(([key]) => !CUSTOM_TIER_STRIPPED_KEYS.includes(key)), + ) as ComplexityRouterConfigPayload; + return { + ...kept, + ...customTierWireFields(customTierSet, classifierLlmConfig, planModeMinTier), + }; }; diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts index be48ff4958d..b0fab49e80a 100644 --- a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts @@ -5,6 +5,7 @@ import { normalizeTierModels, pruneTierModelParams, serializeTierModelConfigs, + tierRowLabel, setTierModelReasoningEffort, } from "./complexity_router_tiers"; import { resolveComplexityDefaultModel } from "./tier_rows"; @@ -234,3 +235,22 @@ describe("pruneTierModelParams", () => { expect(pruneTierModelParams(current, "MEDIUM", [])).toBe(current); }); }); + +describe("tierRowLabel", () => { + it("shows a built-in row's display label while it is untouched", () => { + expect(tierRowLabel({ id: "COMPLEX", name: "COMPLEX" })).toBe("Complex"); + expect(tierRowLabel({ id: "COMPLEX", name: "COMPLEX" }, { COMPLEX: "Deep" })).toBe("Deep"); + }); + + it("shows the operator's name once a built-in row is renamed, since the id stays canonical", () => { + expect(tierRowLabel({ id: "COMPLEX", name: "SECURITY_REVIEW" })).toBe("SECURITY_REVIEW"); + }); + + it("shows a custom row's name", () => { + expect(tierRowLabel({ id: "stored-1", name: "AUDIT" })).toBe("AUDIT"); + }); + + it("calls an unnamed new row New rather than rendering an empty label", () => { + expect(tierRowLabel({ id: "uuid", name: " " })).toBe("New"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts index ea0d34f6581..8cdb730cfa6 100644 --- a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts @@ -133,7 +133,30 @@ export const DEFAULT_TIER_LABELS: Record = { REASONING: "Reasoning", }; +const isBuiltInTier = (tier: string): tier is ComplexityTier => (TIER_ORDER as string[]).includes(tier); + +const builtInTierLabel = ( + tierLabels: Partial> | undefined, + tier: ComplexityTier, +): string => tierLabels?.[tier]?.trim() || DEFAULT_TIER_LABELS[tier]; + +// What a tier row is called on screen. A row the operator named shows that name; an untouched +// built-in row shows its display label. The one owner for every surface that renders a tier. +export const tierRowLabel = ( + row: { id: string; name: string }, + tierLabels?: Partial>, +): string => { + const builtIn = TIER_ORDER.find((tier) => tier === row.id); + const named = row.name.trim(); + if (!builtIn || named !== builtIn) return named || "New"; + return builtInTierLabel(tierLabels, builtIn); +}; + export const tierOptions = ( tierLabels: Partial> | undefined, -): { value: ComplexityTier; label: string }[] => - TIER_ORDER.map((tier) => ({ value: tier, label: tierLabels?.[tier]?.trim() || DEFAULT_TIER_LABELS[tier] })); + tierNames?: readonly string[], +): { value: string; label: string }[] => + (tierNames ?? TIER_ORDER).map((tier) => ({ + value: tier, + label: isBuiltInTier(tier) ? builtInTierLabel(tierLabels, tier) : tier, + })); diff --git a/ui/litellm-dashboard/src/components/add_model/tier_rows.test.ts b/ui/litellm-dashboard/src/components/add_model/tier_rows.test.ts index 54d9e4f3f0a..df50f116f60 100644 --- a/ui/litellm-dashboard/src/components/add_model/tier_rows.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/tier_rows.test.ts @@ -1,12 +1,18 @@ import { describe, expect, it } from "vitest"; +import type { ActiveTierRow, CustomTierSet, TierRow } from "./tier_rows"; import { + CUSTOM_TIER_OMITTED_KEYS, + CUSTOM_TIER_RESTRICTIONS, + MAX_TIER_COUNT, activeTierName, activeTierRows, isBuiltInTierName, resolveComplexityDefaultModel, sameTierIdentity, tierRowById, + getCustomTierRowsError, + tierParamsByRowId, tierRowByName, } from "./tier_rows"; @@ -15,19 +21,17 @@ const tiers = { SIMPLE: ["a"], MEDIUM: ["b"], COMPLEX: ["c"], REASONING: ["d"] } describe("activeTierRows", () => { it("reads the tier set as rows whose id is the canonical tier key, in severity order", () => { expect(activeTierRows({ tiers })).toEqual([ - { id: "SIMPLE", name: "SIMPLE", models: ["a"] }, - { id: "MEDIUM", name: "MEDIUM", models: ["b"] }, - { id: "COMPLEX", name: "COMPLEX", models: ["c"] }, - { id: "REASONING", name: "REASONING", models: ["d"] }, + { id: "SIMPLE", name: "SIMPLE", definition: "", models: ["a"], params: {} }, + { id: "MEDIUM", name: "MEDIUM", definition: "", models: ["b"], params: {} }, + { id: "COMPLEX", name: "COMPLEX", definition: "", models: ["c"], params: {} }, + { id: "REASONING", name: "REASONING", definition: "", models: ["d"], params: {} }, ]); }); it("gives a tier with no models an empty pool rather than dropping the row", () => { - expect(activeTierRows({ tiers: { ...tiers, COMPLEX: [] } })[2]).toEqual({ - id: "COMPLEX", - name: "COMPLEX", - models: [], - }); + const withEmptyComplex = { tiers: { ...tiers, COMPLEX: [] } }; + const emptyComplexRow: ActiveTierRow = { id: "COMPLEX", name: "COMPLEX", definition: "", models: [], params: {} }; + expect(activeTierRows(withEmptyComplex)[2]).toEqual(emptyComplexRow); }); it("finds a row by id and by name", () => { @@ -53,7 +57,8 @@ describe("sameTierIdentity", () => { }); it("trims a row name, since the backend matches fallback_tier and keyword rules exactly", () => { - expect(activeTierName({ id: "1", name: " AUDIT ", models: [] })).toBe("AUDIT"); + const padded: TierRow = { id: "1", name: " AUDIT ", definition: "", models: [] }; + expect(activeTierName(padded)).toBe("AUDIT"); }); }); @@ -68,3 +73,98 @@ describe("resolveComplexityDefaultModel", () => { expect(resolveComplexityDefaultModel({ tiers: { ...tiers, SIMPLE: [], MEDIUM: [] } })).toBeUndefined(); }); }); + +const definedRow = (name: string, models: string[] = ["m"], definition = "what belongs here"): TierRow => ({ + id: name.toLowerCase(), + name, + definition, + models, +}); + +const set = (rows: TierRow[], fallback?: string): CustomTierSet => ({ + tiers: rows, + fallback_tier_id: fallback ?? rows[0]?.id ?? "", +}); + +describe("activeTierRows with an edited set", () => { + it("reads the edited rows instead of the built-in record once a set is present", () => { + const custom = set([definedRow("CASUAL"), definedRow("AUDIT")]); + expect(activeTierRows({ tiers, custom_tier_set: custom }).map((r) => r.name)).toEqual(["CASUAL", "AUDIT"]); + }); + + it("prefers the fallback tier's pool for the default model, mirroring init_complexity_router_deployment", () => { + const custom = set([definedRow("CASUAL", ["casual-model"]), definedRow("AUDIT", ["audit-model"])], "audit"); + expect(resolveComplexityDefaultModel({ tiers, custom_tier_set: custom })).toBe("audit-model"); + }); +}); + +describe("CUSTOM_TIER_RESTRICTIONS", () => { + it("gives every restriction a reason, since each one replaces or explains a control", () => { + const reasons = Object.values(CUSTOM_TIER_RESTRICTIONS).map((restriction) => restriction.reason); + expect(reasons.every((reason) => reason.length > 0)).toBe(true); + expect(new Set(reasons).size).toBe(reasons.length); + }); + + it("collects every omitted key exactly once, so no key is dropped by two owners", () => { + expect(new Set(CUSTOM_TIER_OMITTED_KEYS).size).toBe(CUSTOM_TIER_OMITTED_KEYS.length); + }); + + it("omits the keys the backend rejects beside tier_definitions", () => { + expect(CUSTOM_TIER_OMITTED_KEYS).toEqual( + expect.arrayContaining(["tier_labels", "escalation_keywords", "adaptive", "classifier_fallback"]), + ); + }); +}); + +describe("getCustomTierRowsError", () => { + it("accepts a complete set", () => { + expect(getCustomTierRowsError(set([definedRow("CASUAL"), definedRow("AUDIT")]))).toBeNull(); + }); + + it.each([ + [set([definedRow("CASUAL")]), "A tier set needs 2 to 8 tiers"], + [ + set(Array.from({ length: MAX_TIER_COUNT + 1 }, (_, index) => definedRow(`T${index}`))), + "A tier set needs 2 to 8 tiers", + ], + [set([definedRow(""), definedRow("AUDIT")], "audit"), "Name every tier"], + [set([definedRow("AUDIT"), { ...definedRow("audit"), id: "second" }]), "Tier names must be unique, ignoring case"], + [ + set([definedRow("CASUAL"), { ...definedRow("AUDIT"), definition: " " }]), + "Every custom tier needs a definition: it is the rubric the classifier routes on", + ], + [set([definedRow("CASUAL"), definedRow("AUDIT")], "gone"), "Pick a Fallback Tier for classifier failures"], + ])("reports the row problem the backend would reject", (customTierSet, expected) => { + expect(getCustomTierRowsError(customTierSet)).toBe(expected); + }); + + it("lets a built-in name inherit its definition, which is the one blank the backend allows", () => { + expect(getCustomTierRowsError(set([{ ...definedRow("SIMPLE"), definition: "" }, definedRow("AUDIT")]))).toBeNull(); + }); +}); + +describe("tierParamsByRowId", () => { + const rows = [ + { id: "SIMPLE", name: "SIMPLE", definition: "", models: ["a"] }, + { id: "stored-1", name: "SECURITY_REVIEW", definition: "audits", models: ["b"] }, + ]; + + it("re-keys a stored tier name onto the ephemeral row id the editor reads", () => { + const stored = { SECURITY_REVIEW: { b: { reasoning_effort: "high" } } }; + expect(tierParamsByRowId(stored, rows)).toEqual({ "stored-1": { b: { reasoning_effort: "high" } } }); + }); + + it("leaves a built-in tier untouched, because its row id is already the tier name", () => { + const stored = { SIMPLE: { a: { reasoning_effort: "low" } } }; + expect(tierParamsByRowId(stored, rows)).toEqual(stored); + }); + + it("passes a tier this editor does not render straight through instead of dropping its params", () => { + const stored = { DEEP_RESEARCH: { c: { reasoning_effort: "high" } } }; + expect(tierParamsByRowId(stored, rows)).toEqual(stored); + }); + + it("returns nothing when there are no stored params, keeping the key out of the payload", () => { + expect(tierParamsByRowId(undefined, rows)).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts index c320980916a..df3a26b7a53 100644 --- a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts +++ b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts @@ -1,40 +1,170 @@ import type { ComplexityTiers } from "./ComplexityRouterConfig"; import type { ComplexityTier } from "./KeywordTierRules"; +import type { TierModelParams, TierModelParamsByTier } from "./complexity_router_tiers"; export const TIER_ORDER: ComplexityTier[] = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"]; export interface TierRow { id: string; name: string; + definition: string; models: string[]; } +/** A row plus the per-model params it owns, so the two can never be keyed differently. */ +export type ActiveTierRow = TierRow & { params: Record }; + +export interface CustomTierSet { + tiers: TierRow[]; + fallback_tier_id: string; +} + +// Mirrors TierDefinition in litellm/router_strategy/complexity_router/config.py +export const MIN_TIER_COUNT = 2; +export const MAX_TIER_COUNT = 8; +export const MAX_TIER_NAME_CHARS = 64; +export const MAX_TIER_DEFINITION_CHARS = 500; + export interface ActiveTierSet { tiers: ComplexityTiers; + custom_tier_set?: CustomTierSet; + tier_model_params?: TierModelParamsByTier; } export const activeTierName = (row: TierRow): string => row.name.trim(); +// Casefold, matching the backend's name-uniqueness rule. NOT the comparison the keyword-rule gate +// uses: _validate_keyword_rule_tiers is exact membership, so folding there clears a save the +// backend then rejects. export const sameTierIdentity = (left: string, right: string): boolean => left.trim().toLowerCase() === right.trim().toLowerCase(); export const isBuiltInTierName = (name: string): boolean => TIER_ORDER.some((tier) => sameTierIdentity(tier, name)); -// The only reader of the tier set. A row's id is the canonical tier key, so anything pointing into -// the set (the plan-mode floor, per-model params) points at a row rather than at a position. -export const activeTierRows = (value: ActiveTierSet): TierRow[] => - TIER_ORDER.map((tier) => ({ id: tier, name: tier, models: value.tiers[tier] ?? [] })); +const builtInRow = (tier: keyof ComplexityTiers, tiers: ComplexityTiers): TierRow => ({ + id: tier, + name: tier, + definition: "", + models: tiers[tier] ?? [], +}); -export const tierRowById = (rows: readonly TierRow[], id: string | undefined): TierRow | undefined => +// The only reader of the tier set. Built-in rows carry the canonical tier key as their id, so every +// pointer into the set is a row id in both modes and nothing downstream branches on the mode. +export const activeTierRows = (value: ActiveTierSet): ActiveTierRow[] => { + const rows = value.custom_tier_set?.tiers ?? TIER_ORDER.map((tier) => builtInRow(tier, value.tiers)); + return rows.map((row) => ({ ...row, params: value.tier_model_params?.[row.id] ?? {} })); +}; + +// The wire shape of an edited tier set, shared by the payload builder and the prompt preview so the +// preview cannot ask the proxy to render definitions the save would not send. +export const tierDefinitionsFromRows = (rows: readonly TierRow[]): { name: string; description?: string }[] => + rows.map((row) => ({ + name: activeTierName(row), + ...(row.definition.trim() && { description: row.definition.trim() }), + })); + +export const tierRowById = (rows: readonly T[], id: string | undefined): T | undefined => id === undefined ? undefined : rows.find((row) => row.id === id); -export const tierRowByName = (rows: readonly TierRow[], name: string): TierRow | undefined => +export const tierRowByName = (rows: readonly T[], name: string): T | undefined => rows.find((row) => sameTierIdentity(row.name, name)); -// Mirrors init_complexity_router_deployment (litellm/router.py): a pin wins, then MEDIUM or SIMPLE -// looked up by exact name. +// Mirrors init_complexity_router_deployment (litellm/router.py): a pin wins, then the fallback +// tier's pool, then MEDIUM or SIMPLE looked up by exact name, so a row named `medium` is no match. export const resolveComplexityDefaultModel = (value: ActiveTierSet, pinned?: string): string | undefined => { const rows = activeTierRows(value); const named = (name: string) => rows.find((row) => activeTierName(row) === name)?.models[0]; - return pinned?.trim() || named("MEDIUM") || named("SIMPLE"); + const fallbackPoolModel = tierRowById(rows, value.custom_tier_set?.fallback_tier_id)?.models[0]; + const builtInDefault = named("MEDIUM") || named("SIMPLE"); + return pinned?.trim() || fallbackPoolModel || builtInDefault; +}; + +// Stored params arrive keyed by the wire tier name while the editor keys them by row id, which is +// ephemeral for a custom row. A key matching no row passes through, so a tier this editor does not +// render keeps its params. +export const tierParamsByRowId = ( + params: TierModelParamsByTier | undefined, + rows: readonly TierRow[], +): TierModelParamsByTier | undefined => + params && + Object.fromEntries(Object.entries(params).map(([tier, byModel]) => [tierRowByName(rows, tier)?.id ?? tier, byModel])); + +// Params keyed by the rows that own them, dropping the rows with none so an untouched router keeps +// the key out of its payload. +export const rowParamsByTier = (rows: readonly ActiveTierRow[]): TierModelParamsByTier | undefined => { + const owned = rows.filter((row) => Object.keys(row.params).length > 0); + return owned.length > 0 ? Object.fromEntries(owned.map((row) => [row.id, row.params])) : undefined; +}; + +export interface TierRestriction { + omit: readonly string[]; + reason: string; +} + +// One source for both the disabled control and the wire, so a control cannot grey out while its +// value still ships. All but heuristicScoring are rejected outright by _validate_tier_definitions; +// heuristicScoring the backend accepts, and it is dropped because that scorer never runs here. +export const CUSTOM_TIER_RESTRICTIONS = { + displayNames: { + omit: ["tier_labels"], + reason: "Display names rename the built-in tiers, which your tier set replaces. Name each tier directly", + }, + escalation: { + omit: ["escalation_keywords"], + reason: "Escalation bumps a request along the built-in tier ladder, which your tier set replaces", + }, + adaptive: { + omit: ["adaptive", "adaptive_weights", "tier_distance_penalty", "adaptive_eligible"], + reason: "Adaptive routing scores models along the built-in tier ladder, which your tier set replaces", + }, + sessionAffinity: { + omit: [], + reason: "Session pinning escalates along the built-in tier ladder, which your tier set replaces", + }, + heuristicClassifier: { + omit: ["heuristic_first_max_tier"], + reason: + "The heuristic scorer only produces the built-in tiers, so an edited set needs the LLM classifier. " + + "Heuristic first is out for the same reason: its local scorer decides the cheap traffic", + }, + heuristicScoring: { + omit: [ + "tier_boundaries", + "token_thresholds", + "dimension_weights", + "reasoning_override_min_score", + "custom_technical_keywords", + ], + reason: "The heuristic scorer never runs under an edited tier set, so its inputs have no effect", + }, + classifierPrompt: { + omit: [], + reason: "A replacement prompt drops the tier bullets and the injection guard. Your definitions are the rubric", + }, + classificationRubric: { + omit: [], + reason: "The preset calibration examples are written against the built-in tiers, which your tier set replaces", + }, + classifierFallback: { + omit: ["classifier_fallback"], + reason: "Fallback Tier is where an edited tier set routes when the classifier fails", + }, +} as const satisfies Record; + +export const CUSTOM_TIER_OMITTED_KEYS: readonly string[] = Object.values(CUSTOM_TIER_RESTRICTIONS).flatMap( + (restriction) => restriction.omit, +); + +// Row-shape errors the backend cannot phrase per row. Payload validity is the dry-run's job. +export const getCustomTierRowsError = (customTierSet: CustomTierSet): string | null => { + const rows = customTierSet.tiers; + if (rows.length < MIN_TIER_COUNT || rows.length > MAX_TIER_COUNT) + return `A tier set needs ${MIN_TIER_COUNT} to ${MAX_TIER_COUNT} tiers`; + if (rows.some((row) => !activeTierName(row))) return "Name every tier"; + const folded = rows.map((row) => row.name.trim().toLowerCase()); + if (new Set(folded).size !== folded.length) return "Tier names must be unique, ignoring case"; + if (rows.some((row) => !row.definition.trim() && !isBuiltInTierName(row.name))) + return "Every custom tier needs a definition: it is the rubric the classifier routes on"; + if (!tierRowById(rows, customTierSet.fallback_tier_id)) return "Pick a Fallback Tier for classifier failures"; + return null; }; diff --git a/ui/litellm-dashboard/src/components/add_model/tier_set_actions.test.ts b/ui/litellm-dashboard/src/components/add_model/tier_set_actions.test.ts new file mode 100644 index 00000000000..f939eb95bca --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/tier_set_actions.test.ts @@ -0,0 +1,171 @@ +import { describe, expect, it } from "vitest"; +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { applyTierSetAction, setFallbackTier } from "./tier_set_actions"; + +const tiers = { + SIMPLE: ["gpt-3.5-turbo"], + MEDIUM: ["gpt-3.5-turbo"], + COMPLEX: ["gpt-4"], + REASONING: ["claude-3-opus"], +}; +const builtIn: ComplexityRouterConfigValue = { tiers, classifier_type: "llm" }; +const custom: ComplexityRouterConfigValue = { + ...builtIn, + custom_tier_set: { + tiers: [ + { id: "CASUAL", name: "CASUAL", definition: "small talk", models: ["gpt-3.5-turbo"] }, + { id: "sec", name: "SECURITY_REVIEW", definition: "audits", models: ["gpt-4"] }, + ], + fallback_tier_id: "CASUAL", + }, +}; +const apply = (value: ComplexityRouterConfigValue, action: Parameters[2], rules = []) => + applyTierSetAction(value, rules, action); + +describe("applyTierSetAction", () => { + it("adds a row and moves the form into an edited set, which the built-in record never leaves", () => { + const { value } = apply(builtIn, { kind: "add" }); + expect(value.custom_tier_set?.tiers).toHaveLength(5); + expect(value.tiers).toEqual(tiers); + }); + + it("renames a built-in tier, which is what makes the set custom", () => { + const { value } = apply(builtIn, { kind: "patch", id: "COMPLEX", patch: { name: "SECURITY_REVIEW" } }); + expect(value.custom_tier_set?.tiers.map((row) => row.name)).toEqual([ + "SIMPLE", + "MEDIUM", + "SECURITY_REVIEW", + "REASONING", + ]); + expect(value.tiers).toEqual(tiers); + }); + + it("carries a renamed tier's keyword rules across, so a rule cannot be orphaned by a rename", () => { + const rules = [{ id: "r1", keywords: ["audit"], tier: "COMPLEX" }]; + const next = applyTierSetAction(builtIn, rules, { kind: "patch", id: "COMPLEX", patch: { name: "AUDIT" } }); + expect(next.keywordTierRules).toEqual([{ id: "r1", keywords: ["audit"], tier: "AUDIT" }]); + }); + + it("leaves the rules alone when another row still answers to the old name", () => { + const shared: ComplexityRouterConfigValue = { + ...builtIn, + custom_tier_set: { + tiers: [ + { id: "a", name: "AUDIT", definition: "d", models: ["gpt-4"] }, + { id: "b", name: "audit", definition: "d", models: ["gpt-4"] }, + ], + fallback_tier_id: "a", + }, + }; + const rules = [{ id: "r1", keywords: ["x"], tier: "AUDIT" }]; + const next = applyTierSetAction(shared, rules, { kind: "patch", id: "a", patch: { name: "RENAMED" } }); + expect(next.keywordTierRules).toBe(rules); + }); + + it("prunes the per-model params of a model dropped from a row", () => { + const withParams: ComplexityRouterConfigValue = { + ...custom, + tier_model_params: { sec: { "gpt-4": { reasoning_effort: "high" } } }, + }; + const { value } = apply(withParams, { kind: "models", id: "sec", models: [] }); + expect(value.tier_model_params?.sec ?? {}).toEqual({}); + }); + + it("snapshots a removed built-in row's models back into the record it came from", () => { + const { value } = apply(builtIn, { kind: "remove", id: "COMPLEX" }); + expect(value.custom_tier_set?.tiers.map((row) => row.id)).toEqual(["SIMPLE", "MEDIUM", "REASONING"]); + expect(value.tiers.COMPLEX).toEqual(["gpt-4"]); + }); + + it("re-points a fallback whose row was removed rather than leaving it dangling", () => { + const { value } = apply(custom, { kind: "remove", id: "CASUAL" }); + expect(value.custom_tier_set?.fallback_tier_id).toBe("sec"); + }); + + it("turns the plan-mode floor off when its row is gone, in the same write", () => { + const withFloor: ComplexityRouterConfigValue = { ...custom, plan_mode_min_tier: "sec" }; + const { value } = apply(withFloor, { kind: "remove", id: "sec" }); + expect(value.plan_mode_min_tier).toBeUndefined(); + }); + + it("leaves no custom row's effort settings behind when restoring the built-in tiers", () => { + const withEfforts: ComplexityRouterConfigValue = { + ...custom, + tier_model_params: { sec: { "gpt-4": { reasoning_effort: "high" } }, SIMPLE: { "gpt-3.5-turbo": {} } }, + }; + const { value } = apply(withEfforts, { kind: "restore" }); + expect(value.custom_tier_set).toBeUndefined(); + expect(Object.keys(value.tier_model_params ?? {})).toEqual(["SIMPLE"]); + }); + + it("keeps a renamed built-in row's effort settings across a restore", () => { + const renamed: ComplexityRouterConfigValue = { + ...custom, + tier_model_params: { COMPLEX: { "gpt-4": { reasoning_effort: "high" } } }, + custom_tier_set: { + tiers: [ + { id: "SIMPLE", name: "SIMPLE", definition: "", models: ["gpt-3.5-turbo"] }, + { id: "COMPLEX", name: "DEEP_WORK", definition: "renamed built-in", models: ["gpt-4"] }, + ], + fallback_tier_id: "SIMPLE", + }, + }; + const { value } = apply(renamed, { kind: "restore" }); + expect(value.tiers.COMPLEX).toEqual(["gpt-4"]); + expect(value.tier_model_params).toEqual({ COMPLEX: { "gpt-4": { reasoning_effort: "high" } } }); + }); + + it("re-points a rule that followed a renamed built-in back to its tier on restore", () => { + const renamed: ComplexityRouterConfigValue = { + ...builtIn, + custom_tier_set: { + tiers: [ + { id: "SIMPLE", name: "SIMPLE", definition: "", models: ["gpt-3.5-turbo"] }, + { id: "COMPLEX", name: "DEEP_WORK", definition: "", models: ["gpt-4"] }, + ], + fallback_tier_id: "SIMPLE", + }, + }; + const rules = [{ id: "r1", keywords: ["proof"], tier: "DEEP_WORK" }]; + const next = applyTierSetAction(renamed, rules, { kind: "restore" }); + expect(next.keywordTierRules).toEqual([{ id: "r1", keywords: ["proof"], tier: "COMPLEX" }]); + }); + + it("leaves a rule on a tier restore removes, so the save gate flags it rather than dropping it", () => { + const rules = [{ id: "r1", keywords: ["smalltalk"], tier: "CASUAL" }]; + const next = applyTierSetAction(custom, rules, { kind: "restore" }); + expect(next.keywordTierRules).toBe(rules); + }); + + it("keeps a removed row's rule in the same write, for the save gate to name", () => { + const rules = [{ id: "r1", keywords: ["audit"], tier: "SECURITY_REVIEW" }]; + const next = applyTierSetAction(custom, rules, { kind: "remove", id: "sec" }); + expect(next.keywordTierRules).toBe(rules); + }); + + it("resets a full set to the four built-ins rather than stacking them on top", () => { + const six: ComplexityRouterConfigValue = { + ...custom, + custom_tier_set: { + tiers: Array.from({ length: 6 }, (_, i) => ({ + id: `row-${i}`, + name: `TIER_${i}`, + definition: "d", + models: ["gpt-4"], + })), + fallback_tier_id: "row-0", + }, + }; + const { value } = apply(six, { kind: "restore" }); + expect(value.custom_tier_set).toBeUndefined(); + expect(Object.keys(value.tiers)).toEqual(["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"]); + }); +}); + +describe("setFallbackTier", () => { + it("re-points the fallback without disturbing the rows", () => { + const next = setFallbackTier(custom, "sec"); + expect(next.custom_tier_set?.fallback_tier_id).toBe("sec"); + expect(next.custom_tier_set?.tiers).toHaveLength(2); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/tier_set_actions.ts b/ui/litellm-dashboard/src/components/add_model/tier_set_actions.ts new file mode 100644 index 00000000000..c8254b6bff9 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/tier_set_actions.ts @@ -0,0 +1,150 @@ +import type { KeywordTierRule } from "./KeywordTierRules"; +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { pruneTierModelParams } from "./complexity_router_tiers"; +import { + type ActiveTierRow, + type TierRow, + TIER_ORDER, + activeTierName, + activeTierRows, + rowParamsByTier, + sameTierIdentity, + tierRowById, + tierRowByName, +} from "./tier_rows"; + +export type TierSetAction = + | { kind: "models"; id: string; models: string[] } + | { kind: "patch"; id: string; patch: Partial> } + | { kind: "add" } + | { kind: "remove"; id: string } + | { kind: "restore" }; + +/** What an action produces: the next config, and the keyword rules that followed their rows. */ +export interface TierSetResult { + value: ComplexityRouterConfigValue; + keywordTierRules: readonly KeywordTierRule[]; +} + +// The sole tier-set writer: it owns where rows live and reconciles both row-id pointers, so a +// fallback re-points and a floor whose row is gone turns off in the same write. +const commitTierRows = ( + rows: TierRow[], + fallbackTierId: string, + base: ComplexityRouterConfigValue, +): ComplexityRouterConfigValue => { + const floorGone = base.plan_mode_min_tier !== undefined && !rows.some((row) => row.id === base.plan_mode_min_tier); + const next = floorGone ? { ...base, plan_mode_min_tier: undefined } : base; + if (!next.custom_tier_set) { + return { ...next, tiers: { ...next.tiers, ...Object.fromEntries(rows.map((row) => [row.id, row.models])) } }; + } + const fallback_tier_id = rows.some((row) => row.id === fallbackTierId) + ? fallbackTierId + : (tierRowByName(rows, "MEDIUM") ?? rows[0])?.id ?? ""; + return { ...next, custom_tier_set: { tiers: rows, fallback_tier_id } }; +}; + +const asCustomBase = (base: ComplexityRouterConfigValue): ComplexityRouterConfigValue => + base.custom_tier_set + ? base + : { ...base, custom_tier_set: { tiers: activeTierRows(base), fallback_tier_id: "MEDIUM" } }; + +// A rule follows the row holding its name through any action, so neither a rename nor a restore can +// orphan it. A rule whose name is ambiguous across rows, or whose row the action deleted, stays put +// for the save gate to name loudly rather than being dropped or rewritten by guess. +const followedRuleTier = (before: readonly TierRow[], after: readonly TierRow[], tier: string): string | undefined => { + const holders = before.filter((row) => sameTierIdentity(row.name, tier)); + if (holders.length !== 1 || activeTierName(holders[0]) !== tier) return undefined; + const target = tierRowById(after, holders[0].id); + return target === undefined ? undefined : activeTierName(target); +}; + +const rulesFollowingRows = ( + before: readonly TierRow[], + after: readonly TierRow[], + rules: readonly KeywordTierRule[], +): readonly KeywordTierRule[] => { + const followed = rules.map((rule) => { + const tier = followedRuleTier(before, after, rule.tier); + return tier === undefined || tier === rule.tier ? rule : { ...rule, tier }; + }); + return followed.every((rule, index) => rule === rules[index]) ? rules : followed; +}; + +// Models and params both come from these rows, so the two cannot be keyed differently. +const exitToBuiltInTiers = (value: ComplexityRouterConfigValue, rows: readonly ActiveTierRow[]) => { + const { custom_tier_set: _dropped, ...rest } = value; + const builtInRows: ActiveTierRow[] = TIER_ORDER.map( + (tier) => + tierRowById(rows, tier) ?? { + id: tier, + name: tier, + definition: "", + models: value.tiers[tier], + params: value.tier_model_params?.[tier] ?? {}, + }, + ); + const restored: ComplexityRouterConfigValue = { + ...rest, + tier_model_params: rowParamsByTier(builtInRows), + tiers: { ...value.tiers, ...Object.fromEntries(builtInRows.map((row) => [row.id, row.models])) }, + }; + return commitTierRows(activeTierRows(restored), "", restored); +}; + +const nextTierSetValue = ( + value: ComplexityRouterConfigValue, + rows: ActiveTierRow[], + action: TierSetAction, +): ComplexityRouterConfigValue => { + const fallbackId = value.custom_tier_set?.fallback_tier_id ?? "MEDIUM"; + + switch (action.kind) { + case "models": + return commitTierRows( + rows.map((row) => (row.id === action.id ? { ...row, models: action.models } : row)), + fallbackId, + { ...value, tier_model_params: pruneTierModelParams(value.tier_model_params, action.id, action.models) }, + ); + case "patch": + return commitTierRows( + rows.map((row) => (row.id === action.id ? { ...row, ...action.patch } : row)), + fallbackId, + asCustomBase(value), + ); + case "add": + return commitTierRows( + [...rows, { id: crypto.randomUUID(), name: "", definition: "", models: [] }], + fallbackId, + asCustomBase(value), + ); + case "remove": { + const removed = tierRowById(rows, action.id); + const snapshot = + removed && (TIER_ORDER as string[]).includes(action.id) + ? { ...value, tiers: { ...value.tiers, [action.id]: removed.models } } + : value; + return commitTierRows( + rows.filter((row) => row.id !== action.id), + fallbackId, + asCustomBase(snapshot), + ); + } + case "restore": + return exitToBuiltInTiers(value, rows); + } +}; + +export const applyTierSetAction = ( + value: ComplexityRouterConfigValue, + keywordTierRules: readonly KeywordTierRule[], + action: TierSetAction, +): TierSetResult => { + const rows = activeTierRows(value); + const next = nextTierSetValue(value, rows, action); + return { value: next, keywordTierRules: rulesFollowingRows(rows, activeTierRows(next), keywordTierRules) }; +}; + +/** The fallback-tier select is the only writer that re-points rather than reconciling. */ +export const setFallbackTier = (value: ComplexityRouterConfigValue, id: string): ComplexityRouterConfigValue => + commitTierRows(activeTierRows(value), id, value); From 3300fc3a969182dbc4383e0d7aca602055f15601 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 27 Aug 2026 20:46:46 -0700 Subject: [PATCH 026/105] fix(moonshot, together_ai): send the reasoning effort Kimi K3 accepts (#38611) * fix(moonshot, together_ai): send the reasoning effort Kimi K3 accepts Moonshot documents reasoning_effort as a top-level chat completions field for its reasoning models, and defaults it to max, but MoonshotChatConfig builds its supported params by subtracting from the OpenAI base list, which never carried that param. An explicit level raised UnsupportedParamsError before the request left the proxy, so low and high were unreachable and every call ran at the provider default Together accepts low, high and max on Kimi K3. The per-model clamp added for the gpt-oss family folds max down to high for every model except deepseek-ai/DeepSeek-V4-Pro, so a caller asking for max silently got roughly half the reasoning budget they paid for Moonshot now offers reasoning_effort whenever the registry says the model reasons. Together sends a level the map entry declares unchanged, and keeps its existing table for every level an entry does not name, so the only value that moves is Kimi K3 at max * fix(moonshot): unwrap the bridges' effort object to the level string --- litellm/llms/moonshot/chat/transformation.py | 40 ++++++++----- .../llms/together_ai/chat/transformation.py | 3 + .../reasoning_effort_capability.py | 16 +++++ .../test_moonshot_chat_transformation.py | 59 +++++++++++++++++++ .../test_together_ai_chat_transformation.py | 39 ++++++++++++ 5 files changed, 143 insertions(+), 14 deletions(-) diff --git a/litellm/llms/moonshot/chat/transformation.py b/litellm/llms/moonshot/chat/transformation.py index 8e4b116d79f..7b0fcd24770 100644 --- a/litellm/llms/moonshot/chat/transformation.py +++ b/litellm/llms/moonshot/chat/transformation.py @@ -2,7 +2,7 @@ Translates from OpenAI's `/v1/chat/completions` to Moonshot AI's `/v1/chat/completions` """ -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping from typing import Any, Final, Literal, cast, overload import litellm @@ -16,6 +16,15 @@ from litellm.utils import supports_reasoning from ...openai.chat.gpt_transformation import OpenAIGPTConfig +def _reasoning_effort_string(value: object) -> str | None: + """The /v1/messages and /v1/responses bridges wrap the level as {"effort", "summary"} for + providers with a reasoning-summary surface. Moonshot's API takes only the bare string and 400s + on an object, so the level is unwrapped and the summary, which has no Moonshot equivalent, is + dropped.""" + effort: Final = value.get("effort") if isinstance(value, Mapping) else value + return effort if isinstance(effort, str) else None + + class MoonshotChatConfig(OpenAIGPTConfig): @overload def _transform_messages( @@ -93,20 +102,18 @@ class MoonshotChatConfig(OpenAIGPTConfig): - functions parameter is not supported (use tools instead) - tool_choice doesn't support "required" value - kimi-thinking-preview doesn't support tool calls at all + + A reasoning model additionally takes `reasoning_effort`, which the OpenAI base list this + subtracts from does not carry, so it has to be added back rather than merely kept. """ - excluded_params: Final[list[str]] = ["functions"] - - # kimi-thinking-preview has additional limitations - if "kimi-thinking-preview" in model: - excluded_params.extend(["tools", "tool_choice"]) - + excluded_params: Final = frozenset( + ("functions", "tools", "tool_choice") if "kimi-thinking-preview" in model else ("functions",) + ) base_openai_params: Final = super().get_supported_openai_params(model=model) - final_params: Final[list[str]] = [] - for param in base_openai_params: - if param not in excluded_params: - final_params.append(param) - - return final_params + supported: Final = [param for param in base_openai_params if param not in excluded_params] + if supports_reasoning(model=model, custom_llm_provider="moonshot"): + return [*supported, "reasoning_effort"] + return supported def map_openai_params( self, @@ -126,7 +133,12 @@ class MoonshotChatConfig(OpenAIGPTConfig): for param, value in non_default_params.items(): if param == "max_completion_tokens": optional_params["max_tokens"] = value - elif param in supported_openai_params: + elif param not in supported_openai_params: + continue + elif param == "reasoning_effort": + if (effort := _reasoning_effort_string(value)) is not None: + optional_params["reasoning_effort"] = effort + else: optional_params[param] = value ########################################## diff --git a/litellm/llms/together_ai/chat/transformation.py b/litellm/llms/together_ai/chat/transformation.py index 47a11230cb1..449cd3ecbc5 100644 --- a/litellm/llms/together_ai/chat/transformation.py +++ b/litellm/llms/together_ai/chat/transformation.py @@ -18,6 +18,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.exceptions import UnsupportedParamsError +from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model from litellm.types.llms.openai import AllMessageValues from litellm.utils import supports_function_calling, supports_reasoning, supports_response_schema @@ -139,6 +140,8 @@ def _reasoning_effort_payload(effort: str, model: str) -> Mapping[str, object]: if effort == "none": disable_reasoning: Final[TogetherReasoningToggle] = {"enabled": False} return MappingProxyType({"reasoning": disable_reasoning}) + if effort in (declared_reasoning_efforts_for_model(model, "together_ai") or ()): + return MappingProxyType({"reasoning_effort": effort}) if model.startswith(HIGH_MAX_EFFORT_MODEL_PREFIX): return MappingProxyType({"reasoning_effort": HIGH_MAX_EFFORT_TRANSLATION.get(effort, effort)}) return MappingProxyType({"reasoning_effort": EFFORT_TRANSLATION.get(effort, effort)}) diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 08feb96e36a..2a4fae4109f 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -87,6 +87,22 @@ def declared_reasoning_efforts(model_info: Mapping[str, object]) -> tuple[str, . return tuple(effort for effort in REASONING_EFFORT_ADVERTISEMENT_ORDER if effort in declared) +def declared_reasoning_efforts_for_model(model: str, custom_llm_provider: str) -> tuple[str, ...] | None: + """The levels an entry declares, resolved from the model string a provider config holds rather + than from a router deployment's model_info. + + None means the map has no opinion, either because the entry declares nothing or because it + describes no such model, so a caller keeps whatever it did before the entry was described. The + entry is read straight off the map rather than through get_model_info, which raises for a model + it does not know: a provider config runs on the request path for every model it serves, most of + which the map never named, and a lookup miss there must not fail the call. + """ + entry: Final = litellm.model_cost.get(f"{custom_llm_provider}/{model}") or litellm.model_cost.get(model) + if not isinstance(entry, dict): + return None + return declared_reasoning_efforts(entry) + + def _supports_none_reasoning_effort(model_info: Mapping[str, object], flag: object) -> bool: """Opt-in only where a request path refuses the level. AzureOpenAIGPT5Config raises UnsupportedParamsError on reasoning_effort='none' without an explicit true, and it is selected diff --git a/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py b/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py index 50f476eaaaa..8c8bea00dea 100644 --- a/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py +++ b/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py @@ -769,3 +769,62 @@ class TestMoonshotResponseSchemaSupport: def test_supports_response_schema_utility_reports_true(self, model_cost_map, monkeypatch): monkeypatch.setattr(litellm, "model_cost", model_cost_map) assert litellm.utils.supports_response_schema(model="moonshot/kimi-k2.5") is True + + +class TestMoonshotReasoningEffort: + """Moonshot documents reasoning_effort as a top-level chat completions field for its reasoning + models, defaulting to max, but the OpenAI base list this config subtracts from never carried it, + so an explicit level raised UnsupportedParamsError before it reached the wire.""" + + @pytest.fixture(autouse=True) + def force_local_model_cost(self, monkeypatch): + monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map()) + + @pytest.mark.parametrize("model", ["kimi-k3", "kimi-k2.5", "kimi-k2.6", "kimi-k2-thinking"]) + def test_reasoning_model_supports_reasoning_effort(self, model): + assert "reasoning_effort" in MoonshotChatConfig().get_supported_openai_params(model) + + @pytest.mark.parametrize("model", ["moonshot-v1-8k", "kimi-latest", "kimi-k2-turbo-preview"]) + def test_non_reasoning_model_does_not_support_reasoning_effort(self, model): + assert "reasoning_effort" not in MoonshotChatConfig().get_supported_openai_params(model) + + @pytest.mark.parametrize("effort", ["low", "high", "max"]) + def test_declared_effort_reaches_optional_params(self, effort): + optional_params = litellm.get_optional_params( + model="kimi-k3", + custom_llm_provider="moonshot", + reasoning_effort=effort, + drop_params=False, + ) + + assert optional_params["reasoning_effort"] == effort + + def test_non_reasoning_model_still_rejects_reasoning_effort(self): + with pytest.raises(litellm.UnsupportedParamsError): + litellm.get_optional_params( + model="moonshot-v1-8k", + custom_llm_provider="moonshot", + reasoning_effort="high", + drop_params=False, + ) + + def test_bridge_effort_dict_is_unwrapped_to_the_level_string(self): + optional_params = MoonshotChatConfig().map_openai_params( + non_default_params={"reasoning_effort": {"effort": "high", "summary": "detailed"}}, + optional_params={}, + model="kimi-k3", + drop_params=False, + ) + + assert optional_params["reasoning_effort"] == "high" + + @pytest.mark.parametrize("value", [{"summary": "detailed"}, {"effort": 3}, 7]) + def test_effort_without_a_level_string_is_omitted(self, value): + optional_params = MoonshotChatConfig().map_openai_params( + non_default_params={"reasoning_effort": value}, + optional_params={}, + model="kimi-k3", + drop_params=False, + ) + + assert "reasoning_effort" not in optional_params diff --git a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py index 6b803578067..7eb7dc41d4f 100644 --- a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py +++ b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py @@ -1069,3 +1069,42 @@ def test_anthropic_messages_streams_together_tool_call_as_input_json_delta(): ) assert thinking_text == "Need weather and time." assert [event["delta"]["stop_reason"] for event in events if event["type"] == "message_delta"] == ["tool_use"] + + +DECLARED_LEVELS_MODEL = "moonshotai/Kimi-K3" + + +@pytest.mark.parametrize("effort", ["low", "high", "max"]) +def test_declared_level_is_sent_unchanged(effort): + """Kimi K3 declares low, high and max in the model map and Together accepts all three, but the + per-model clamp below only spares deepseek-ai/DeepSeek-V4-Pro, so max used to arrive as high and + the caller silently lost half the reasoning budget they asked for.""" + mapped = _map_reasoning_effort(DECLARED_LEVELS_MODEL, effort) + + assert mapped["reasoning_effort"] == effort + + +@pytest.mark.parametrize("effort, expected", [("minimal", "low"), ("medium", "medium"), ("xhigh", "high")]) +def test_undeclared_level_still_uses_the_clamp(effort, expected): + """The declared set is not a licence to widen: a level the entry does not name keeps whatever + the hardcoded table did for it.""" + mapped = _map_reasoning_effort(DECLARED_LEVELS_MODEL, effort) + + assert mapped["reasoning_effort"] == expected + + +def test_declared_levels_model_still_disables_reasoning_on_none(): + mapped = _map_reasoning_effort(DECLARED_LEVELS_MODEL, "none") + + assert mapped["reasoning"] == {"enabled": False} + assert "reasoning_effort" not in mapped + + +def test_get_optional_params_preserves_max_for_declared_levels_model(): + optional_params = litellm.get_optional_params( + model=DECLARED_LEVELS_MODEL, + custom_llm_provider="together_ai", + reasoning_effort="max", + ) + + assert optional_params["reasoning_effort"] == "max" From bc127a0b827953766ee97e6e36250954815fc724 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 27 Aug 2026 20:49:22 -0700 Subject: [PATCH 027/105] feat(proxy): reject wildcard project models under enforce_project_model_quota Project auth expands all-proxy-models, * patterns, and access-group names to many concrete models, but the rate limiter looks quotas up by the exact requested model name, so a quota keyed on one of those entries is never applied. Fail loudly with a 400 instead of storing an unenforceable quota --- .../management_endpoints/project_endpoints.py | 57 ++++++- .../test_project_endpoints_prisma.py | 140 +++++++++++++----- 2 files changed, 158 insertions(+), 39 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py index 7ea27498c1b..5ec482c385a 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py @@ -12,7 +12,7 @@ Endpoints for /project operations import json from collections.abc import Sequence -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Final from fastapi import APIRouter, Depends, HTTPException, Request @@ -35,6 +35,8 @@ if TYPE_CHECKING: LiteLLM_VerificationTokenActions, ) + from litellm import Router + router = APIRouter() @@ -225,7 +227,40 @@ def _project_models_missing_positive_quota( return [model for model in (models or []) if not _is_positive(rpm.get(model)) or not _is_positive(tpm.get(model))] -def _raise_on_missing_project_model_quota(data: NewProjectRequest | UpdateProjectRequest) -> None: +def _router_access_group_names(llm_router: "Router | None") -> frozenset[str]: + return frozenset(llm_router.get_model_access_groups()) if llm_router is not None else frozenset() + + +def _project_models_expanding_at_request_time( + models: Sequence[str] | None, access_group_names: frozenset[str] +) -> tuple[str, ...]: + """Entries project auth expands to many concrete models (`all-proxy-models`, `*` patterns, + access groups). The rate limiter looks quotas up by the exact requested model name, so a + quota keyed on one of these entries is never applied.""" + return tuple( + model + for model in (models or ()) + if model == SpecialModelNames.all_proxy_models.value or "*" in model or model in access_group_names + ) + + +def _raise_on_project_models_expanding_at_request_time( + models: Sequence[str] | None, access_group_names: frozenset[str] +) -> None: + expanding: Final = _project_models_expanding_at_request_time(models, access_group_names) + if not expanding: + return + raise HTTPException( + status_code=400, + detail={ + "error": f"models {list(expanding)} expand to multiple models at request time, so a per-model rpm/tpm quota cannot be enforced for them while 'enforce_project_model_quota' is enabled. List concrete model names instead." + }, + ) + + +def _raise_on_missing_project_model_quota( + data: NewProjectRequest | UpdateProjectRequest, access_group_names: frozenset[str] = frozenset() +) -> None: """Require a positive `rpm`/`tpm` quota for every model on project CREATE. `model_rpm_limit`/`model_tpm_limit` are relocated into `metadata` by the request @@ -234,6 +269,7 @@ def _raise_on_missing_project_model_quota(data: NewProjectRequest | UpdateProjec Only invoked when `general_settings.enforce_project_model_quota` is enabled (default off), so it is opt-in and does not change behavior for existing users. """ + _raise_on_project_models_expanding_at_request_time(data.models, access_group_names) metadata = data.metadata or {} missing = _project_models_missing_positive_quota( data.models, metadata.get("model_rpm_limit"), metadata.get("model_tpm_limit") @@ -248,7 +284,9 @@ def _raise_on_missing_project_model_quota(data: NewProjectRequest | UpdateProjec ) -def _raise_on_missing_project_model_quota_on_update(data: UpdateProjectRequest, existing_project: object) -> None: +def _raise_on_missing_project_model_quota_on_update( + data: UpdateProjectRequest, existing_project: object, access_group_names: frozenset[str] = frozenset() +) -> None: """Require a positive `rpm`/`tpm` quota over the RESULTING state on project UPDATE. `/project/update` replaces `models` and `metadata` when they are provided, so the @@ -260,7 +298,10 @@ def _raise_on_missing_project_model_quota_on_update(data: UpdateProjectRequest, (default off), so it is opt-in and does not change behavior for existing users. """ resulting_models = data.models if data.models is not None else (getattr(existing_project, "models", None) or []) - resulting_metadata = data.metadata if data.metadata is not None else (getattr(existing_project, "metadata", None) or {}) + resulting_metadata = ( + data.metadata if data.metadata is not None else (getattr(existing_project, "metadata", None) or {}) + ) + _raise_on_project_models_expanding_at_request_time(resulting_models, access_group_names) missing = _project_models_missing_positive_quota( resulting_models, resulting_metadata.get("model_rpm_limit"), resulting_metadata.get("model_tpm_limit") ) @@ -423,6 +464,7 @@ async def new_project( from litellm.proxy.proxy_server import ( general_settings, litellm_proxy_admin_name, + llm_router, premium_user, prisma_client, ) @@ -471,7 +513,7 @@ async def new_project( # Opt-in (default off): require rpm/tpm for every model added to the project. if general_settings.get("enforce_project_model_quota", False): - _raise_on_missing_project_model_quota(data) + _raise_on_missing_project_model_quota(data, _router_access_group_names(llm_router)) # Check if user has permission to create projects for this team # only team admins can create projects for their team @@ -614,6 +656,7 @@ async def update_project( from litellm.proxy.proxy_server import ( general_settings, litellm_proxy_admin_name, + llm_router, premium_user, prisma_client, user_api_key_cache, @@ -719,7 +762,9 @@ async def update_project( # Opt-in (default off): require rpm/tpm for every model the update would leave on the project. if general_settings.get("enforce_project_model_quota", False): - _raise_on_missing_project_model_quota_on_update(data, existing_project) + _raise_on_missing_project_model_quota_on_update( + data, existing_project, _router_access_group_names(llm_router) + ) # Prepare update data update_data = _jsonified(prisma_client, data.model_dump(exclude_none=True, exclude={"project_id"})) diff --git a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py b/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py index 362b7908052..d7dd2d9adc4 100644 --- a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py +++ b/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py @@ -4,7 +4,7 @@ from litellm._uuid import uuid from unittest import mock from dotenv import load_dotenv -from fastapi import Request +from fastapi import HTTPException, Request load_dotenv() import time @@ -1080,8 +1080,7 @@ def test_enforce_project_model_quota_all_present_passes(): model_rpm_limit={"gpt-5.5": 100}, model_tpm_limit={"gpt-5.5": 1000}, ) - # Should not raise. - _raise_on_missing_project_model_quota(data) + assert _raise_on_missing_project_model_quota(data) is None def test_enforce_project_model_quota_no_models_passes(): @@ -1091,8 +1090,7 @@ def test_enforce_project_model_quota_no_models_passes(): ) data = NewProjectRequest(team_id="test-team") - # Should not raise. - _raise_on_missing_project_model_quota(data) + assert _raise_on_missing_project_model_quota(data) is None def test_enforce_project_model_quota_zero_rejected(): @@ -1158,8 +1156,7 @@ def test_update_quota_adds_model_with_quota_passes(): model_rpm_limit={"gpt-5.5": 100}, model_tpm_limit={"gpt-5.5": 1000}, ) - # Should not raise. - _raise_on_missing_project_model_quota_on_update(data, existing) + assert _raise_on_missing_project_model_quota_on_update(data, existing) is None def test_update_quota_partial_update_keeps_existing_valid_passes(): @@ -1176,8 +1173,7 @@ def test_update_quota_partial_update_keeps_existing_valid_passes(): metadata={"model_rpm_limit": {"gpt-5.5": 100}, "model_tpm_limit": {"gpt-5.5": 1000}}, ) data = UpdateProjectRequest(project_id="p", description="unrelated change") - # Should not raise (existing quota is valid, update doesn't touch it). - _raise_on_missing_project_model_quota_on_update(data, existing) + assert _raise_on_missing_project_model_quota_on_update(data, existing) is None def test_update_quota_existing_quotaless_model_rejected(): @@ -1195,34 +1191,34 @@ def test_update_quota_existing_quotaless_model_rejected(): _raise_on_missing_project_model_quota_on_update(data, existing) -@pytest.mark.asyncio -async def test_new_project_flag_on_missing_rpm_tpm_returns_400(): - """End-to-end: with the flag on, POST /project/new rejects a model added without rpm/tpm.""" - from unittest.mock import AsyncMock, MagicMock, patch - - from fastapi import Request - +def _enforced_new_project_mocks(monkeypatch, team_models: list[str], llm_router: mock.MagicMock | None) -> None: from litellm.proxy._types import LiteLLM_TeamTable from litellm_enterprise.proxy.management_endpoints import project_endpoints as pe - team = LiteLLM_TeamTable(team_id="test-team", models=["gpt-5.5"]) - data = NewProjectRequest(team_id="test-team", models=["gpt-5.5"]) # no rpm/tpm + team = LiteLLM_TeamTable(team_id="test-team", models=team_models) + monkeypatch.setattr(litellm.proxy.proxy_server, "prisma_client", mock.MagicMock()) + monkeypatch.setattr(litellm.proxy.proxy_server, "premium_user", True) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) + monkeypatch.setattr(litellm.proxy.proxy_server, "general_settings", {"enforce_project_model_quota": True}) + monkeypatch.setattr(pe, "_validate_team_exists", mock.AsyncMock(return_value=team)) + monkeypatch.setattr(pe, "_check_user_permission_for_project", mock.AsyncMock(return_value=True)) - with ( - patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), - patch("litellm.proxy.proxy_server.premium_user", True), - patch("litellm.proxy.proxy_server.general_settings", {"enforce_project_model_quota": True}), - patch.object(pe, "_validate_team_exists", AsyncMock(return_value=team)), - patch.object(pe, "_check_user_permission_for_project", AsyncMock(return_value=True)), - ): - with pytest.raises(Exception) as exc_info: - await pe.new_project( - data=data, - http_request=Request(scope={"type": "http"}), - user_api_key_dict=UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234", user_id="1234" - ), - ) + +async def _run_new_project(data: NewProjectRequest) -> None: + await new_project( + data=data, + http_request=Request(scope={"type": "http"}), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234", user_id="1234"), + ) + + +@pytest.mark.asyncio +async def test_new_project_flag_on_missing_rpm_tpm_returns_400(monkeypatch): + """End-to-end: with the flag on, POST /project/new rejects a model added without rpm/tpm.""" + _enforced_new_project_mocks(monkeypatch, team_models=["gpt-5.5"], llm_router=None) + + with pytest.raises(Exception) as exc_info: + await _run_new_project(NewProjectRequest(team_id="test-team", models=["gpt-5.5"])) # new_project re-wraps the HTTPException, so assert on the string form. assert "rpm/tpm quota" in str(exc_info.value) @@ -1293,3 +1289,81 @@ async def test_update_project_leaves_metadata_untouched_when_no_limit_is_sent(mo await _run_project_update(project_id, description="renamed only") assert "metadata" not in _written_project_data(mock_prisma) + + +@pytest.mark.parametrize("entry", ["all-proxy-models", "*", "azure/*"]) +def test_enforce_project_model_quota_rejects_entries_that_expand_at_request_time(entry): + """A quota keyed on a wildcard entry is never applied by the limiter, so it fails loudly.""" + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota, + ) + + data = NewProjectRequest( + team_id="test-team", + models=[entry], + model_rpm_limit={entry: 10}, + model_tpm_limit={entry: 1000}, + ) + with pytest.raises(HTTPException) as exc_info: + _raise_on_missing_project_model_quota(data) + assert exc_info.value.status_code == 400 + assert entry in str(exc_info.value.detail) + assert "expand to multiple models at request time" in str(exc_info.value.detail) + + +def test_enforce_project_model_quota_rejects_access_group_only_when_router_defines_it(): + """A plain model name passes; the same name is rejected once the router reports it as an access group.""" + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota, + ) + + data = NewProjectRequest( + team_id="test-team", + models=["prod-models"], + model_rpm_limit={"prod-models": 10}, + model_tpm_limit={"prod-models": 1000}, + ) + assert _raise_on_missing_project_model_quota(data, access_group_names=frozenset()) is None + with pytest.raises(HTTPException) as exc_info: + _raise_on_missing_project_model_quota(data, access_group_names=frozenset({"prod-models"})) + assert "prod-models" in str(exc_info.value.detail) + assert "expand to multiple models at request time" in str(exc_info.value.detail) + + +def test_update_quota_rejects_wildcard_left_on_project(): + """An update that leaves a wildcard entry on the project is rejected even when it carries a quota.""" + import types + + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota_on_update, + ) + + existing = types.SimpleNamespace( + models=["all-proxy-models"], + metadata={"model_rpm_limit": {"all-proxy-models": 10}, "model_tpm_limit": {"all-proxy-models": 1000}}, + ) + data = UpdateProjectRequest(project_id="p", description="unrelated change") + with pytest.raises(HTTPException) as exc_info: + _raise_on_missing_project_model_quota_on_update(data, existing) + assert "all-proxy-models" in str(exc_info.value.detail) + assert "expand to multiple models at request time" in str(exc_info.value.detail) + + +@pytest.mark.asyncio +async def test_new_project_flag_on_access_group_model_returns_400(monkeypatch): + """End-to-end: the router's access groups reach the check, so an access-group entry is rejected.""" + llm_router = mock.MagicMock() + llm_router.get_model_access_groups.return_value = {"prod-models": ["gpt-5.5"]} + _enforced_new_project_mocks(monkeypatch, team_models=["prod-models"], llm_router=llm_router) + data = NewProjectRequest( + team_id="test-team", + models=["prod-models"], + model_rpm_limit={"prod-models": 10}, + model_tpm_limit={"prod-models": 1000}, + ) + + with pytest.raises(Exception) as exc_info: + await _run_new_project(data) + + assert "prod-models" in str(exc_info.value) + assert "expand to multiple models at request time" in str(exc_info.value) From 929946bdc17e375946ebea27e216302425d3c641 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 27 Aug 2026 20:50:41 -0700 Subject: [PATCH 028/105] fix(ui): give the shared product-link class a focus ring Docs went from a ghost Button to a plain anchor, which dropped the focus treatment the Button was supplying, so tabbing to Docs showed nothing while tabbing to Blog showed a ring. The ring now lives on the shared class both sides use, matching the Button primitive's values. Kept Docs as a real anchor rather than routing it back through Button: nativeButton={false} stamps role="button" onto the element, so the old DashboardHeader markup announced Docs as a button and lost its link semantics. Tests pin both the ring and the link role. --- .../components/Navbar/DocsLink/DocsLink.test.tsx | 15 +++++++++++++++ .../src/components/Navbar/DocsLink/DocsLink.tsx | 7 +++++-- .../src/components/Navbar/navProductLinkClass.ts | 4 ++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/components/Navbar/DocsLink/DocsLink.test.tsx b/ui/litellm-dashboard/src/components/Navbar/DocsLink/DocsLink.test.tsx index d5a155cf503..dec76041802 100644 --- a/ui/litellm-dashboard/src/components/Navbar/DocsLink/DocsLink.test.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/DocsLink/DocsLink.test.tsx @@ -24,4 +24,19 @@ describe("DocsLink", () => { } expect(link).not.toHaveClass("text-muted-foreground"); }); + + it("carries a focus ring, so tabbing to Docs looks like tabbing to Blog", () => { + render(); + + const link = screen.getByRole("link", { name: "Docs" }); + expect(link).toHaveClass("focus-visible:ring-3"); + expect(link).toHaveClass("focus-visible:ring-ring/50"); + }); + + it("stays a link rather than being relabelled as a button by the Button primitive", () => { + render(); + + expect(screen.getByRole("link", { name: "Docs" })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Docs" })).not.toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/Navbar/DocsLink/DocsLink.tsx b/ui/litellm-dashboard/src/components/Navbar/DocsLink/DocsLink.tsx index 9176e1f9c38..9b4f4c90b64 100644 --- a/ui/litellm-dashboard/src/components/Navbar/DocsLink/DocsLink.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/DocsLink/DocsLink.tsx @@ -4,11 +4,14 @@ import React from "react"; export const DOCS_URL = "https://docs.litellm.ai/docs/"; +const ChevronWidthSpacer: React.FC = () => ( + +); + export const DocsLink: React.FC = () => ( Docs - {/* Docs is a single outbound link; the hidden chevron keeps its box identical to the Blog dropdown trigger. */} - + ); diff --git a/ui/litellm-dashboard/src/components/Navbar/navProductLinkClass.ts b/ui/litellm-dashboard/src/components/Navbar/navProductLinkClass.ts index e0fd000ab97..56b0d51401d 100644 --- a/ui/litellm-dashboard/src/components/Navbar/navProductLinkClass.ts +++ b/ui/litellm-dashboard/src/components/Navbar/navProductLinkClass.ts @@ -1,3 +1,3 @@ -/** Shared styling for Docs / Blog in the top nav (product navigation zone). */ +/** Shared styling for Docs / Blog in the top nav (product navigation zone). Focus ring matches the Button primitive. */ export const NAV_PRODUCT_LINK_CLASS = - "inline-flex h-9 shrink-0 items-center justify-center gap-1 rounded-md px-2 text-sm font-medium leading-none text-foreground transition-colors hover:bg-accent "; + "inline-flex h-9 shrink-0 items-center justify-center gap-1 rounded-md px-2 text-sm font-medium leading-none text-foreground outline-none transition-colors hover:bg-accent focus-visible:ring-3 focus-visible:ring-ring/50 "; From 4026aa6575ab13b86e08e9d149d64e5aeb24e232 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:58:14 -0700 Subject: [PATCH 029/105] fix(a2a): merge fresh agent vectors into the live cache and drop entries of another dimension --- litellm/proxy/agent_endpoints/agent_search.py | 12 +++++++--- .../agent_endpoints/test_agent_search.py | 22 +++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/agent_endpoints/agent_search.py b/litellm/proxy/agent_endpoints/agent_search.py index c249079ccac..46ab36d7b72 100644 --- a/litellm/proxy/agent_endpoints/agent_search.py +++ b/litellm/proxy/agent_endpoints/agent_search.py @@ -193,6 +193,14 @@ class AgentSearchIndex: def __init__(self) -> None: self._vectors: Mapping[str, Mapping[str, Vector]] = MappingProxyType({}) + def _merged(self, embedding_model: str, embedded: _Embedded) -> Mapping[str, Vector]: + kept: Final = { + text: vector + for text, vector in self._vectors.get(embedding_model, _NO_VECTORS).items() + if len(vector) == len(embedded.query_vector) + } + return MappingProxyType({**kept, **embedded.vectors}) + async def search( self, query: str, agents: Sequence[AgentResponse], top_k: int, embed: Embedder, embedding_model: str ) -> AgentSearchHits | AgentSearchEmbeddingFailed: @@ -207,9 +215,7 @@ class AgentSearchIndex: return AgentSearchEmbeddingFailed( reason=f"embedding model {embedding_model} returned vectors of mixed dimensions" ) - self._vectors = MappingProxyType( - {**self._vectors, embedding_model: MappingProxyType({**cached, **embedded.vectors})} - ) + self._vectors = MappingProxyType({**self._vectors, embedding_model: self._merged(embedding_model, embedded)}) ranked: Final = sorted( ( AgentSearchHit(agent=agent, score=cosine_similarity(embedded.query_vector, embedded.vectors[text])) diff --git a/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py b/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py index 93f29c4a1d2..3fb09076e5f 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py @@ -1,3 +1,4 @@ +import asyncio from collections.abc import Sequence from types import MappingProxyType from typing import Final @@ -84,6 +85,7 @@ class FixedDimensionEmbedder: async def __call__(self, texts: Sequence[str]) -> Sequence[Vector]: self.calls.append(tuple(texts)) + await asyncio.sleep(0) return tuple((1.0,) * self.dimensions for _ in texts) @@ -156,6 +158,26 @@ class TestAgentSearchIndex: ("language translation", *(agent_search_text(agent) for agent in AGENTS)), ] + @pytest.mark.asyncio + async def test_re_embedding_a_subset_drops_the_other_agents_old_vectors(self) -> None: + index = AgentSearchIndex() + await index.search("language translation", AGENTS, top_k=5, embed=FakeEmbedder(), embedding_model="m") + wide = FixedDimensionEmbedder(2) + await index.search("language translation", AGENTS[:1], top_k=5, embed=wide, embedding_model="m") + await index.search("language translation", AGENTS, top_k=5, embed=wide, embedding_model="m") + assert wide.calls[-1] == ("language translation", *(agent_search_text(agent) for agent in AGENTS[1:])) + + @pytest.mark.asyncio + async def test_concurrent_searches_keep_each_others_vectors(self) -> None: + index = AgentSearchIndex() + embedder = FixedDimensionEmbedder(3) + await asyncio.gather( + index.search("q", AGENTS[:1], top_k=5, embed=embedder, embedding_model="m"), + index.search("q", AGENTS[1:], top_k=5, embed=embedder, embedding_model="m"), + ) + await index.search("q", AGENTS, top_k=5, embed=embedder, embedding_model="m") + assert embedder.calls[-1] == ("q",) + @pytest.mark.asyncio async def test_mixed_dimensions_in_one_batch_become_embedding_failed(self) -> None: async def mixed(texts: Sequence[str]) -> Sequence[Vector]: From b72b9126b59aa72184ba81fc1acbc3b497386116 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 27 Aug 2026 20:58:16 -0700 Subject: [PATCH 030/105] fix(ui): restore the Blog hover highlight in the top bar The Blog trigger carried `bg-transparent!`, which emits an important background-color and so beat the non-important `hover:bg-accent` the shared product-link class supplies. Docs lit up on hover and Blog stayed flat, the same Docs/Blog inconsistency this branch is about on a different axis. Dropping the override lets the shared hover through. `border-0!` stays, since it keeps the trigger's box identical to the plain Docs anchor. Verified in the browser: both now paint lab(96.1596 -0.0823438 -1.13575) on hover at 36px tall. --- .../components/Navbar/BlogDropdown/BlogDropdown.test.tsx | 8 ++++++++ .../src/components/Navbar/BlogDropdown/BlogDropdown.tsx | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.test.tsx b/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.test.tsx index 0581a381a9a..4da50a266f0 100644 --- a/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.test.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.test.tsx @@ -66,6 +66,14 @@ describe("BlogDropdown", () => { expect(screen.getByRole("button", { name: /blog/i })).toBeInTheDocument(); }); + it("should keep the shared hover highlight rather than pinning the background transparent", () => { + renderWithProviders(); + + const trigger = screen.getByRole("button", { name: /blog/i }); + expect(trigger).toHaveClass("hover:bg-accent"); + expect(trigger.className).not.toMatch(/\bbg-\S*!/); + }); + it("should not render menu content before the trigger is hovered", () => { mockUseBlogPostsResult = { ...mockUseBlogPostsResult, data: { posts: MOCK_POSTS.slice(0, 1) } }; renderWithProviders(); diff --git a/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.tsx b/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.tsx index c4ff1ec72dd..eaaf3301a8b 100644 --- a/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/BlogDropdown/BlogDropdown.tsx @@ -85,7 +85,7 @@ export const BlogDropdown: React.FC = () => { } + render={ + + + + {isCustomSet && ( + + )} + + ) : ( + onEditingChange && ( + + ) + )} + + {editing && ( + + Add or remove tiers to define your own set. Every custom tier needs a definition the LLM classifier routes on, + and an edited set requires the LLM classification method + + )} + +); + +const FallbackTierField: React.FC<{ + rows: readonly TierRow[]; + fallbackTierId: string; + onValueChange: (rowId: string) => void; +}> = ({ rows, fallbackTierId, onValueChange }) => ( +
+
+ Fallback Tier + + + +
+ activeTierName(row)).map((row) => ({ value: row.id, label: activeTierName(row) }))} + value={fallbackTierId || null} + onValueChange={onValueChange} + placeholder="Pick the tier classifier failures route to" + /> +
+); + +const TierRowHeader: React.FC<{ + row: TierRow; + index: number; + rowCount: number; + label: string; + description: string | undefined; + editing: boolean; + isCustomSet: boolean; + onRemove: () => void; +}> = ({ row, index, rowCount, label, description, editing, isCustomSet, onRemove }) => ( +
+ {label} Tier + + + + + Tier {index + 1} of {rowCount} · {rowOrigin(row, isCustomSet)} + + {editing && ( + + )} +
+); + +const TierRowEditFields: React.FC<{ + row: TierRow; + index: number; + definitionMissing: boolean; + onPatch: (patch: Partial>) => void; +}> = ({ row, index, definitionMissing, onPatch }) => ( + <> + onPatch({ name: event.target.value })} + placeholder="Tier name, e.g. SECURITY_REVIEW" + aria-label={`Name for tier ${index + 1}`} + maxLength={MAX_TIER_NAME_CHARS} + className="mb-2" + /> +