mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
chore: merge litellm_internal_staging
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI
commit
e11866ec5d
43 changed files with 1705 additions and 177 deletions
44
.github/workflows/check-ui-api-types.yml
vendored
44
.github/workflows/check-ui-api-types.yml
vendored
|
|
@ -2,18 +2,19 @@ name: Check UI API Types Sync
|
|||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "litellm/proxy/**"
|
||||
- "litellm/types/**"
|
||||
- "ui/litellm-dashboard/src/lib/http/schema.d.ts"
|
||||
- "ui/litellm-dashboard/scripts/gen-api-types.mjs"
|
||||
- "ui/litellm-dashboard/package.json"
|
||||
- "ui/litellm-dashboard/package-lock.json"
|
||||
- ".github/workflows/check-ui-api-types.yml"
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_staging
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
check-sync:
|
||||
name: Verify schema.d.ts matches the proxy OpenAPI spec
|
||||
|
|
@ -24,18 +25,39 @@ jobs:
|
|||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 2
|
||||
|
||||
- name: Detect changes that can affect the generated types
|
||||
id: changes
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! base="$(git rev-parse --verify --quiet HEAD^2 >/dev/null && git rev-parse HEAD^1)"; then
|
||||
echo "Not a pull request merge commit, running the full check."
|
||||
echo "relevant=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
files="$(git diff --name-only "$base" HEAD)"
|
||||
if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/check-ui-api-types\.yml$)' <<< "$files"; then
|
||||
echo "relevant=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "No proxy, types or generator changes in this pull request, nothing to verify."
|
||||
echo "relevant=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Set up Python
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache uv dependencies
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: |
|
||||
|
|
@ -46,14 +68,17 @@ jobs:
|
|||
${{ runner.os }}-uv-
|
||||
|
||||
- name: Install backend dependencies
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
|
||||
|
||||
- name: Generate Prisma client
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
env:
|
||||
PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
|
||||
run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
||||
|
||||
- name: Set up Node.js
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
|
||||
with:
|
||||
node-version-file: ui/litellm-dashboard/.nvmrc
|
||||
|
|
@ -61,16 +86,19 @@ jobs:
|
|||
cache-dependency-path: ui/litellm-dashboard/package-lock.json
|
||||
|
||||
- name: Install dashboard dependencies
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
working-directory: ui/litellm-dashboard
|
||||
run: npm ci
|
||||
|
||||
- name: Regenerate types from the live spec
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
working-directory: ui/litellm-dashboard
|
||||
env:
|
||||
LITELLM_PYTHON: "uv run --no-sync python"
|
||||
run: npm run gen:api
|
||||
|
||||
- name: Fail if types are stale
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
run: |
|
||||
if ! git diff --exit-code -- ui/litellm-dashboard/src/lib/http/schema.d.ts; then
|
||||
echo "::error file=ui/litellm-dashboard/src/lib/http/schema.d.ts::Generated API types are out of sync with the proxy OpenAPI spec."
|
||||
|
|
|
|||
|
|
@ -1,15 +1,15 @@
|
|||
{
|
||||
"reportAny": {
|
||||
"limit": 27534
|
||||
"limit": 28842
|
||||
},
|
||||
"reportArgumentType": {
|
||||
"limit": 1758
|
||||
"limit": 2634
|
||||
},
|
||||
"reportAssignmentType": {
|
||||
"limit": 329
|
||||
},
|
||||
"reportAttributeAccessIssue": {
|
||||
"limit": 508
|
||||
"limit": 514
|
||||
},
|
||||
"reportCallIssue": {
|
||||
"limit": 117
|
||||
|
|
@ -24,7 +24,7 @@
|
|||
"limit": 19
|
||||
},
|
||||
"reportExplicitAny": {
|
||||
"limit": 8941
|
||||
"limit": 9103
|
||||
},
|
||||
"reportFunctionMemberAccess": {
|
||||
"limit": 7
|
||||
|
|
@ -57,7 +57,7 @@
|
|||
"limit": 5843
|
||||
},
|
||||
"reportMissingTypeArgument": {
|
||||
"limit": 15756
|
||||
"limit": 15816
|
||||
},
|
||||
"reportMissingTypeStubs": {
|
||||
"limit": 40
|
||||
|
|
@ -72,7 +72,7 @@
|
|||
"limit": 0
|
||||
},
|
||||
"reportOptionalMemberAccess": {
|
||||
"limit": 0
|
||||
"limit": 1078
|
||||
},
|
||||
"reportOptionalOperand": {
|
||||
"limit": 0
|
||||
|
|
@ -90,7 +90,7 @@
|
|||
"limit": 8
|
||||
},
|
||||
"reportReturnType": {
|
||||
"limit": 140
|
||||
"limit": 218
|
||||
},
|
||||
"reportTypedDictNotRequiredAccess": {
|
||||
"limit": 27
|
||||
|
|
@ -99,22 +99,22 @@
|
|||
"limit": 0
|
||||
},
|
||||
"reportUnknownArgumentType": {
|
||||
"limit": 45110
|
||||
"limit": 45098
|
||||
},
|
||||
"reportUnknownLambdaType": {
|
||||
"limit": 113
|
||||
},
|
||||
"reportUnknownMemberType": {
|
||||
"limit": 39796
|
||||
"limit": 39826
|
||||
},
|
||||
"reportUnknownParameterType": {
|
||||
"limit": 20225
|
||||
"limit": 20237
|
||||
},
|
||||
"reportUnknownVariableType": {
|
||||
"limit": 31233
|
||||
"limit": 31371
|
||||
},
|
||||
"reportUnnecessaryCast": {
|
||||
"limit": 110
|
||||
"limit": 122
|
||||
},
|
||||
"reportUnnecessaryComparison": {
|
||||
"limit": 701
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
ALTER TABLE "LiteLLM_AutoRouterSession" ADD COLUMN IF NOT EXISTS "tier_turns" JSONB NOT NULL DEFAULT '{}';
|
||||
|
|
@ -1439,6 +1439,7 @@ model LiteLLM_AutoRouterSession {
|
|||
total_tokens BigInt @default(0)
|
||||
spend Float @default(0)
|
||||
saved_spend Float @default(0)
|
||||
tier_turns Json @default("{}")
|
||||
|
||||
@@id([api_key, session_id, router_name])
|
||||
@@index([last_turn_at], map: "idx_autorouter_session_last_turn")
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ server-side using litellm router's search tools.
|
|||
import asyncio
|
||||
import math
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator, Mapping
|
||||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
|
||||
import litellm
|
||||
|
|
@ -37,6 +37,8 @@ from litellm.types.integrations.custom_logger import (
|
|||
AgenticLoopRequestPatch,
|
||||
)
|
||||
from litellm.types.integrations.websearch_interception import (
|
||||
AnthropicSearchQuery,
|
||||
AnthropicServerToolUseBlock,
|
||||
WebSearchInterceptionConfig,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
|
@ -833,22 +835,48 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
def _build_native_result_blocks(
|
||||
tool_calls: list[dict],
|
||||
structured_results: list[SearchResponse | None],
|
||||
) -> list[dict[str, object]]:
|
||||
"""Build one ``web_search_tool_result`` block per tool_call."""
|
||||
blocks: Final[list[dict[str, object]]] = []
|
||||
for i, tool_call in enumerate(tool_calls):
|
||||
tool_use_id = tool_call.get("id") or ""
|
||||
structured = structured_results[i] if i < len(structured_results) else None
|
||||
blocks.append(
|
||||
WebSearchTransformation.build_web_search_tool_result_block(
|
||||
tool_use_id=tool_use_id,
|
||||
search_response=structured,
|
||||
)
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
"""
|
||||
Build a ``server_tool_use`` + ``web_search_tool_result`` pair per tool_call.
|
||||
|
||||
The pair is what Anthropic's spec requires: a bare result block, or one
|
||||
keyed by the model's ``toolu_...`` id instead of a ``srvtoolu_...`` one,
|
||||
is rejected on replay ("String should match pattern '^srvtoolu_'") and
|
||||
leaves native clients without a search to attach the sources to.
|
||||
"""
|
||||
return tuple(
|
||||
block
|
||||
for i, tool_call in enumerate(tool_calls)
|
||||
for block in WebSearchInterceptionLogger._native_result_pair(
|
||||
query=WebSearchInterceptionLogger._tool_call_query(tool_call),
|
||||
search_response=structured_results[i] if i < len(structured_results) else None,
|
||||
)
|
||||
return blocks
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _inject_native_blocks(response: Any, native_blocks: list[dict[str, object]]) -> Any:
|
||||
def _tool_call_query(tool_call: Mapping[str, object]) -> str:
|
||||
tool_input: Final = tool_call.get("input")
|
||||
if not isinstance(tool_input, Mapping):
|
||||
return ""
|
||||
query: Final = tool_input.get("query")
|
||||
return query if isinstance(query, str) else ""
|
||||
|
||||
@staticmethod
|
||||
def _native_result_pair(
|
||||
query: str,
|
||||
search_response: SearchResponse | None,
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
tool_use_id: Final = f"srvtoolu_{uuid.uuid4().hex}"
|
||||
return (
|
||||
AnthropicServerToolUseBlock(id=tool_use_id, input=AnthropicSearchQuery(query=query)).model_dump(),
|
||||
WebSearchTransformation.build_web_search_tool_result_block(
|
||||
tool_use_id=tool_use_id,
|
||||
search_response=search_response,
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _inject_native_blocks(response: Any, native_blocks: Sequence[Mapping[str, object]]) -> Any:
|
||||
"""Prepend native blocks to response content, dict or object form."""
|
||||
if not native_blocks:
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -412,6 +412,15 @@ class WebSearchTransformation:
|
|||
block that should accompany the model's text reply when the original
|
||||
request used a native ``web_search_*`` tool.
|
||||
|
||||
The spec'd shape carries page text only in ``encrypted_content``, an
|
||||
opaque server-issued blob that we cannot mint. Emitting the four spec
|
||||
fields alone would drop the snippet entirely, leaving the client (and
|
||||
the model, on any replayed follow-up turn) with URLs and titles but no
|
||||
evidence to answer from, forcing a fetch per result. So the snippet is
|
||||
carried in an additive ``snippet`` key alongside the spec fields.
|
||||
``encrypted_content`` stays empty rather than holding plaintext, which
|
||||
would assert encryption semantics that do not hold.
|
||||
|
||||
Spec reference:
|
||||
https://docs.anthropic.com/en/api/web-search-tool
|
||||
|
||||
|
|
@ -438,6 +447,7 @@ class WebSearchTransformation:
|
|||
"title": title,
|
||||
"page_age": page_age,
|
||||
"encrypted_content": "",
|
||||
"snippet": getattr(r, "snippet", "") or "",
|
||||
}
|
||||
)
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -4,9 +4,12 @@ This file contains common utils for anthropic calls.
|
|||
|
||||
import copy
|
||||
import re
|
||||
from typing import Any, Final
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, Literal
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -1057,6 +1060,152 @@ def sanitize_tool_use_ids_in_anthropic_messages(messages: list[Any]) -> list[Any
|
|||
return out
|
||||
|
||||
|
||||
class _ReplayedSearchQuery(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
query: str = ""
|
||||
|
||||
|
||||
class _ReplayedWebSearchResult(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
type: Literal["web_search_result"]
|
||||
url: str = ""
|
||||
title: str = ""
|
||||
snippet: str = ""
|
||||
encrypted_content: str = ""
|
||||
|
||||
|
||||
class _ReplayedWebSearchToolResult(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
type: Literal["web_search_tool_result"]
|
||||
tool_use_id: str
|
||||
content: tuple[_ReplayedWebSearchResult, ...]
|
||||
|
||||
|
||||
class _ReplayedServerToolUse(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
type: Literal["server_tool_use"]
|
||||
id: str
|
||||
input: _ReplayedSearchQuery = _ReplayedSearchQuery()
|
||||
|
||||
|
||||
class _TextBlock(BaseModel):
|
||||
type: Literal["text"] = "text"
|
||||
text: str
|
||||
|
||||
|
||||
_WEB_SEARCH_TOOL_RESULT_ADAPTER: Final = TypeAdapter(_ReplayedWebSearchToolResult)
|
||||
_SERVER_TOOL_USE_ADAPTER: Final = TypeAdapter(_ReplayedServerToolUse)
|
||||
|
||||
|
||||
def _flattenable_web_search_tool_result(block: object) -> _ReplayedWebSearchToolResult | None:
|
||||
"""
|
||||
The parsed block when it is a ``web_search_tool_result`` carrying no
|
||||
``encrypted_content``, else None for anything Anthropic itself issued.
|
||||
|
||||
An empty ``content`` list is flattenable too. It is what the interceptor emits
|
||||
when a search legitimately returns nothing and when a search raises, and it
|
||||
carries neither evidence to preserve nor an ``encrypted_content`` to respect,
|
||||
so leaving it in place only buys the 400 this whole function exists to avoid.
|
||||
"""
|
||||
try:
|
||||
parsed: Final = _WEB_SEARCH_TOOL_RESULT_ADAPTER.validate_python(block)
|
||||
except ValidationError:
|
||||
return None
|
||||
if any(result.encrypted_content for result in parsed.content):
|
||||
return None
|
||||
return parsed
|
||||
|
||||
|
||||
def _replayed_server_tool_use(block: object) -> _ReplayedServerToolUse | None:
|
||||
try:
|
||||
return _SERVER_TOOL_USE_ADAPTER.validate_python(block)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _render_web_search_results(query: str, results: tuple[_ReplayedWebSearchResult, ...]) -> str:
|
||||
header: Final = f"Web search results for '{query}':" if query else "Web search results:"
|
||||
if not results:
|
||||
return f"{header}\n\nNo results were returned."
|
||||
body: Final = "\n\n".join(
|
||||
"\n".join(
|
||||
line
|
||||
for line in (
|
||||
f"Title: {result.title}" if result.title else "",
|
||||
f"URL: {result.url}" if result.url else "",
|
||||
f"Snippet: {result.snippet}" if result.snippet else "",
|
||||
)
|
||||
if line
|
||||
)
|
||||
for result in results
|
||||
)
|
||||
return f"{header}\n\n{body}" if body else header
|
||||
|
||||
|
||||
def _rewrite_replayed_web_search_block(
|
||||
block: object,
|
||||
flattenable: Mapping[str, _ReplayedWebSearchToolResult],
|
||||
queries: Mapping[str, str],
|
||||
) -> object | None:
|
||||
parsed_result: Final = _flattenable_web_search_tool_result(block)
|
||||
if parsed_result is not None:
|
||||
return _TextBlock(
|
||||
text=_render_web_search_results(queries.get(parsed_result.tool_use_id, ""), parsed_result.content)
|
||||
).model_dump()
|
||||
parsed_use: Final = _replayed_server_tool_use(block)
|
||||
if parsed_use is not None and parsed_use.id in flattenable:
|
||||
return None
|
||||
return block
|
||||
|
||||
|
||||
def _flatten_web_search_results_in_message(message: object) -> object:
|
||||
if not isinstance(message, Mapping) or not isinstance(message.get("content"), Sequence):
|
||||
return message
|
||||
content: Final = message["content"]
|
||||
if isinstance(content, str):
|
||||
return message
|
||||
flattenable: Final = MappingProxyType(
|
||||
{
|
||||
parsed.tool_use_id: parsed
|
||||
for parsed in (_flattenable_web_search_tool_result(block) for block in content)
|
||||
if parsed is not None
|
||||
}
|
||||
)
|
||||
if not flattenable:
|
||||
return message
|
||||
queries: Final = MappingProxyType(
|
||||
{
|
||||
parsed.id: parsed.input.query
|
||||
for parsed in (_replayed_server_tool_use(block) for block in content)
|
||||
if parsed is not None
|
||||
}
|
||||
)
|
||||
rewritten: Final = tuple(_rewrite_replayed_web_search_block(block, flattenable, queries) for block in content)
|
||||
return {**message, "content": [b for b in rewritten if b is not None]} # mutable-ok: JSON wire format
|
||||
|
||||
|
||||
def flatten_unencrypted_web_search_results_in_anthropic_messages( # mutable-ok: as sibling sanitizers
|
||||
messages: list[Any],
|
||||
) -> list[Any]:
|
||||
"""
|
||||
Return a new message list with replayed ``web_search_tool_result`` blocks that
|
||||
carry no ``encrypted_content`` rewritten into plain ``text`` blocks holding the
|
||||
same title / url / snippet evidence.
|
||||
|
||||
``encrypted_content`` is an opaque blob only Anthropic's own search backend can
|
||||
mint, so blocks synthesized by LiteLLM (websearch interception against a search
|
||||
provider) are rejected with ``Invalid encrypted_content in search_result block``
|
||||
when a native client loops them back as history. Flattening them keeps the
|
||||
evidence in the conversation instead of 400ing the follow-up turn, and leaves
|
||||
genuine Anthropic-issued blocks untouched.
|
||||
"""
|
||||
return [_flatten_web_search_results_in_message(m) for m in messages] # mutable-ok: JSON wire format
|
||||
|
||||
|
||||
def process_anthropic_headers(headers: httpx.Headers | dict) -> dict:
|
||||
openai_headers: Final = {}
|
||||
if "anthropic-ratelimit-requests-limit" in headers:
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from typing import Any, Final, cast
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
flatten_unencrypted_web_search_results_in_anthropic_messages,
|
||||
sanitize_tool_use_ids_in_anthropic_messages,
|
||||
strip_empty_text_blocks_from_anthropic_messages,
|
||||
)
|
||||
|
|
@ -222,6 +223,7 @@ async def anthropic_messages(
|
|||
# Replay of cross-provider tool history (e.g. kimi -> Anthropic) may carry
|
||||
# ids like ``functions.Bash:0`` that violate Anthropic's id pattern.
|
||||
messages = sanitize_tool_use_ids_in_anthropic_messages(messages)
|
||||
messages = flatten_unencrypted_web_search_results_in_anthropic_messages(messages)
|
||||
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
AnthropicCacheControlHook,
|
||||
|
|
@ -413,6 +415,7 @@ def anthropic_messages_handler(
|
|||
if not kwargs.pop("_litellm_messages_presanitized", False):
|
||||
messages = strip_empty_text_blocks_from_anthropic_messages(messages)
|
||||
messages = sanitize_tool_use_ids_in_anthropic_messages(messages)
|
||||
messages = flatten_unencrypted_web_search_results_in_anthropic_messages(messages)
|
||||
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
AnthropicCacheControlHook,
|
||||
|
|
|
|||
|
|
@ -3894,12 +3894,19 @@ class OrganizationMemberUpdateResponse(MemberUpdateResponse):
|
|||
##########################################
|
||||
|
||||
|
||||
class TeamAccessGroupModelGrant(LiteLLMPydanticObjectBase):
|
||||
access_group_id: str
|
||||
access_group_name: str
|
||||
models: tuple[str, ...]
|
||||
|
||||
|
||||
class TeamInfoResponseObjectTeamTable(LiteLLM_TeamTable):
|
||||
team_member_budget_table: LiteLLM_BudgetTableFull | None = None
|
||||
# Resources inherited from access groups (separate from direct assignments)
|
||||
access_group_models: list[str] | None = None
|
||||
access_group_mcp_server_ids: list[str] | None = None
|
||||
access_group_agent_ids: list[str] | None = None
|
||||
access_group_details: tuple[TeamAccessGroupModelGrant, ...] | None = None
|
||||
|
||||
|
||||
class TeamInfoResponseObject(TypedDict):
|
||||
|
|
|
|||
|
|
@ -253,7 +253,7 @@ async def get_agents(
|
|||
)
|
||||
|
||||
try:
|
||||
returned_agents: list[AgentResponse] = []
|
||||
returned_agents: Sequence[AgentResponse] = ()
|
||||
|
||||
# Admin users get all agents
|
||||
if (
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import asyncio
|
|||
import math
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast
|
||||
|
||||
from fastapi import HTTPException, Request, status
|
||||
|
|
@ -2834,7 +2835,7 @@ async def get_org_object(
|
|||
|
||||
|
||||
async def _get_resources_from_access_groups(
|
||||
access_group_ids: list[str],
|
||||
access_group_ids: Sequence[str],
|
||||
resource_field: Literal["access_model_names", "access_mcp_server_ids", "access_agent_ids"],
|
||||
prisma_client: PrismaClient | None = None,
|
||||
user_api_key_cache: UserApiKeyCache | None = None,
|
||||
|
|
@ -2893,7 +2894,7 @@ async def _get_resources_from_access_groups(
|
|||
|
||||
|
||||
async def _get_models_from_access_groups(
|
||||
access_group_ids: list[str],
|
||||
access_group_ids: Sequence[str],
|
||||
prisma_client: PrismaClient | None = None,
|
||||
user_api_key_cache: UserApiKeyCache | None = None,
|
||||
proxy_logging_obj: ProxyLogging | None = None,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
# What is this?
|
||||
## Common checks for /v1/models and `/model/info`
|
||||
import copy
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Final
|
||||
|
||||
import litellm
|
||||
|
|
@ -178,8 +179,8 @@ def get_team_models(
|
|||
|
||||
|
||||
def get_complete_model_list(
|
||||
key_models: list[str],
|
||||
team_models: list[str],
|
||||
key_models: Sequence[str],
|
||||
team_models: Sequence[str],
|
||||
proxy_model_list: list[str],
|
||||
user_model: str | None,
|
||||
infer_model_from_keys: bool | None,
|
||||
|
|
@ -203,7 +204,7 @@ def get_complete_model_list(
|
|||
|
||||
def append_unique(models):
|
||||
for model in models:
|
||||
if model not in unique_models:
|
||||
if model not in unique_models and model != SpecialModelNames.no_default_models.value:
|
||||
unique_models.append(model)
|
||||
|
||||
if key_models:
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ class AutoRouterTurnTransaction:
|
|||
cache_hit: bool
|
||||
cache_ttl_seconds: int | None
|
||||
cache_touched: bool
|
||||
tier: str | None = None
|
||||
|
||||
|
||||
class TurnCacheFacts(NamedTuple):
|
||||
|
|
@ -152,11 +153,13 @@ def build_autorouter_turn_transaction(
|
|||
return None
|
||||
usage_object_raw: Final = metadata.get("usage_object")
|
||||
cache: Final = turn_cache_facts(usage_object_raw if isinstance(usage_object_raw, Mapping) else None)
|
||||
tier_raw: Final = routing_decision.get("tier")
|
||||
return AutoRouterTurnTransaction(
|
||||
api_key=api_key,
|
||||
session_id=_bounded_session_id(session_id),
|
||||
router_name=router_name,
|
||||
router_type=str(routing_decision.get("router_type") or "unknown"),
|
||||
tier=tier_raw if isinstance(tier_raw, str) and tier_raw else None,
|
||||
model=model,
|
||||
turn_at=turn_at,
|
||||
total_tokens=int(payload.get("prompt_tokens") or 0) + int(payload.get("completion_tokens") or 0),
|
||||
|
|
@ -184,6 +187,8 @@ _COVERED: Final = _p("covered")
|
|||
_CACHE_HIT: Final = _p("cache_hit")
|
||||
_CACHE_TTL: Final = _p("cache_ttl_seconds")
|
||||
_TOUCHED: Final = _p("cache_touched")
|
||||
_TIER: Final = f"{_p('tier')}::text"
|
||||
_TIER_DELTA: Final = f"(CASE WHEN {_TIER} IS NULL THEN '{{}}'::jsonb ELSE jsonb_build_object({_TIER}, 1) END)"
|
||||
|
||||
_IN_ORDER: Final = f"{_TURN_AT}::timestamp >= t.last_turn_at"
|
||||
_SAME: Final = f"{_IN_ORDER} AND t.last_model = {_MODEL}"
|
||||
|
|
@ -201,7 +206,7 @@ INSERT INTO "LiteLLM_AutoRouterSession" AS t (
|
|||
last_model, models, turns, unordered_turns, covered_turns, cache_hits,
|
||||
same_model_turns, same_model_hits, first_visit_turns, first_visit_hits,
|
||||
return_turns, return_hits, return_expired_misses, return_within_ttl_misses,
|
||||
ttl_5m_turns, ttl_1h_turns, total_tokens, spend, saved_spend
|
||||
ttl_5m_turns, ttl_1h_turns, total_tokens, spend, saved_spend, tier_turns
|
||||
)
|
||||
VALUES (
|
||||
{_p("api_key")}, {_p("session_id")}, {_p("router_name")}, {_p("router_type")}, {_TURN_AT}::timestamp, {_TURN_AT}::timestamp,
|
||||
|
|
@ -211,7 +216,8 @@ VALUES (
|
|||
0, 0, 0, 0,
|
||||
(CASE WHEN {_CACHE_TTL}::int = {CACHE_TTL_5M_SECONDS} THEN 1 ELSE 0 END),
|
||||
(CASE WHEN {_CACHE_TTL}::int = {CACHE_TTL_1H_SECONDS} THEN 1 ELSE 0 END),
|
||||
{_p("total_tokens")}::bigint, {_p("spend")}::float8, {_p("saved_spend")}::float8
|
||||
{_p("total_tokens")}::bigint, {_p("spend")}::float8, {_p("saved_spend")}::float8,
|
||||
{_TIER_DELTA}
|
||||
)
|
||||
ON CONFLICT (api_key, session_id, router_name) DO UPDATE SET
|
||||
turns = t.turns + 1,
|
||||
|
|
@ -242,6 +248,9 @@ ON CONFLICT (api_key, session_id, router_name) DO UPDATE SET
|
|||
ELSE COALESCE((t.models -> {_MODEL} ->> 'ttl')::int, {_CACHE_TTL}::int) END)
|
||||
)),
|
||||
last_model = (CASE WHEN {_IN_ORDER} THEN {_MODEL} ELSE t.last_model END),
|
||||
tier_turns = (CASE WHEN {_TIER} IS NOT NULL AND t.router_type = {_p("router_type")}
|
||||
THEN t.tier_turns || jsonb_build_object({_TIER}, COALESCE((t.tier_turns ->> {_TIER})::int, 0) + 1)
|
||||
ELSE t.tier_turns END),
|
||||
first_turn_at = LEAST(t.first_turn_at, EXCLUDED.first_turn_at),
|
||||
last_turn_at = GREATEST(t.last_turn_at, EXCLUDED.last_turn_at)
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -4,8 +4,9 @@ AUTO ROUTER MANAGEMENT ENDPOINTS
|
|||
POST /auto_router/test_routing - Route one prompt through an unsaved complexity-router config
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Annotated, Final
|
||||
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
|
@ -260,6 +261,7 @@ async def preview_auto_router_routing(
|
|||
class _SessionAggRow(BaseModel):
|
||||
router_name: str
|
||||
router_type: str
|
||||
tier_turns: Mapping[str, int]
|
||||
sessions: int
|
||||
turns: int
|
||||
unordered_turns: int
|
||||
|
|
@ -284,6 +286,23 @@ class _SessionAggRow(BaseModel):
|
|||
_SESSION_AGG_ROWS: Final = TypeAdapter(list[_SessionAggRow])
|
||||
|
||||
_BENCHMARKS_SQL: Final = """
|
||||
WITH windowed AS (
|
||||
SELECT * FROM "LiteLLM_AutoRouterSession"
|
||||
WHERE last_turn_at >= $1::timestamp AND first_turn_at < $2::timestamp
|
||||
),
|
||||
tier_maps AS (
|
||||
SELECT router_name, router_type, jsonb_object_agg(tier, tier_turns) AS tier_turns
|
||||
FROM (
|
||||
SELECT router_name, router_type, kv.key AS tier, SUM((kv.value)::int)::int AS tier_turns
|
||||
FROM windowed, LATERAL jsonb_each_text(tier_turns) AS kv
|
||||
GROUP BY router_name, router_type, kv.key
|
||||
) per_tier
|
||||
GROUP BY router_name, router_type
|
||||
)
|
||||
SELECT
|
||||
agg.*,
|
||||
COALESCE(tier_maps.tier_turns, '{}'::jsonb) AS tier_turns
|
||||
FROM (
|
||||
SELECT
|
||||
router_name,
|
||||
router_type,
|
||||
|
|
@ -306,10 +325,11 @@ SELECT
|
|||
COALESCE(SUM(spend), 0)::float8 AS spend,
|
||||
COALESCE(SUM(saved_spend), 0)::float8 AS saved_spend,
|
||||
COALESCE(SUM(EXTRACT(EPOCH FROM (last_turn_at - first_turn_at))), 0)::float8 AS session_seconds
|
||||
FROM "LiteLLM_AutoRouterSession"
|
||||
WHERE last_turn_at >= $1::timestamp AND first_turn_at < $2::timestamp
|
||||
FROM windowed
|
||||
GROUP BY router_name, router_type
|
||||
ORDER BY SUM(spend) DESC
|
||||
) agg
|
||||
LEFT JOIN tier_maps USING (router_name, router_type)
|
||||
ORDER BY agg.spend DESC
|
||||
"""
|
||||
|
||||
|
||||
|
|
@ -366,6 +386,7 @@ def _summed_agg_row(rows: Sequence[_SessionAggRow]) -> _SessionAggRow:
|
|||
return _SessionAggRow(
|
||||
router_name="",
|
||||
router_type="",
|
||||
tier_turns=MappingProxyType({}),
|
||||
sessions=sum(row.sessions for row in rows),
|
||||
turns=sum(row.turns for row in rows),
|
||||
unordered_turns=sum(row.unordered_turns for row in rows),
|
||||
|
|
@ -443,6 +464,7 @@ async def get_auto_router_benchmarks(
|
|||
AutoRouterBenchmarkGroup(
|
||||
router_name=row.router_name,
|
||||
router_type=row.router_type,
|
||||
tier_turns=row.tier_turns,
|
||||
**_benchmark_totals(row).model_dump(),
|
||||
)
|
||||
for row in rows
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ from litellm.proxy._types import (
|
|||
SpecialManagementEndpointEnums,
|
||||
SpecialModelNames,
|
||||
SpecialProxyStrings,
|
||||
TeamAccessGroupModelGrant,
|
||||
TeamAddMemberResponse,
|
||||
TeamInfoResponseObject,
|
||||
TeamInfoResponseObjectTeamTable,
|
||||
|
|
@ -3829,7 +3830,7 @@ async def _add_team_member_budget_table(
|
|||
) -> TeamInfoResponseObjectTeamTable:
|
||||
try:
|
||||
team_budget: Final = await _budget_db(prisma_client).find_unique(where={"budget_id": team_member_budget_id})
|
||||
team_info_response_object.team_member_budget_table = team_budget
|
||||
return team_info_response_object.model_copy(update={"team_member_budget_table": team_budget})
|
||||
except Exception:
|
||||
verbose_proxy_logger.info(
|
||||
"Team member budget table not found, passed team_member_budget_id=%s", team_member_budget_id
|
||||
|
|
@ -3838,21 +3839,34 @@ async def _add_team_member_budget_table(
|
|||
return team_info_response_object
|
||||
|
||||
|
||||
async def _resolve_team_access_group_resources(_team_info: TeamInfoResponseObjectTeamTable) -> None:
|
||||
"""Populate access_group_models / mcp_server_ids / agent_ids on the team
|
||||
info response by resolving inherited resources from its access groups."""
|
||||
async def _resolve_team_access_group_resources(
|
||||
_team_info: TeamInfoResponseObjectTeamTable,
|
||||
) -> TeamInfoResponseObjectTeamTable:
|
||||
"""Return a copy of the team info with access_group_models / mcp_server_ids /
|
||||
agent_ids / details resolved from its access groups."""
|
||||
if not _team_info.access_group_ids:
|
||||
return
|
||||
return _team_info
|
||||
ag_lookup: Final = await _batch_resolve_access_group_resources(_team_info.access_group_ids)
|
||||
models, mcp_ids, agent_ids = set(), set(), set()
|
||||
for ag_id in _team_info.access_group_ids:
|
||||
if ag_id in ag_lookup:
|
||||
models.update(ag_lookup[ag_id]["models"])
|
||||
mcp_ids.update(ag_lookup[ag_id]["mcp_server_ids"])
|
||||
agent_ids.update(ag_lookup[ag_id]["agent_ids"])
|
||||
_team_info.access_group_models = list(models)
|
||||
_team_info.access_group_mcp_server_ids = list(mcp_ids)
|
||||
_team_info.access_group_agent_ids = list(agent_ids)
|
||||
resolved_groups: Final = tuple(
|
||||
ag_lookup[ag_id] for ag_id in dict.fromkeys(_team_info.access_group_ids) if ag_id in ag_lookup
|
||||
)
|
||||
return _team_info.model_copy(
|
||||
update={
|
||||
"access_group_models": list({m for group in resolved_groups for m in (group.access_model_names or [])}),
|
||||
"access_group_mcp_server_ids": list(
|
||||
{s for group in resolved_groups for s in (group.access_mcp_server_ids or [])}
|
||||
),
|
||||
"access_group_agent_ids": list({a for group in resolved_groups for a in (group.access_agent_ids or [])}),
|
||||
"access_group_details": tuple(
|
||||
TeamAccessGroupModelGrant(
|
||||
access_group_id=group.access_group_id,
|
||||
access_group_name=group.access_group_name,
|
||||
models=tuple(group.access_model_names or ()),
|
||||
)
|
||||
for group in resolved_groups
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/team/info", tags=["team management"], dependencies=[Depends(user_api_key_auth)])
|
||||
|
|
@ -3958,11 +3972,11 @@ async def team_info(
|
|||
)
|
||||
|
||||
# Resolve resources inherited from access groups
|
||||
await _resolve_team_access_group_resources(_team_info)
|
||||
resolved_team_info: Final = await _resolve_team_access_group_resources(_team_info)
|
||||
|
||||
response_object: Final = TeamInfoResponseObject(
|
||||
team_id=team_id,
|
||||
team_info=_team_info,
|
||||
team_info=resolved_team_info,
|
||||
keys=keys,
|
||||
team_memberships=returned_tm,
|
||||
)
|
||||
|
|
@ -4391,32 +4405,21 @@ async def _build_team_list_where_conditions(
|
|||
|
||||
async def _batch_resolve_access_group_resources(
|
||||
all_access_group_ids: list[str],
|
||||
) -> dict[str, dict[str, list[str]]]:
|
||||
) -> dict[str, LiteLLM_AccessGroupTable]:
|
||||
"""
|
||||
Batch-fetch access groups in a single DB query and return a per-group
|
||||
resource map.
|
||||
|
||||
Returns {ag_id: {"models": [...], "mcp_server_ids": [...], "agent_ids": [...]}}.
|
||||
Missing/invalid groups are silently omitted.
|
||||
Batch-fetch access groups in a single DB query and return them keyed by
|
||||
access_group_id. Missing/invalid groups are silently omitted.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import prisma_client as _prisma_client
|
||||
|
||||
if not all_access_group_ids or _prisma_client is None:
|
||||
return {}
|
||||
|
||||
unique_ids: Final = list(set(all_access_group_ids))
|
||||
unique_ids: Final = tuple(frozenset(all_access_group_ids))
|
||||
rows: Final = await _access_group_db(_prisma_client).find_many(
|
||||
where={"access_group_id": {"in": unique_ids}},
|
||||
)
|
||||
|
||||
result: Final[dict[str, dict[str, list[str]]]] = {}
|
||||
for row in rows:
|
||||
result[row.access_group_id] = {
|
||||
"models": list(row.access_model_names or []),
|
||||
"mcp_server_ids": list(row.access_mcp_server_ids or []),
|
||||
"agent_ids": list(row.access_agent_ids or []),
|
||||
}
|
||||
return result
|
||||
return {row.access_group_id: row for row in rows}
|
||||
|
||||
|
||||
def _convert_teams_to_response_models(
|
||||
|
|
@ -4710,15 +4713,18 @@ async def list_team_v2(
|
|||
all_ag_ids: Final = [ag_id for t in team_items_with_ag for ag_id in (t.access_group_ids or [])]
|
||||
ag_lookup: Final = await _batch_resolve_access_group_resources(all_ag_ids)
|
||||
for team_item in team_items_with_ag:
|
||||
models, mcp_ids, agent_ids = set(), set(), set()
|
||||
for ag_id in team_item.access_group_ids or []:
|
||||
if ag_id in ag_lookup:
|
||||
models.update(ag_lookup[ag_id]["models"])
|
||||
mcp_ids.update(ag_lookup[ag_id]["mcp_server_ids"])
|
||||
agent_ids.update(ag_lookup[ag_id]["agent_ids"])
|
||||
team_item.access_group_models = list(models)
|
||||
team_item.access_group_mcp_server_ids = list(mcp_ids)
|
||||
team_item.access_group_agent_ids = list(agent_ids)
|
||||
team_groups = tuple(
|
||||
ag_lookup[ag_id] for ag_id in (team_item.access_group_ids or []) if ag_id in ag_lookup
|
||||
)
|
||||
team_item.access_group_models = list(
|
||||
{m for group in team_groups for m in (group.access_model_names or [])}
|
||||
)
|
||||
team_item.access_group_mcp_server_ids = list(
|
||||
{s for group in team_groups for s in (group.access_mcp_server_ids or [])}
|
||||
)
|
||||
team_item.access_group_agent_ids = list(
|
||||
{a for group in team_groups for a in (group.access_agent_ids or [])}
|
||||
)
|
||||
|
||||
return {
|
||||
"teams": team_list,
|
||||
|
|
|
|||
|
|
@ -1439,6 +1439,7 @@ model LiteLLM_AutoRouterSession {
|
|||
total_tokens BigInt @default(0)
|
||||
spend Float @default(0)
|
||||
saved_spend Float @default(0)
|
||||
tier_turns Json @default("{}")
|
||||
|
||||
@@id([api_key, session_id, router_name])
|
||||
@@index([last_turn_at], map: "idx_autorouter_session_last_turn")
|
||||
|
|
|
|||
|
|
@ -165,6 +165,7 @@ if TYPE_CHECKING:
|
|||
from prisma.client import TransactionManager
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.models.team import LiteLLM_TeamTableCachedObj
|
||||
from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction
|
||||
from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction
|
||||
|
||||
|
|
@ -6419,6 +6420,74 @@ def construct_database_url_from_env_vars() -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
async def _get_validated_team_object(
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
team_id: str,
|
||||
prisma_client: "PrismaClient",
|
||||
user_api_key_cache: "UserApiKeyCache",
|
||||
proxy_logging_obj: "ProxyLogging",
|
||||
) -> "LiteLLM_TeamTableCachedObj":
|
||||
from litellm.proxy.auth.auth_checks import get_team_object
|
||||
from litellm.proxy.management_endpoints.team_endpoints import validate_membership
|
||||
|
||||
team_object: Final = await get_team_object(
|
||||
team_id=team_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
await validate_membership(user_api_key_dict=user_api_key_dict, team_table=team_object)
|
||||
return team_object
|
||||
|
||||
|
||||
async def _get_team_object_for_access_groups(
|
||||
team_id: str | None,
|
||||
prisma_client: Optional["PrismaClient"],
|
||||
user_api_key_cache: Optional["UserApiKeyCache"],
|
||||
proxy_logging_obj: Optional["ProxyLogging"],
|
||||
) -> Optional["LiteLLM_TeamTableCachedObj"]:
|
||||
from litellm.proxy.auth.auth_checks import get_team_object
|
||||
|
||||
if team_id is None or prisma_client is None or user_api_key_cache is None or proxy_logging_obj is None:
|
||||
return None
|
||||
try:
|
||||
return await get_team_object(
|
||||
team_id=team_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
except HTTPException:
|
||||
verbose_proxy_logger.debug("Could not fetch team %s while listing models", team_id)
|
||||
return None
|
||||
|
||||
|
||||
async def _get_access_group_models(
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
team_object: Optional["LiteLLM_TeamTableCachedObj"],
|
||||
prisma_client: Optional["PrismaClient"],
|
||||
user_api_key_cache: Optional["UserApiKeyCache"],
|
||||
proxy_logging_obj: Optional["ProxyLogging"],
|
||||
) -> tuple[str, ...]:
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
_get_models_from_access_groups,
|
||||
get_authorized_resources_from_key_access_groups,
|
||||
)
|
||||
|
||||
team_group_models: Final = await _get_models_from_access_groups(
|
||||
access_group_ids=(team_object.access_group_ids or ()) if team_object is not None else (),
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
key_group_models: Final = await get_authorized_resources_from_key_access_groups(
|
||||
valid_token=user_api_key_dict,
|
||||
team_object=team_object,
|
||||
resource_field="access_model_names",
|
||||
)
|
||||
return tuple(dict.fromkeys((*team_group_models, *key_group_models)))
|
||||
|
||||
|
||||
async def get_available_models_for_user(
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
llm_router: Optional["Router"],
|
||||
|
|
@ -6450,13 +6519,11 @@ async def get_available_models_for_user(
|
|||
Returns:
|
||||
List of model names available to the user
|
||||
"""
|
||||
from litellm.proxy.auth.auth_checks import get_team_object
|
||||
from litellm.proxy.auth.model_checks import (
|
||||
get_complete_model_list,
|
||||
get_key_models,
|
||||
get_team_models,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.team_endpoints import validate_membership
|
||||
|
||||
# Get proxy model list and access groups
|
||||
if llm_router is None:
|
||||
|
|
@ -6466,31 +6533,33 @@ async def get_available_models_for_user(
|
|||
proxy_model_list = llm_router.get_model_names()
|
||||
model_access_groups = llm_router.get_model_access_groups()
|
||||
|
||||
# Get key models
|
||||
key_models = get_key_models(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_model_list=proxy_model_list,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=include_model_access_groups,
|
||||
)
|
||||
|
||||
# Get team models
|
||||
team_models: list[str] = user_api_key_dict.team_models
|
||||
|
||||
# If specific team_id is provided, validate and get team models
|
||||
if team_id and prisma_client and proxy_logging_obj and user_api_key_cache:
|
||||
key_models = []
|
||||
team_object: Final = await get_team_object(
|
||||
requested_team_object: Final = (
|
||||
await _get_validated_team_object(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
team_id=team_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
await validate_membership(user_api_key_dict=user_api_key_dict, team_table=team_object)
|
||||
team_models = team_object.models
|
||||
if team_id and prisma_client and proxy_logging_obj and user_api_key_cache
|
||||
else None
|
||||
)
|
||||
|
||||
team_models = get_team_models(
|
||||
team_models=team_models,
|
||||
key_models: Final[Sequence[str]] = (
|
||||
()
|
||||
if requested_team_object is not None
|
||||
else get_key_models(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_model_list=proxy_model_list,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=include_model_access_groups,
|
||||
)
|
||||
)
|
||||
|
||||
team_models: Final = get_team_models(
|
||||
team_models=(
|
||||
requested_team_object.models if requested_team_object is not None else user_api_key_dict.team_models
|
||||
),
|
||||
proxy_model_list=proxy_model_list,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=include_model_access_groups,
|
||||
|
|
@ -6498,10 +6567,31 @@ async def get_available_models_for_user(
|
|||
|
||||
effective_team_id: Final = team_id or user_api_key_dict.team_id
|
||||
|
||||
access_group_models: Final = (
|
||||
await _get_access_group_models(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
team_object=requested_team_object
|
||||
or await _get_team_object_for_access_groups(
|
||||
team_id=effective_team_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
),
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
if key_models or team_models
|
||||
else ()
|
||||
)
|
||||
|
||||
granted_key_models: Final = (*key_models, *access_group_models) if key_models else key_models
|
||||
granted_team_models: Final = (*team_models, *access_group_models) if team_models else team_models
|
||||
|
||||
# Get complete model list
|
||||
all_models: Final = get_complete_model_list(
|
||||
key_models=key_models,
|
||||
team_models=team_models,
|
||||
key_models=granted_key_models,
|
||||
team_models=granted_team_models,
|
||||
proxy_model_list=proxy_model_list,
|
||||
user_model=user_model,
|
||||
infer_model_from_keys=general_settings.get("infer_model_from_keys", False),
|
||||
|
|
|
|||
|
|
@ -1730,6 +1730,7 @@ class ComplexityRouter(CustomLogger):
|
|||
routing_decision=self._build_routing_decision(
|
||||
routed_model=routed_model,
|
||||
cause=cause,
|
||||
tier=self._tier_for_model(routed_model),
|
||||
escalation_keyword=pin_escalation_keyword,
|
||||
escalated=escalated,
|
||||
conversation_continuing=conversation_continuing,
|
||||
|
|
@ -1797,7 +1798,8 @@ class ComplexityRouter(CustomLogger):
|
|||
|
||||
if user_message is None:
|
||||
verbose_router_logger.debug("ComplexityRouter: No user message found, routing to default model")
|
||||
if not self.config.plugins and self.config.default_model:
|
||||
default_model_first: Final = not self.config.plugins and self.config.default_model
|
||||
if default_model_first:
|
||||
# No plugins configured: preserve the pre-existing default_model-first
|
||||
# priority exactly (changing it would be a silent behavior change for
|
||||
# every non-plugin user, not just a security fix).
|
||||
|
|
@ -1809,12 +1811,14 @@ class ComplexityRouter(CustomLogger):
|
|||
routed_model = await self._pick_model_for_tier(
|
||||
ComplexityTier.MEDIUM, messages, resolved_messages, request_kwargs
|
||||
)
|
||||
fallback_tier: Final = None if default_model_first else ComplexityTier.MEDIUM
|
||||
return PreRoutingHookResponse(
|
||||
model=routed_model,
|
||||
messages=messages if has_original_messages else None,
|
||||
routing_decision=self._build_routing_decision(
|
||||
routed_model=routed_model,
|
||||
cause="default_fallback",
|
||||
tier=fallback_tier,
|
||||
conversation_continuing=conversation_continuing,
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,28 @@
|
|||
Type definitions for WebSearch Interception integration.
|
||||
"""
|
||||
|
||||
from typing import TypedDict
|
||||
from typing import Literal, TypedDict
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class AnthropicSearchQuery(BaseModel):
|
||||
"""``input`` of an Anthropic ``server_tool_use`` block for a web search."""
|
||||
|
||||
query: str
|
||||
|
||||
|
||||
class AnthropicServerToolUseBlock(BaseModel):
|
||||
"""
|
||||
The ``server_tool_use`` block that must accompany a ``web_search_tool_result``.
|
||||
|
||||
Anthropic requires the pair, with a ``srvtoolu_``-prefixed id shared by both.
|
||||
"""
|
||||
|
||||
type: Literal["server_tool_use"] = "server_tool_use"
|
||||
id: str
|
||||
name: Literal["web_search"] = "web_search"
|
||||
input: AnthropicSearchQuery
|
||||
|
||||
|
||||
class WebSearchInterceptionConfig(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
Types for auto-router management endpoints
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
|
@ -120,6 +121,16 @@ class AutoRouterBenchmarkGroup(AutoRouterBenchmarkTotals):
|
|||
|
||||
router_name: str = Field(description="The auto-router alias requests were sent to")
|
||||
router_type: str = Field(description="complexity, adaptive or quality")
|
||||
tier_turns: Mapping[str, int] = Field(
|
||||
default_factory=dict,
|
||||
description="Turns per tier, keyed by the tier name the routing decision recorded at "
|
||||
"request time (never re-derived at read time, since the tier-to-model mapping is "
|
||||
"mutable config). Tier names are scoped to this group's router_type and are not "
|
||||
"comparable across types: a complexity router reports 'simple'/'medium'/'complex'/"
|
||||
"'reasoning', a quality router reports its numeric quality tier, and an adaptive router "
|
||||
"records no tier at all. Turns no tier served (the classifier fell back to default_model) "
|
||||
"are absent rather than pooled under a sentinel key, so the values may sum to less than turns",
|
||||
)
|
||||
|
||||
|
||||
class AutoRouterBenchmarksResponse(BaseModel):
|
||||
|
|
|
|||
|
|
@ -3,6 +3,11 @@ id = "GHSA-fwg2-594c-jp42"
|
|||
ignoreUntil = 2026-08-12
|
||||
reason = "pypdf 6.15.0 (the fix) published 2026-08-06 and is still inside the P3D exclude-newer window, so uv cannot lock it yet; bump and drop this entry from 2026-08-09"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-fp3f-mc75-235c"
|
||||
ignoreUntil = 2026-08-12
|
||||
reason = "second pypdf advisory with the same 6.15.0 fix, published 2026-08-07 after the first; drop alongside GHSA-fwg2-594c-jp42 in the same bump"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-w8v5-vhqr-4h9v"
|
||||
ignoreUntil = 2026-09-09
|
||||
|
|
|
|||
|
|
@ -1439,6 +1439,7 @@ model LiteLLM_AutoRouterSession {
|
|||
total_tokens BigInt @default(0)
|
||||
spend Float @default(0)
|
||||
saved_spend Float @default(0)
|
||||
tier_turns Json @default("{}")
|
||||
|
||||
@@id([api_key, session_id, router_name])
|
||||
@@index([last_turn_at], map: "idx_autorouter_session_last_turn")
|
||||
|
|
|
|||
|
|
@ -38,11 +38,13 @@ async def _turn(
|
|||
tokens: int = 100,
|
||||
spend: float = 0.01,
|
||||
saved: float = 0.02,
|
||||
tier: "str | None" = None,
|
||||
) -> None:
|
||||
touched: Final = 1 if (hit or ttl is not None or not covered) else 0
|
||||
await db.execute_raw(
|
||||
UPSERT_AUTOROUTER_SESSION_SQL,
|
||||
key, session_id, router, router_type, model, at.isoformat(), tokens, spend, saved, covered, hit, ttl, touched,
|
||||
tier,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -195,6 +197,106 @@ async def test_a_reconfigured_alias_reports_each_router_type_as_its_own_group(db
|
|||
assert [(row["router_type"], row["sessions"]) for row in matching] == [("complexity", 1), ("quality", 1)]
|
||||
|
||||
|
||||
async def test_tier_turns_count_each_tier_that_served_a_turn(db):
|
||||
key = f"k-{uuid.uuid4()}"
|
||||
await _turn(db, key, "A", T0, tier="simple")
|
||||
await _turn(db, key, "B", T0 + timedelta(seconds=10), tier="complex")
|
||||
await _turn(db, key, "A", T0 + timedelta(seconds=20), tier="simple")
|
||||
|
||||
assert (await _row(db, key))["tier_turns"] == {"simple": 2, "complex": 1}
|
||||
|
||||
|
||||
async def test_an_untiered_turn_increments_no_tier_counter(db):
|
||||
key = f"k-{uuid.uuid4()}"
|
||||
await _turn(db, key, "A", T0, tier=None)
|
||||
assert (await _row(db, key))["tier_turns"] == {}
|
||||
|
||||
await _turn(db, key, "A", T0 + timedelta(seconds=10), tier="medium")
|
||||
await _turn(db, key, "A", T0 + timedelta(seconds=20), tier=None)
|
||||
row = await _row(db, key)
|
||||
assert row["tier_turns"] == {"medium": 1}
|
||||
assert row["turns"] == 3
|
||||
|
||||
|
||||
async def test_a_mid_session_router_type_change_keeps_foreign_tier_names_out_of_the_map(db):
|
||||
key = f"k-{uuid.uuid4()}"
|
||||
await _turn(db, key, "A", T0, router_type="complexity", tier="medium")
|
||||
await _turn(db, key, "A", T0 + timedelta(seconds=10), router_type="quality", tier="2")
|
||||
await _turn(db, key, "A", T0 + timedelta(seconds=20), router_type="complexity", tier="medium")
|
||||
|
||||
row = await _row(db, key)
|
||||
assert row["tier_turns"] == {"medium": 2}
|
||||
assert row["turns"] == 3
|
||||
|
||||
|
||||
async def test_an_out_of_order_turn_still_counts_toward_its_tier(db):
|
||||
key = f"k-{uuid.uuid4()}"
|
||||
await _turn(db, key, "A", T0 + timedelta(seconds=60), tier="simple")
|
||||
await _turn(db, key, "A", T0, tier="simple")
|
||||
|
||||
row = await _row(db, key)
|
||||
assert row["tier_turns"] == {"simple": 2}
|
||||
assert row["unordered_turns"] == 1
|
||||
|
||||
|
||||
async def test_the_benchmarks_aggregate_sums_tier_turns_across_sessions(db):
|
||||
key = f"k-{uuid.uuid4()}"
|
||||
router = f"r-{uuid.uuid4()}"
|
||||
await _turn(db, key, "A", T0, session_id=f"s-{uuid.uuid4()}", router=router, tier="simple")
|
||||
await _turn(db, key, "A", T0 + timedelta(seconds=10), session_id=f"s-{uuid.uuid4()}", router=router, tier="simple")
|
||||
await _turn(db, key, "B", T0 + timedelta(seconds=20), session_id=f"s-{uuid.uuid4()}", router=router, tier="complex")
|
||||
await _turn(db, key, "C", T0 + timedelta(seconds=30), session_id=f"s-{uuid.uuid4()}", router=router, tier=None)
|
||||
|
||||
rows = await db.query_raw(
|
||||
_BENCHMARKS_SQL,
|
||||
(T0 - timedelta(days=1)).isoformat(),
|
||||
(T0 + timedelta(days=1)).isoformat(),
|
||||
)
|
||||
grouped = next(row for row in rows if row["router_name"] == router)
|
||||
assert grouped["tier_turns"] == {"simple": 2, "complex": 1}
|
||||
assert grouped["turns"] == 4
|
||||
|
||||
|
||||
async def test_tier_maps_stay_separate_per_router_type_on_a_reconfigured_alias(db):
|
||||
key = f"k-{uuid.uuid4()}"
|
||||
router = f"r-{uuid.uuid4()}"
|
||||
await _turn(
|
||||
db, key, "A", T0, session_id=f"s-{uuid.uuid4()}", router=router, router_type="complexity", tier="medium"
|
||||
)
|
||||
await _turn(
|
||||
db,
|
||||
key,
|
||||
"A",
|
||||
T0 + timedelta(seconds=10),
|
||||
session_id=f"s-{uuid.uuid4()}",
|
||||
router=router,
|
||||
router_type="quality",
|
||||
tier="2",
|
||||
)
|
||||
|
||||
rows = await db.query_raw(
|
||||
_BENCHMARKS_SQL,
|
||||
(T0 - timedelta(days=1)).isoformat(),
|
||||
(T0 + timedelta(days=1)).isoformat(),
|
||||
)
|
||||
by_type = {row["router_type"]: row["tier_turns"] for row in rows if row["router_name"] == router}
|
||||
assert by_type == {"complexity": {"medium": 1}, "quality": {"2": 1}}
|
||||
|
||||
|
||||
async def test_a_window_with_no_tiered_turns_aggregates_to_an_empty_map(db):
|
||||
key = f"k-{uuid.uuid4()}"
|
||||
router = f"r-{uuid.uuid4()}"
|
||||
await _turn(db, key, "A", T0, session_id=f"s-{uuid.uuid4()}", router=router, tier=None)
|
||||
|
||||
rows = await db.query_raw(
|
||||
_BENCHMARKS_SQL,
|
||||
(T0 - timedelta(days=1)).isoformat(),
|
||||
(T0 + timedelta(days=1)).isoformat(),
|
||||
)
|
||||
grouped = next(row for row in rows if row["router_name"] == router)
|
||||
assert grouped["tier_turns"] == {}
|
||||
|
||||
|
||||
async def test_a_miss_that_touched_no_cache_does_not_advance_the_ttl_clock(db):
|
||||
key = f"k-{uuid.uuid4()}"
|
||||
await _turn(db, key, "A", T0, ttl=300)
|
||||
|
|
|
|||
|
|
@ -134,6 +134,30 @@ class TestBuildWebSearchToolResultBlock:
|
|||
assert first["title"] == "LiteLLM Docs"
|
||||
assert first["page_age"] == "2025-01-15"
|
||||
assert first["encrypted_content"] == ""
|
||||
assert first["snippet"] == "Unified interface for LLMs."
|
||||
|
||||
def test_snippet_carried_for_every_result(self):
|
||||
# The snippet is the only field carrying page text. Losing it leaves the
|
||||
# client and the model with nothing to answer from, forcing a fetch per
|
||||
# result.
|
||||
block = WebSearchTransformation.build_web_search_tool_result_block(
|
||||
tool_use_id="toolu_abc",
|
||||
search_response=_make_search_response(),
|
||||
)
|
||||
assert [r["snippet"] for r in block["content"]] == [
|
||||
"Unified interface for LLMs.",
|
||||
"Pay-per-use pricing model.",
|
||||
]
|
||||
|
||||
def test_missing_snippet_degrades_to_empty_string(self):
|
||||
response = SearchResponse(
|
||||
results=[SearchResult(title="T", url="https://x/", snippet="")]
|
||||
)
|
||||
block = WebSearchTransformation.build_web_search_tool_result_block(
|
||||
tool_use_id="toolu_abc",
|
||||
search_response=response,
|
||||
)
|
||||
assert block["content"][0]["snippet"] == ""
|
||||
|
||||
def test_handles_none_search_response(self):
|
||||
block = WebSearchTransformation.build_web_search_tool_result_block(
|
||||
|
|
@ -223,11 +247,15 @@ class TestBuildPlanAttachesBlocks:
|
|||
)
|
||||
|
||||
blocks = plan.metadata.get(WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY)
|
||||
assert isinstance(blocks, list)
|
||||
assert len(blocks) == 1
|
||||
assert blocks[0]["type"] == "web_search_tool_result"
|
||||
assert blocks[0]["tool_use_id"] == "toolu_one"
|
||||
assert blocks[0]["content"][0]["url"] == "https://docs.litellm.ai/"
|
||||
assert isinstance(blocks, tuple)
|
||||
assert [b["type"] for b in blocks] == [
|
||||
"server_tool_use",
|
||||
"web_search_tool_result",
|
||||
]
|
||||
assert blocks[0]["id"].startswith("srvtoolu_")
|
||||
assert blocks[0]["input"] == {"query": "what is litellm"}
|
||||
assert blocks[1]["tool_use_id"] == blocks[0]["id"]
|
||||
assert blocks[1]["content"][0]["url"] == "https://docs.litellm.ai/"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_does_not_carry_blocks_when_flag_absent(self):
|
||||
|
|
@ -479,6 +507,9 @@ class TestLegacyPathMatchesNewPath:
|
|||
kwargs={WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True},
|
||||
)
|
||||
|
||||
assert out["content"][0]["type"] == "web_search_tool_result"
|
||||
assert out["content"][0]["tool_use_id"] == "toolu_legacy"
|
||||
assert out["content"][1]["type"] == "text"
|
||||
assert [b["type"] for b in out["content"]] == [
|
||||
"server_tool_use",
|
||||
"web_search_tool_result",
|
||||
"text",
|
||||
]
|
||||
assert out["content"][1]["tool_use_id"] == out["content"][0]["id"]
|
||||
|
|
|
|||
|
|
@ -732,6 +732,61 @@ def test_handler_skips_strip_when_presanitized():
|
|||
assert result is not None
|
||||
|
||||
|
||||
def test_handler_flattens_replayed_unencrypted_web_search_results():
|
||||
"""Synthesized search blocks replayed as history must reach the provider as text."""
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages import handler
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_base_handler(*args, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return "stub"
|
||||
|
||||
with patch.object(
|
||||
handler.base_llm_http_handler,
|
||||
"anthropic_messages_handler",
|
||||
side_effect=fake_base_handler,
|
||||
):
|
||||
handler.anthropic_messages_handler(
|
||||
max_tokens=10,
|
||||
messages=[
|
||||
{"role": "user", "content": "latest litellm version?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_1",
|
||||
"name": "web_search",
|
||||
"input": {"query": "latest litellm version"},
|
||||
},
|
||||
{
|
||||
"type": "web_search_tool_result",
|
||||
"tool_use_id": "srvtoolu_1",
|
||||
"content": [
|
||||
{
|
||||
"type": "web_search_result",
|
||||
"url": "https://github.com/BerriAI/litellm/releases",
|
||||
"title": "Releases",
|
||||
"page_age": None,
|
||||
"encrypted_content": "",
|
||||
"snippet": "Latest release v1.95.0",
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "which version?"},
|
||||
],
|
||||
model="anthropic/claude-3-5-sonnet-20241022",
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
replayed = captured["messages"][1]["content"]
|
||||
assert [b["type"] for b in replayed] == ["text"]
|
||||
assert "Snippet: Latest release v1.95.0" in replayed[0]["text"]
|
||||
|
||||
|
||||
def test_presanitized_flag_not_leaked_to_provider_params():
|
||||
"""The private sentinel must be popped, never forwarded as a request param."""
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages import handler
|
||||
|
|
|
|||
|
|
@ -10,8 +10,10 @@ Verifies that:
|
|||
- ANTHROPIC_API_KEY / ANTHROPIC_API_BASE take precedence over their aliases.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -1457,6 +1459,190 @@ class TestAnthropicThinkingSignatureSelfHeal:
|
|||
out = strip_empty_text_blocks_from_anthropic_messages(msgs)
|
||||
assert [b["type"] for b in out[0]["content"]] == ["tool_result"]
|
||||
|
||||
def test_flatten_unencrypted_web_search_results_keeps_snippet_evidence(self):
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
flatten_unencrypted_web_search_results_in_anthropic_messages,
|
||||
)
|
||||
|
||||
msgs = [
|
||||
{"role": "user", "content": "latest litellm version?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_1",
|
||||
"name": "web_search",
|
||||
"input": {"query": "latest litellm version"},
|
||||
},
|
||||
{
|
||||
"type": "web_search_tool_result",
|
||||
"tool_use_id": "srvtoolu_1",
|
||||
"content": [
|
||||
{
|
||||
"type": "web_search_result",
|
||||
"url": "https://github.com/BerriAI/litellm/releases",
|
||||
"title": "Releases",
|
||||
"page_age": None,
|
||||
"encrypted_content": "",
|
||||
"snippet": "Latest release v1.95.0",
|
||||
}
|
||||
],
|
||||
},
|
||||
{"type": "text", "text": "v1.95.0"},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
out = flatten_unencrypted_web_search_results_in_anthropic_messages(msgs)
|
||||
|
||||
assert out[0] is msgs[0]
|
||||
assert [b["type"] for b in out[1]["content"]] == ["text", "text"]
|
||||
flattened = out[1]["content"][0]["text"]
|
||||
assert "Web search results for 'latest litellm version':" in flattened
|
||||
assert "URL: https://github.com/BerriAI/litellm/releases" in flattened
|
||||
assert "Snippet: Latest release v1.95.0" in flattened
|
||||
assert msgs[1]["content"][0]["type"] == "server_tool_use"
|
||||
|
||||
@pytest.mark.parametrize("results", [[], None], ids=["empty_list", "search_raised"])
|
||||
def test_flatten_unencrypted_web_search_results_flattens_a_resultless_search(self, results):
|
||||
"""A search that found nothing, or that raised, still has to be flattened.
|
||||
|
||||
Both cases reach the client as ``content: []``, and leaving that block in
|
||||
place ships an unsupported tag to Bedrock on the next turn just as surely
|
||||
as a populated one does.
|
||||
"""
|
||||
from litellm.integrations.websearch_interception.transformation import (
|
||||
WebSearchTransformation,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
flatten_unencrypted_web_search_results_in_anthropic_messages,
|
||||
)
|
||||
|
||||
block = WebSearchTransformation.build_web_search_tool_result_block(
|
||||
tool_use_id="srvtoolu_1",
|
||||
search_response=None if results is None else SimpleNamespace(results=results),
|
||||
)
|
||||
assert block["content"] == [], "fixture drifted from what the interceptor emits"
|
||||
|
||||
msgs = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_1",
|
||||
"name": "web_search",
|
||||
"input": {"query": "who won"},
|
||||
},
|
||||
block,
|
||||
{"type": "text", "text": "I could not find that."},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
out = flatten_unencrypted_web_search_results_in_anthropic_messages(msgs)
|
||||
|
||||
assert [b["type"] for b in out[0]["content"]] == ["text", "text"]
|
||||
assert out[0]["content"][0]["text"] == ("Web search results for 'who won':\n\nNo results were returned.")
|
||||
|
||||
@pytest.mark.parametrize("results", [[SimpleNamespace(title="Rome", url="u", snippet="s", date=None)], []])
|
||||
def test_flatten_unencrypted_web_search_results_is_idempotent(self, results):
|
||||
"""Flattening twice must equal flattening once.
|
||||
|
||||
The agentic loop re-enters the same entry point for its follow-up call and
|
||||
hands it the original history, so this runs again on already-flattened
|
||||
messages once per iteration. A pass that appended instead of replacing
|
||||
would duplicate the evidence on every loop.
|
||||
"""
|
||||
from litellm.integrations.websearch_interception.transformation import (
|
||||
WebSearchTransformation,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
flatten_unencrypted_web_search_results_in_anthropic_messages,
|
||||
)
|
||||
|
||||
msgs = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "when"}},
|
||||
WebSearchTransformation.build_web_search_tool_result_block(
|
||||
tool_use_id="srvtoolu_1",
|
||||
search_response=SimpleNamespace(results=results),
|
||||
),
|
||||
{"type": "text", "text": "753 BC."},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
once = flatten_unencrypted_web_search_results_in_anthropic_messages(msgs)
|
||||
twice = flatten_unencrypted_web_search_results_in_anthropic_messages(once)
|
||||
|
||||
assert [b["type"] for b in once[0]["content"]] == ["text", "text"]
|
||||
assert json.dumps(twice) == json.dumps(once)
|
||||
|
||||
def test_flatten_unencrypted_web_search_results_preserves_real_anthropic_blocks(self):
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
flatten_unencrypted_web_search_results_in_anthropic_messages,
|
||||
)
|
||||
|
||||
msgs = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_1",
|
||||
"name": "web_search",
|
||||
"input": {"query": "q"},
|
||||
},
|
||||
{
|
||||
"type": "web_search_tool_result",
|
||||
"tool_use_id": "srvtoolu_1",
|
||||
"content": [
|
||||
{
|
||||
"type": "web_search_result",
|
||||
"url": "https://example.com",
|
||||
"title": "Example",
|
||||
"page_age": None,
|
||||
"encrypted_content": "EqgfCioIARgBIiQ4",
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
out = flatten_unencrypted_web_search_results_in_anthropic_messages(msgs)
|
||||
|
||||
assert out[0] is msgs[0]
|
||||
|
||||
def test_flatten_unencrypted_web_search_results_leaves_error_blocks_alone(self):
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
flatten_unencrypted_web_search_results_in_anthropic_messages,
|
||||
)
|
||||
|
||||
msgs = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "web_search_tool_result",
|
||||
"tool_use_id": "srvtoolu_1",
|
||||
"content": {
|
||||
"type": "web_search_tool_result_error",
|
||||
"error_code": "max_uses_exceeded",
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
out = flatten_unencrypted_web_search_results_in_anthropic_messages(msgs)
|
||||
|
||||
assert out[0] is msgs[0]
|
||||
|
||||
def test_sanitize_tool_use_ids_in_anthropic_messages(self):
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
sanitize_tool_use_ids_in_anthropic_messages,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import json
|
|||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
|
@ -2515,3 +2516,67 @@ def test_bedrock_messages_thinking_shape_follows_exact_bedrock_entry_flag(
|
|||
assert thinking.get("type") == "enabled"
|
||||
assert isinstance(thinking.get("budget_tokens"), int)
|
||||
assert "output_config" not in flipped
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"search_results, expected_evidence",
|
||||
[
|
||||
pytest.param(
|
||||
[SimpleNamespace(title="Rome", url="https://ex.com/rome", snippet="Founded 753 BC.", date=None)],
|
||||
"Snippet: Founded 753 BC.",
|
||||
id="search_returned_results",
|
||||
),
|
||||
pytest.param([], "No results were returned.", id="search_returned_nothing"),
|
||||
],
|
||||
)
|
||||
def test_replayed_intercepted_search_turn_leaves_no_unsupported_block_for_bedrock(search_results, expected_evidence):
|
||||
"""A native client replaying an intercepted search turn must not 400 on Bedrock.
|
||||
|
||||
``websearch_interception`` hands Claude Desktop an Anthropic-native
|
||||
``server_tool_use`` + ``web_search_tool_result`` pair, and Anthropic's protocol
|
||||
obliges the client to replay that assistant turn verbatim on every later turn.
|
||||
Bedrock's Anthropic schema defines neither tag, so both have to be gone from the
|
||||
outbound body by the time it is signed, with the search evidence carried forward
|
||||
as text instead. Built from the real builder rather than a hand-written fixture
|
||||
so the two cannot drift apart.
|
||||
"""
|
||||
from litellm.integrations.websearch_interception.transformation import (
|
||||
WebSearchTransformation,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
flatten_unencrypted_web_search_results_in_anthropic_messages,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
replayed_turn = [
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_1",
|
||||
"name": "web_search",
|
||||
"input": {"query": "when was Rome founded"},
|
||||
},
|
||||
WebSearchTransformation.build_web_search_tool_result_block(
|
||||
tool_use_id="srvtoolu_1",
|
||||
search_response=SimpleNamespace(results=search_results),
|
||||
),
|
||||
{"type": "text", "text": "Rome was founded in 753 BC."},
|
||||
]
|
||||
messages = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "When was Rome founded?"}]},
|
||||
{"role": "assistant", "content": replayed_turn},
|
||||
{"role": "user", "content": [{"type": "text", "text": "Repeat the year."}]},
|
||||
]
|
||||
|
||||
body = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request(
|
||||
model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
messages=flatten_unencrypted_web_search_results_in_anthropic_messages(messages),
|
||||
anthropic_messages_optional_request_params={"max_tokens": 64},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
serialized = json.dumps(body)
|
||||
assert "web_search_tool_result" not in serialized
|
||||
assert "server_tool_use" not in serialized
|
||||
assert expected_evidence in serialized
|
||||
assert "Rome was founded in 753 BC." in serialized
|
||||
|
|
|
|||
|
|
@ -735,3 +735,28 @@ def test_add_known_models_refreshes_models_by_provider_for_wildcard_expansion():
|
|||
litellm.vertex_language_models.discard(fake_model)
|
||||
litellm.add_known_models(model_cost_map={})
|
||||
assert fake_model not in litellm.models_by_provider["vertex_ai"]
|
||||
|
||||
def test_get_complete_model_list_drops_no_default_models_sentinel():
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list
|
||||
|
||||
result = get_complete_model_list(
|
||||
key_models=["no-default-models", "model-a"],
|
||||
team_models=[],
|
||||
proxy_model_list=["model-a", "model-b"],
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
)
|
||||
assert result == ["model-a"]
|
||||
|
||||
|
||||
def test_get_complete_model_list_sentinel_only_grants_nothing():
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list
|
||||
|
||||
result = get_complete_model_list(
|
||||
key_models=["no-default-models"],
|
||||
team_models=["no-default-models"],
|
||||
proxy_model_list=["model-a", "model-b"],
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
)
|
||||
assert result == []
|
||||
|
|
|
|||
|
|
@ -93,6 +93,19 @@ class TestBuildTransaction:
|
|||
def test_requests_without_a_routing_decision_are_skipped(self, metadata: dict):
|
||||
assert _build(metadata=metadata) is None
|
||||
|
||||
def test_the_tier_the_decision_recorded_is_carried_onto_the_transaction(self):
|
||||
transaction = _build(metadata=_metadata(routing_decision={**ROUTING_DECISION, "tier": "reasoning"}))
|
||||
assert transaction is not None and transaction.tier == "reasoning"
|
||||
|
||||
@pytest.mark.parametrize("tier", [None, "", 3, {"tier": "medium"}])
|
||||
def test_a_decision_without_a_usable_tier_records_no_tier(self, tier: object):
|
||||
transaction = _build(metadata=_metadata(routing_decision={**ROUTING_DECISION, "tier": tier}))
|
||||
assert transaction is not None and transaction.tier is None
|
||||
|
||||
def test_a_decision_that_never_mentions_tier_records_no_tier(self):
|
||||
transaction = _build()
|
||||
assert transaction is not None and transaction.tier is None
|
||||
|
||||
def test_router_name_falls_back_to_the_payload_model_group(self):
|
||||
transaction = _build(metadata=_metadata(routing_decision={"router_type": "complexity"}))
|
||||
assert transaction is not None and transaction.router_name == "live-auto"
|
||||
|
|
@ -167,7 +180,11 @@ class _FakeClient:
|
|||
self.db = _FakeDB(failures, poison_session)
|
||||
|
||||
|
||||
def _transaction(session_id: str = "s1", at: datetime = datetime(2026, 8, 1, 12, 0, 0)) -> AutoRouterTurnTransaction:
|
||||
def _transaction(
|
||||
session_id: str = "s1",
|
||||
at: datetime = datetime(2026, 8, 1, 12, 0, 0),
|
||||
tier: str | None = "medium",
|
||||
) -> AutoRouterTurnTransaction:
|
||||
return AutoRouterTurnTransaction(
|
||||
api_key="k1",
|
||||
session_id=session_id,
|
||||
|
|
@ -182,6 +199,7 @@ def _transaction(session_id: str = "s1", at: datetime = datetime(2026, 8, 1, 12,
|
|||
cache_hit=False,
|
||||
cache_ttl_seconds=None,
|
||||
cache_touched=False,
|
||||
tier=tier,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -201,7 +219,7 @@ class TestFlush:
|
|||
assert sql == UPSERT_AUTOROUTER_SESSION_SQL
|
||||
assert params == (
|
||||
"k1", "s1", "live-auto", "complexity", "bedrock/haiku",
|
||||
"2026-08-01T12:00:00", 100, 0.01, 0.02, 1, 0, None, 0,
|
||||
"2026-08-01T12:00:00", 100, 0.01, 0.02, 1, 0, None, 0, "medium",
|
||||
)
|
||||
|
||||
def test_a_connect_error_retries_the_same_statement(self):
|
||||
|
|
|
|||
|
|
@ -292,6 +292,7 @@ class TestAutoRouterBenchmarks:
|
|||
ROW = _SessionAggRow(
|
||||
router_name="live-auto",
|
||||
router_type="complexity",
|
||||
tier_turns={},
|
||||
sessions=4,
|
||||
turns=40,
|
||||
unordered_turns=1,
|
||||
|
|
@ -377,6 +378,21 @@ class TestAutoRouterBenchmarks:
|
|||
assert totals.avg_turns_per_session == 10.0
|
||||
assert totals.spend == 10.0
|
||||
|
||||
def test_tier_names_stay_scoped_to_the_router_type_that_recorded_them(self):
|
||||
quality = self.ROW.model_copy(
|
||||
update={"router_name": "quality-auto", "router_type": "quality", "tier_turns": {"2": 7}}
|
||||
)
|
||||
complexity = self.ROW.model_copy(update={"tier_turns": {"medium": 7}})
|
||||
assert complexity.tier_turns == {"medium": 7}
|
||||
assert quality.tier_turns == {"2": 7}
|
||||
|
||||
def test_summed_totals_carry_no_tier_map_because_names_are_router_scoped(self):
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import _summed_agg_row
|
||||
|
||||
quality = self.ROW.model_copy(update={"router_type": "quality", "tier_turns": {"2": 7}})
|
||||
complexity = self.ROW.model_copy(update={"tier_turns": {"medium": 7}})
|
||||
assert _summed_agg_row([complexity, quality]).tier_turns == {}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_admin_roles_cannot_read_benchmarks(self):
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks
|
||||
|
|
@ -427,3 +443,26 @@ class TestAutoRouterBenchmarks:
|
|||
assert response.routers_in_scope == 1
|
||||
assert response.groups[0].router_name == "live-auto"
|
||||
assert response.groups[0].saved_pct == response.totals.saved_pct == 75.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"wire_value, expected", [({"simple": 24, "complex": 16}, {"simple": 24, "complex": 16}), ({}, {})]
|
||||
)
|
||||
async def test_the_tier_map_reaches_the_response_as_the_jsonb_column_returns_it(
|
||||
self, wire_value: dict, expected: dict, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks
|
||||
|
||||
class _DB:
|
||||
async def query_raw(self, sql: str, *params: object):
|
||||
return [{**TestAutoRouterBenchmarks.ROW.model_dump(), "tier_turns": wire_value}]
|
||||
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", type("P", (), {"db": _DB()})())
|
||||
|
||||
response = await get_auto_router_benchmarks(
|
||||
user_api_key_dict=ADMIN,
|
||||
start_date="2026-07-01",
|
||||
end_date="2026-08-01",
|
||||
)
|
||||
assert response.groups[0].tier_turns == expected
|
||||
|
|
|
|||
|
|
@ -8637,9 +8637,9 @@ class TestBatchResolveAccessGroupResources:
|
|||
with patch("litellm.proxy.proxy_server.prisma_client", fake_prisma):
|
||||
result = await _batch_resolve_access_group_resources(["ag-1"])
|
||||
|
||||
assert sorted(result["ag-1"]["models"]) == ["claude-3", "gpt-4"]
|
||||
assert result["ag-1"]["mcp_server_ids"] == ["mcp-1"]
|
||||
assert sorted(result["ag-1"]["agent_ids"]) == ["agent-1", "agent-2"]
|
||||
assert sorted(result["ag-1"].access_model_names) == ["claude-3", "gpt-4"]
|
||||
assert result["ag-1"].access_mcp_server_ids == ["mcp-1"]
|
||||
assert sorted(result["ag-1"].access_agent_ids) == ["agent-1", "agent-2"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_access_groups(self):
|
||||
|
|
@ -8668,8 +8668,8 @@ class TestBatchResolveAccessGroupResources:
|
|||
with patch("litellm.proxy.proxy_server.prisma_client", fake_prisma):
|
||||
result = await _batch_resolve_access_group_resources(["ag-1", "ag-2"])
|
||||
|
||||
assert result["ag-1"]["models"] == ["gpt-4"]
|
||||
assert result["ag-2"]["models"] == ["gemini"]
|
||||
assert result["ag-1"].access_model_names == ["gpt-4"]
|
||||
assert result["ag-2"].access_model_names == ["gemini"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_access_group_omitted(self):
|
||||
|
|
@ -8735,6 +8735,75 @@ class TestBatchResolveAccessGroupResources:
|
|||
assert "ag-1" in result
|
||||
|
||||
|
||||
class TestResolveTeamAccessGroupResources:
|
||||
"""Tests for the per-team access group resolution on /team/info."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_populates_flat_lists_and_per_group_details(self):
|
||||
"""access_group_details must attribute each model to the group granting it,
|
||||
so the UI can show provenance on hover; flat lists stay for back-compat.
|
||||
Duplicated ids must collapse to one entry (response amplification), and the
|
||||
input object must stay untouched (resolution returns a copy)."""
|
||||
from litellm.proxy._types import TeamInfoResponseObjectTeamTable
|
||||
from litellm.proxy.management_endpoints.team_endpoints import (
|
||||
_resolve_team_access_group_resources,
|
||||
)
|
||||
|
||||
row1 = MagicMock()
|
||||
row1.access_group_id = "ag-1"
|
||||
row1.access_group_name = "shared-models"
|
||||
row1.access_model_names = ["gpt-4", "claude-3"]
|
||||
row1.access_mcp_server_ids = ["mcp-1"]
|
||||
row1.access_agent_ids = []
|
||||
|
||||
row2 = MagicMock()
|
||||
row2.access_group_id = "ag-2"
|
||||
row2.access_group_name = "extra-models"
|
||||
row2.access_model_names = ["claude-3", "gemini"]
|
||||
row2.access_mcp_server_ids = []
|
||||
row2.access_agent_ids = ["agent-1"]
|
||||
|
||||
fake_prisma = MagicMock()
|
||||
fake_prisma.db.litellm_accessgrouptable.find_many = AsyncMock(
|
||||
return_value=[row1, row2]
|
||||
)
|
||||
|
||||
team_info = TeamInfoResponseObjectTeamTable(
|
||||
team_id="team-1", access_group_ids=["ag-1", "ag-2", "ag-1", "ag-missing"]
|
||||
)
|
||||
with patch("litellm.proxy.proxy_server.prisma_client", fake_prisma):
|
||||
resolved = await _resolve_team_access_group_resources(team_info)
|
||||
|
||||
assert team_info.access_group_details is None
|
||||
assert sorted(resolved.access_group_models or []) == [
|
||||
"claude-3",
|
||||
"gemini",
|
||||
"gpt-4",
|
||||
]
|
||||
assert resolved.access_group_mcp_server_ids == ["mcp-1"]
|
||||
assert resolved.access_group_agent_ids == ["agent-1"]
|
||||
assert [
|
||||
(d.access_group_id, d.access_group_name, d.models)
|
||||
for d in (resolved.access_group_details or [])
|
||||
] == [
|
||||
("ag-1", "shared-models", ("gpt-4", "claude-3")),
|
||||
("ag-2", "extra-models", ("claude-3", "gemini")),
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_access_groups_leaves_details_unset(self):
|
||||
from litellm.proxy._types import TeamInfoResponseObjectTeamTable
|
||||
from litellm.proxy.management_endpoints.team_endpoints import (
|
||||
_resolve_team_access_group_resources,
|
||||
)
|
||||
|
||||
team_info = TeamInfoResponseObjectTeamTable(team_id="team-1", access_group_ids=[])
|
||||
resolved = await _resolve_team_access_group_resources(team_info)
|
||||
|
||||
assert resolved.access_group_details is None
|
||||
assert resolved.access_group_models is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_verify_team_access_denies_unauthorized_user():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -2638,4 +2638,4 @@ async def test_ProxyConfig__init_non_llm_configs_empty_agents_key_clears_remembe
|
|||
|
||||
assert clean_agent_registry.config_agents == ()
|
||||
clean_agent_registry.load_agents_from_db_and_config(db_agents=None)
|
||||
assert clean_agent_registry.get_agent_list() == []
|
||||
assert clean_agent_registry.get_agent_list() == ()
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from litellm.proxy._types import UserAPIKeyAuth
|
|||
from litellm.proxy.utils import (
|
||||
create_model_info_response,
|
||||
get_available_models_for_user,
|
||||
hash_token,
|
||||
is_known_model,
|
||||
is_known_vector_store_index,
|
||||
model_dump_with_preserved_fields,
|
||||
|
|
@ -404,3 +405,119 @@ async def test_get_available_models_for_user_error_path_complete_list_raises(
|
|||
general_settings={},
|
||||
user_model=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_available_models_for_user_resolves_team_access_group_models(
|
||||
monkeypatch,
|
||||
):
|
||||
from litellm.models.access_group import LiteLLM_AccessGroupTable
|
||||
from litellm.models.team import LiteLLM_TeamTableCachedObj
|
||||
|
||||
team = LiteLLM_TeamTableCachedObj(
|
||||
team_id="team-1",
|
||||
models=["no-default-models"],
|
||||
access_group_ids=["ag-1"],
|
||||
)
|
||||
access_group = LiteLLM_AccessGroupTable(
|
||||
access_group_id="ag-1",
|
||||
access_group_name="repro-group",
|
||||
access_model_names=["model-a", "model-b"],
|
||||
assigned_team_ids=["team-1"],
|
||||
)
|
||||
|
||||
async def _get_team_object(**_kwargs):
|
||||
return team
|
||||
|
||||
async def _get_access_object(**_kwargs):
|
||||
return access_group
|
||||
|
||||
monkeypatch.setattr("litellm.proxy.auth.auth_checks.get_team_object", _get_team_object)
|
||||
monkeypatch.setattr("litellm.proxy.auth.auth_checks.get_access_object", _get_access_object)
|
||||
|
||||
result = await get_available_models_for_user(
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
api_key="sk-test-key",
|
||||
user_id="user-1",
|
||||
team_id="team-1",
|
||||
models=["all-team-models"],
|
||||
team_models=["no-default-models"],
|
||||
),
|
||||
llm_router=_router_with_models(["model-a", "model-b", "model-c"]),
|
||||
general_settings={},
|
||||
user_model=None,
|
||||
prisma_client=MagicMock(),
|
||||
proxy_logging_obj=MagicMock(),
|
||||
user_api_key_cache=MagicMock(),
|
||||
)
|
||||
assert sorted(result) == ["model-a", "model-b"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_available_models_for_user_without_access_groups_grants_nothing(
|
||||
monkeypatch,
|
||||
):
|
||||
from litellm.models.team import LiteLLM_TeamTableCachedObj
|
||||
|
||||
async def _get_team_object(**_kwargs):
|
||||
return LiteLLM_TeamTableCachedObj(team_id="team-1", models=["no-default-models"])
|
||||
|
||||
monkeypatch.setattr("litellm.proxy.auth.auth_checks.get_team_object", _get_team_object)
|
||||
|
||||
result = await get_available_models_for_user(
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
api_key="sk-test-key",
|
||||
user_id="user-1",
|
||||
team_id="team-1",
|
||||
models=["all-team-models"],
|
||||
team_models=["no-default-models"],
|
||||
),
|
||||
llm_router=_router_with_models(["model-a", "model-b"]),
|
||||
general_settings={},
|
||||
user_model=None,
|
||||
prisma_client=MagicMock(),
|
||||
proxy_logging_obj=MagicMock(),
|
||||
user_api_key_cache=MagicMock(),
|
||||
)
|
||||
assert result == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_available_models_for_user_resolves_key_access_group_models(
|
||||
monkeypatch,
|
||||
):
|
||||
from litellm.models.access_group import LiteLLM_AccessGroupTable
|
||||
from litellm.models.team import LiteLLM_TeamTableCachedObj
|
||||
|
||||
async def _get_team_object(**_kwargs):
|
||||
return LiteLLM_TeamTableCachedObj(team_id="team-1", models=["no-default-models"])
|
||||
|
||||
async def _get_access_object(**_kwargs):
|
||||
return LiteLLM_AccessGroupTable(
|
||||
access_group_id="ag-1",
|
||||
access_group_name="key-group",
|
||||
access_model_names=["model-b"],
|
||||
assigned_key_ids=[hash_token("sk-test-key")],
|
||||
)
|
||||
|
||||
monkeypatch.setattr("litellm.proxy.auth.auth_checks.get_team_object", _get_team_object)
|
||||
monkeypatch.setattr("litellm.proxy.auth.auth_checks.get_access_object", _get_access_object)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MagicMock())
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", MagicMock())
|
||||
|
||||
result = await get_available_models_for_user(
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
api_key="sk-test-key",
|
||||
user_id="user-1",
|
||||
team_id="team-1",
|
||||
models=["no-default-models"],
|
||||
team_models=["no-default-models"],
|
||||
access_group_ids=["ag-1"],
|
||||
),
|
||||
llm_router=_router_with_models(["model-a", "model-b"]),
|
||||
general_settings={},
|
||||
user_model=None,
|
||||
prisma_client=MagicMock(),
|
||||
proxy_logging_obj=MagicMock(),
|
||||
user_api_key_cache=MagicMock(),
|
||||
)
|
||||
assert result == ["model-b"]
|
||||
|
|
|
|||
|
|
@ -3477,6 +3477,23 @@ class TestSessionAffinity:
|
|||
# Pinned to the first turn's model, not re-classified down to SIMPLE.
|
||||
assert second.model == "o1-preview"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_pinned_turn_reports_the_tier_that_serves_it(self, mock_router_instance, session_affinity_config):
|
||||
mock_router_instance.cache = DualCache()
|
||||
router = ComplexityRouter(
|
||||
model_name="test-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config=session_affinity_config,
|
||||
)
|
||||
request_kwargs = self._request_kwargs("session-1")
|
||||
await router.async_pre_routing_hook(
|
||||
model="test-model", request_kwargs=request_kwargs, messages=self.REASONING_MESSAGE
|
||||
)
|
||||
pinned = await router.async_pre_routing_hook(
|
||||
model="test-model", request_kwargs=request_kwargs, messages=self.SIMPLE_MESSAGE
|
||||
)
|
||||
assert pinned.routing_decision["tier"] == "REASONING"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_different_sessions_classify_independently(self, mock_router_instance, session_affinity_config):
|
||||
mock_router_instance.cache = DualCache()
|
||||
|
|
@ -4362,7 +4379,24 @@ class TestRoutingDecisionContents:
|
|||
assert decision is not None
|
||||
assert decision["cause"] == "default_fallback"
|
||||
assert decision["routed_model"] == response.model
|
||||
assert "tier" not in decision
|
||||
assert decision.get("tier") == "MEDIUM"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_default_model_fallback_claims_no_tier(self, mock_router_instance, basic_config):
|
||||
router = ComplexityRouter(
|
||||
model_name="test-complexity-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config={**basic_config, "default_model": "gpt-4o"},
|
||||
)
|
||||
response = await router.async_pre_routing_hook(
|
||||
model="test-complexity-router",
|
||||
request_kwargs={},
|
||||
messages=[{"role": "system", "content": "be nice"}],
|
||||
)
|
||||
assert response is not None
|
||||
assert response.routing_decision is not None
|
||||
assert response.routing_decision["cause"] == "default_fallback"
|
||||
assert "tier" not in response.routing_decision
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_pin_decision(self, mock_router_instance, basic_config):
|
||||
|
|
@ -5907,11 +5941,21 @@ class TestConversationShapeDiscriminator:
|
|||
)
|
||||
builds = source.split("self._build_routing_decision(")[1:]
|
||||
assert builds
|
||||
missing = [
|
||||
i
|
||||
for i, block in enumerate(builds)
|
||||
if "conversation_continuing=conversation_continuing" not in block.split("),")[0]
|
||||
]
|
||||
missing = []
|
||||
for i, block in enumerate(builds):
|
||||
depth = 0
|
||||
end = 0
|
||||
for j, char in enumerate(block):
|
||||
if char == "(":
|
||||
depth += 1
|
||||
elif char == ")":
|
||||
depth -= 1
|
||||
if depth < 0:
|
||||
end = j
|
||||
break
|
||||
extracted = block[:end]
|
||||
if "conversation_continuing=conversation_continuing" not in extracted:
|
||||
missing.append(i)
|
||||
assert not missing, f"routing decisions {missing} do not carry the conversation shape"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
{
|
||||
"LIT001": {
|
||||
"limit": 23237
|
||||
"limit": 23235
|
||||
},
|
||||
"LIT002": {
|
||||
"limit": 27166
|
||||
"limit": 27176
|
||||
},
|
||||
"LIT003": {
|
||||
"limit": 269
|
||||
|
|
@ -15,7 +15,7 @@
|
|||
"limit": 0
|
||||
},
|
||||
"LIT006": {
|
||||
"limit": 1083
|
||||
"limit": 1091
|
||||
},
|
||||
"LIT007": {
|
||||
"limit": 0
|
||||
|
|
@ -27,9 +27,9 @@
|
|||
"limit": 0
|
||||
},
|
||||
"LIT010": {
|
||||
"limit": 16763
|
||||
"limit": 16769
|
||||
},
|
||||
"LIT011": {
|
||||
"limit": 5602
|
||||
"limit": 5598
|
||||
}
|
||||
}
|
||||
|
|
|
|||
6
ui/litellm-dashboard/package-lock.json
generated
6
ui/litellm-dashboard/package-lock.json
generated
|
|
@ -10319,9 +10319,9 @@
|
|||
"license": "MIT"
|
||||
},
|
||||
"node_modules/nanoid": {
|
||||
"version": "3.3.16",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz",
|
||||
"integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==",
|
||||
"version": "3.3.17",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.17.tgz",
|
||||
"integrity": "sha512-xQLf0A3HOMlgHq0n247/LRuAOYmB7dXJ/DvAxGvsSBij45XtBSmQycu+F8ODbHwns/XyFZagyL1+J0Offw1E0g==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
|
|
|
|||
|
|
@ -2197,4 +2197,176 @@ describe("EntityUsageExport utils", () => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("display name resolution from entity metadata", () => {
|
||||
const entityMetrics = {
|
||||
spend: 12.25,
|
||||
api_requests: 40,
|
||||
successful_requests: 39,
|
||||
failed_requests: 1,
|
||||
total_tokens: 900,
|
||||
prompt_tokens: 500,
|
||||
completion_tokens: 400,
|
||||
cache_read_input_tokens: 20,
|
||||
cache_creation_input_tokens: 10,
|
||||
};
|
||||
|
||||
const makeSpendData = (entity: string, metadata?: Record<string, any>): EntitySpendData => ({
|
||||
results: [
|
||||
{
|
||||
date: "2025-04-01",
|
||||
breakdown: {
|
||||
entities: {
|
||||
[entity]: {
|
||||
metrics: entityMetrics,
|
||||
metadata,
|
||||
api_key_breakdown: {
|
||||
key1: {
|
||||
metrics: entityMetrics,
|
||||
metadata: { key_alias: "prod-key" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
metadata: mockSpendData.metadata,
|
||||
});
|
||||
|
||||
it("should export the user email as the entity label and keep the raw user id in the id column", () => {
|
||||
const result = generateDailyData(
|
||||
makeSpendData("user-123", { user_email: "ada@example.com", user_alias: "Ada" }),
|
||||
"User",
|
||||
);
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0]["User"]).toBe("ada@example.com");
|
||||
expect(result[0]["User ID"]).toBe("user-123");
|
||||
});
|
||||
|
||||
it("should fall back to the user alias when the user has no email", () => {
|
||||
const nullEmail = generateDailyData(
|
||||
makeSpendData("user-123", { user_email: null, user_alias: "Ada Lovelace" }),
|
||||
"User",
|
||||
);
|
||||
const missingEmail = generateDailyData(makeSpendData("user-123", { user_alias: "Ada Lovelace" }), "User");
|
||||
|
||||
expect(nullEmail[0]["User"]).toBe("Ada Lovelace");
|
||||
expect(missingEmail[0]["User"]).toBe("Ada Lovelace");
|
||||
});
|
||||
|
||||
it("should fall back to the raw entity key when the entity carries no metadata", () => {
|
||||
const noMetadata = generateDailyData(makeSpendData("my-tag"), "Tag");
|
||||
const emptyMetadata = generateDailyData(makeSpendData("customer-9", {}), "Customer");
|
||||
const blankNames = generateDailyData(makeSpendData("user-123", { user_email: null, user_alias: null }), "User");
|
||||
|
||||
expect(noMetadata[0]["Tag"]).toBe("my-tag");
|
||||
expect(emptyMetadata[0]["Customer"]).toBe("customer-9");
|
||||
expect(blankNames[0]["User"]).toBe("user-123");
|
||||
});
|
||||
|
||||
it("should prefer the team alias map over any alias in entity metadata", () => {
|
||||
const result = generateDailyData(
|
||||
makeSpendData("team-1", { team_alias: "Stale Alias", user_email: "ada@example.com" }),
|
||||
"Team",
|
||||
mockTeamAliasMap,
|
||||
);
|
||||
|
||||
expect(result[0]["Team"]).toBe("Team One");
|
||||
});
|
||||
|
||||
it("should use the team alias from entity metadata when the alias map has no entry for the team", () => {
|
||||
const result = generateDailyData(
|
||||
makeSpendData("team-9", { team_alias: "Team Nine", user_email: "ada@example.com" }),
|
||||
"Team",
|
||||
mockTeamAliasMap,
|
||||
);
|
||||
|
||||
expect(result[0]["Team"]).toBe("Team Nine");
|
||||
});
|
||||
|
||||
it("should resolve metadata.alias to the user email in getEntityBreakdown", () => {
|
||||
const withEmail = getEntityBreakdown(
|
||||
makeSpendData("user-123", { user_email: "ada@example.com", user_alias: "Ada" }),
|
||||
);
|
||||
const withoutEmail = getEntityBreakdown(makeSpendData("user-123", { user_alias: "Ada" }));
|
||||
|
||||
expect(withEmail[0].metadata.alias).toBe("ada@example.com");
|
||||
expect(withEmail[0].metadata.id).toBe("user-123");
|
||||
expect(withoutEmail[0].metadata.alias).toBe("Ada");
|
||||
});
|
||||
|
||||
it("should resolve the user email on every key row of the keys scope", () => {
|
||||
const spendData: EntitySpendData = {
|
||||
results: [
|
||||
{
|
||||
date: "2025-04-01",
|
||||
breakdown: {
|
||||
entities: {
|
||||
"user-123": {
|
||||
metrics: entityMetrics,
|
||||
metadata: { user_email: "ada@example.com", user_alias: "Ada" },
|
||||
api_key_breakdown: {
|
||||
key1: { metrics: entityMetrics, metadata: { key_alias: "prod-key" } },
|
||||
key2: { metrics: entityMetrics, metadata: { key_alias: "dev-key" } },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
metadata: mockSpendData.metadata,
|
||||
};
|
||||
|
||||
const result = generateDailyWithKeysData(spendData, "User");
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result.map((r) => r["User"])).toEqual(["ada@example.com", "ada@example.com"]);
|
||||
expect(result.map((r) => r["User ID"])).toEqual(["user-123", "user-123"]);
|
||||
expect(result.find((r) => r["Key ID"] === "key1")?.["Key Alias"]).toBe("prod-key");
|
||||
expect(result.find((r) => r["Key ID"] === "key2")?.["Key Alias"]).toBe("dev-key");
|
||||
});
|
||||
|
||||
it("should resolve each entity's own email in the models scope", () => {
|
||||
const spendData: EntitySpendData = {
|
||||
results: [
|
||||
{
|
||||
date: "2025-04-01",
|
||||
breakdown: {
|
||||
entities: {
|
||||
"user-a": {
|
||||
metrics: entityMetrics,
|
||||
metadata: { user_email: "ada@example.com", user_alias: "Ada" },
|
||||
api_key_breakdown: { key1: { metrics: entityMetrics, metadata: {} } },
|
||||
},
|
||||
"user-b": {
|
||||
metrics: entityMetrics,
|
||||
metadata: { user_email: null, user_alias: "Grace" },
|
||||
api_key_breakdown: { key2: { metrics: entityMetrics, metadata: {} } },
|
||||
},
|
||||
},
|
||||
models: {
|
||||
"claude-sonnet-4-5": {
|
||||
metrics: entityMetrics,
|
||||
api_key_breakdown: {
|
||||
key1: { metrics: entityMetrics, metadata: {} },
|
||||
key2: { metrics: entityMetrics, metadata: {} },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
metadata: mockSpendData.metadata,
|
||||
};
|
||||
|
||||
const result = generateDailyWithModelsData(spendData, "User");
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result.every((r) => r.Model === "claude-sonnet-4-5")).toBe(true);
|
||||
expect(result.find((r) => r["User ID"] === "user-a")?.["User"]).toBe("ada@example.com");
|
||||
expect(result.find((r) => r["User ID"] === "user-b")?.["User"]).toBe("Grace");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -3,12 +3,18 @@ import type { DateRangePickerValue } from "@tremor/react";
|
|||
import Papa from "papaparse";
|
||||
import type { EntityBreakdown, EntitySpendData, EntityType, ExportMetadata, ExportScope } from "./types";
|
||||
|
||||
// Resolve display name for an entity. For teams the teamAliasMap provides
|
||||
// a human-readable alias; for every other entity type the entity key itself
|
||||
// (tag name, org id, customer id, …) is already the correct label.
|
||||
const resolveEntityDisplay = (entity: string, teamAliasMap: Record<string, string>): { id: string; alias: string } => ({
|
||||
const resolveEntityDisplay = (
|
||||
entity: string,
|
||||
teamAliasMap: Record<string, string>,
|
||||
entityMetadata?: Record<string, any>,
|
||||
): { id: string; alias: string } => ({
|
||||
id: entity,
|
||||
alias: teamAliasMap[entity] || entity,
|
||||
alias:
|
||||
teamAliasMap[entity] ||
|
||||
entityMetadata?.team_alias ||
|
||||
entityMetadata?.user_email ||
|
||||
entityMetadata?.user_alias ||
|
||||
entity,
|
||||
});
|
||||
|
||||
// Mirrors backend SpendMetrics fields (litellm/types/activity_tracking.py).
|
||||
|
|
@ -68,7 +74,7 @@ export const getEntityBreakdown = (
|
|||
|
||||
spendData.results.forEach((day) => {
|
||||
Object.entries(resolveEntities(day.breakdown)).forEach(([entity, data]: [string, any]) => {
|
||||
const { id, alias } = resolveEntityDisplay(entity, teamAliasMap);
|
||||
const { id, alias } = resolveEntityDisplay(entity, teamAliasMap, data.metadata);
|
||||
|
||||
if (!entitySpend[entity]) {
|
||||
entitySpend[entity] = {
|
||||
|
|
@ -113,7 +119,7 @@ export const generateDailyData = (
|
|||
|
||||
spendData.results.forEach((day) => {
|
||||
Object.entries(resolveEntities(day.breakdown)).forEach(([entity, data]: [string, any]) => {
|
||||
const { id, alias } = resolveEntityDisplay(entity, teamAliasMap);
|
||||
const { id, alias } = resolveEntityDisplay(entity, teamAliasMap, data.metadata);
|
||||
|
||||
dailyBreakdown.push({
|
||||
Date: day.date,
|
||||
|
|
@ -164,7 +170,7 @@ export const generateDailyWithKeysData = (
|
|||
|
||||
spendData.results.forEach((day) => {
|
||||
Object.entries(resolveEntities(day.breakdown)).forEach(([entity, data]: [string, any]) => {
|
||||
const { id: entityId, alias: entityAlias } = resolveEntityDisplay(entity, teamAliasMap);
|
||||
const { id: entityId, alias: entityAlias } = resolveEntityDisplay(entity, teamAliasMap, data.metadata);
|
||||
const apiKeyBreakdown = data.api_key_breakdown || {};
|
||||
|
||||
// Iterate through each API key in the breakdown
|
||||
|
|
@ -241,11 +247,13 @@ export const generateDailyWithModelsData = (
|
|||
|
||||
spendData.results.forEach((day) => {
|
||||
const dailyEntityModels: { [key: string]: { [key: string]: any } } = {};
|
||||
const dailyEntityMetadata: { [key: string]: Record<string, any> | undefined } = {};
|
||||
|
||||
Object.entries(resolveEntities(day.breakdown)).forEach(([entity, entityData]: [string, any]) => {
|
||||
if (!dailyEntityModels[entity]) {
|
||||
dailyEntityModels[entity] = {};
|
||||
}
|
||||
dailyEntityMetadata[entity] = entityData.metadata;
|
||||
|
||||
Object.entries(day.breakdown.models || {}).forEach(([model, modelData]: [string, any]) => {
|
||||
const entityApiKeys = entityData.api_key_breakdown || {};
|
||||
|
|
@ -282,7 +290,7 @@ export const generateDailyWithModelsData = (
|
|||
});
|
||||
|
||||
Object.entries(dailyEntityModels).forEach(([entity, models]) => {
|
||||
const { id, alias } = resolveEntityDisplay(entity, teamAliasMap);
|
||||
const { id, alias } = resolveEntityDisplay(entity, teamAliasMap, dailyEntityMetadata[entity]);
|
||||
|
||||
Object.entries(models).forEach(([model, metrics]: [string, any]) => {
|
||||
dailyModelBreakdown.push({
|
||||
|
|
|
|||
|
|
@ -613,6 +613,34 @@ describe("Teams - access_group_ids in team create", () => {
|
|||
);
|
||||
});
|
||||
});
|
||||
|
||||
it("creates a team with no models selected, sending the no-default-models sentinel instead of an empty list", async () => {
|
||||
renderWithQueryClient(<Teams accessToken="test-token" userID="user-123" userRole="Admin" />);
|
||||
|
||||
const createButton = screen.getAllByRole("button", { name: /create team/i })[0];
|
||||
act(() => {
|
||||
fireEvent.click(createButton);
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByLabelText(/team name/i)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
fireEvent.change(screen.getByLabelText(/team name/i), { target: { value: "Group Only Team" } });
|
||||
|
||||
const createTeamSubmitButtons = screen.getAllByRole("button", { name: /create team/i });
|
||||
fireEvent.click(createTeamSubmitButtons[createTeamSubmitButtons.length - 1]);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(teamCreateCall).toHaveBeenCalledWith(
|
||||
"test-token",
|
||||
expect.objectContaining({
|
||||
team_alias: "Group Only Team",
|
||||
models: ["no-default-models"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Teams - metadata key-value pairs in team create", () => {
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ interface TeamProps {
|
|||
|
||||
import DeleteResourceModal from "./common_components/DeleteResourceModal";
|
||||
import { teamCreateCall } from "./networking";
|
||||
import { normalizeTeamModelSelection } from "./team/teamModelAccess";
|
||||
import { ModelSelect } from "./ModelSelect/ModelSelect";
|
||||
|
||||
const canCreateOrManageTeams = (
|
||||
|
|
@ -351,7 +352,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, premiumUser
|
|||
}
|
||||
}
|
||||
|
||||
await teamCreateCall(accessToken, formValues);
|
||||
await teamCreateCall(accessToken, { ...formValues, models: normalizeTeamModelSelection(formValues.models) });
|
||||
NotificationsManager.success("Team created");
|
||||
await refreshTeams();
|
||||
form.resetFields();
|
||||
|
|
@ -618,17 +619,11 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, premiumUser
|
|||
label={
|
||||
<span>
|
||||
Models{" "}
|
||||
<Tooltip title="These are the models that your selected team has access to">
|
||||
<Tooltip title="These are the models that your selected team has access to. Leave empty to grant no models directly, e.g. when the team gets its models from access groups">
|
||||
<InfoCircleOutlined style={{ marginLeft: "4px" }} />
|
||||
</Tooltip>
|
||||
</span>
|
||||
}
|
||||
rules={[
|
||||
{
|
||||
required: true,
|
||||
message: "Please select at least one model",
|
||||
},
|
||||
]}
|
||||
name="models"
|
||||
>
|
||||
<ModelSelect
|
||||
|
|
|
|||
|
|
@ -35,6 +35,12 @@ import { CheckIcon, CopyIcon } from "lucide-react";
|
|||
import React, { useEffect, useMemo, useState } from "react";
|
||||
import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils";
|
||||
import AccessGroupSelector from "../common_components/AccessGroupSelector";
|
||||
import {
|
||||
computeTeamModelBadges,
|
||||
normalizeTeamModelSelection,
|
||||
TeamAccessGroupModelGrant,
|
||||
TeamModelBadgeKind,
|
||||
} from "./teamModelAccess";
|
||||
import MetadataKeyValueFields, {
|
||||
metadataObjectToPairs,
|
||||
metadataPairsToObject,
|
||||
|
|
@ -82,6 +88,13 @@ const UI_MANAGED_METADATA_KEYS: ReadonlySet<string> = new Set([
|
|||
"disable_global_guardrails",
|
||||
]);
|
||||
|
||||
const TEAM_MODEL_BADGE_COLORS: Record<TeamModelBadgeKind, "red" | "gray" | "blue" | "green"> = {
|
||||
"all-proxy": "red",
|
||||
"no-default": "gray",
|
||||
direct: "blue",
|
||||
"access-group": "green",
|
||||
};
|
||||
|
||||
export interface TeamMembership {
|
||||
user_id: string;
|
||||
team_id: string;
|
||||
|
|
@ -132,6 +145,7 @@ export interface TeamData {
|
|||
access_group_models?: string[];
|
||||
access_group_mcp_server_ids?: string[];
|
||||
access_group_agent_ids?: string[];
|
||||
access_group_details?: TeamAccessGroupModelGrant[];
|
||||
router_settings?: Record<string, any>;
|
||||
guardrails?: string[];
|
||||
policies?: string[];
|
||||
|
|
@ -483,7 +497,7 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
|
|||
const updateData: any = {
|
||||
team_id: teamId,
|
||||
team_alias: values.team_alias,
|
||||
models: values.models,
|
||||
models: normalizeTeamModelSelection(values.models),
|
||||
tpm_limit: sanitizeNumeric(values.tpm_limit),
|
||||
rpm_limit: sanitizeNumeric(values.rpm_limit),
|
||||
model_tpm_limit: modelTpmLimit,
|
||||
|
|
@ -764,21 +778,14 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
|
|||
<Card>
|
||||
<Text>Models</Text>
|
||||
<div className="mt-2 flex flex-wrap gap-2">
|
||||
{info.models.length === 0 || info.models.includes("all-proxy-models") ? (
|
||||
<Badge color="red">All proxy models</Badge>
|
||||
) : (
|
||||
<>
|
||||
{info.models.map((model: string, index: number) => (
|
||||
<Badge key={`direct-${index}`} color="blue">
|
||||
{model}
|
||||
</Badge>
|
||||
))}
|
||||
{(info.access_group_models || []).map((model: string, index: number) => (
|
||||
<Badge key={`ag-${index}`} color="green" title="From access group">
|
||||
{model}
|
||||
</Badge>
|
||||
))}
|
||||
</>
|
||||
{computeTeamModelBadges(info.models, info.access_group_models || [], info.access_group_details).map(
|
||||
(badge, index) => (
|
||||
<Tooltip key={`${badge.kind}-${badge.label}-${index}`} title={badge.tooltip}>
|
||||
<span>
|
||||
<Badge color={TEAM_MODEL_BADGE_COLORS[badge.kind]}>{badge.label}</Badge>
|
||||
</span>
|
||||
</Tooltip>
|
||||
),
|
||||
)}
|
||||
</div>
|
||||
</Card>
|
||||
|
|
@ -982,7 +989,7 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
|
|||
<Form.Item
|
||||
label="Models"
|
||||
name="models"
|
||||
rules={[{ required: true, message: "Please select at least one model" }]}
|
||||
extra="Leave empty to grant no models directly. The team keeps any models granted through its access groups"
|
||||
>
|
||||
<ModelSelect
|
||||
value={form.getFieldValue("models") || []}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,86 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { computeTeamModelBadges, normalizeTeamModelSelection, TeamAccessGroupModelGrant } from "./teamModelAccess";
|
||||
|
||||
const GRANTS: TeamAccessGroupModelGrant[] = [
|
||||
{ access_group_id: "ag-1", access_group_name: "shared", models: ["haiku", "gpt-4o-mini"] },
|
||||
{ access_group_id: "ag-2", access_group_name: "extra", models: ["haiku", "sonnet"] },
|
||||
];
|
||||
|
||||
describe("normalizeTeamModelSelection", () => {
|
||||
it("substitutes the no-default-models sentinel for an empty selection", () => {
|
||||
expect(normalizeTeamModelSelection([])).toEqual(["no-default-models"]);
|
||||
expect(normalizeTeamModelSelection(undefined)).toEqual(["no-default-models"]);
|
||||
});
|
||||
|
||||
it("passes a non-empty selection through untouched", () => {
|
||||
expect(normalizeTeamModelSelection(["gpt-4o-mini"])).toEqual(["gpt-4o-mini"]);
|
||||
expect(normalizeTeamModelSelection(["all-proxy-models"])).toEqual(["all-proxy-models"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("computeTeamModelBadges", () => {
|
||||
it("attributes group-only models to the groups granting them", () => {
|
||||
const badges = computeTeamModelBadges(["sonnet-direct"], [], GRANTS);
|
||||
expect(badges).toEqual([
|
||||
{
|
||||
label: "sonnet-direct",
|
||||
kind: "direct",
|
||||
tooltip: "Granted directly in the team's model list",
|
||||
},
|
||||
{ label: "haiku", kind: "access-group", tooltip: "Granted via access groups shared, extra" },
|
||||
{ label: "gpt-4o-mini", kind: "access-group", tooltip: "Granted via access group shared" },
|
||||
{ label: "sonnet", kind: "access-group", tooltip: "Granted via access group extra" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("marks a model both direct and group-granted on the direct badge, without a duplicate badge", () => {
|
||||
const badges = computeTeamModelBadges(["haiku"], [], GRANTS);
|
||||
expect(badges).toEqual([
|
||||
{
|
||||
label: "haiku",
|
||||
kind: "direct",
|
||||
tooltip: "Granted directly in the team's model list, and also via access groups shared, extra",
|
||||
},
|
||||
{ label: "gpt-4o-mini", kind: "access-group", tooltip: "Granted via access group shared" },
|
||||
{ label: "sonnet", kind: "access-group", tooltip: "Granted via access group extra" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("shows the no-default-models sentinel as its own badge and keeps group badges visible", () => {
|
||||
const badges = computeTeamModelBadges(["no-default-models"], [], [GRANTS[0]]);
|
||||
expect(badges.map((b) => [b.label, b.kind])).toEqual([
|
||||
["No default models", "no-default"],
|
||||
["haiku", "access-group"],
|
||||
["gpt-4o-mini", "access-group"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("still shows group badges when the empty model list grants everything", () => {
|
||||
const badges = computeTeamModelBadges([], [], [GRANTS[0]]);
|
||||
expect(badges[0]).toEqual({
|
||||
label: "All proxy models",
|
||||
kind: "all-proxy",
|
||||
tooltip: "The team's model list is empty, so it can access every model on the proxy",
|
||||
});
|
||||
expect(badges.slice(1).map((b) => b.label)).toEqual(["haiku", "gpt-4o-mini"]);
|
||||
});
|
||||
|
||||
it("distinguishes the all-proxy-models sentinel from an empty list in the tooltip", () => {
|
||||
const badges = computeTeamModelBadges(["all-proxy-models"], [], []);
|
||||
expect(badges).toEqual([
|
||||
{
|
||||
label: "All proxy models",
|
||||
kind: "all-proxy",
|
||||
tooltip: "Granted by the All Proxy Models entry in the team's model list",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("falls back to the flat access_group_models list when per-group details are absent", () => {
|
||||
const badges = computeTeamModelBadges(["direct-model"], ["haiku"], undefined);
|
||||
expect(badges).toEqual([
|
||||
{ label: "direct-model", kind: "direct", tooltip: "Granted directly in the team's model list" },
|
||||
{ label: "haiku", kind: "access-group", tooltip: "Granted via an access group" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
82
ui/litellm-dashboard/src/components/team/teamModelAccess.ts
Normal file
82
ui/litellm-dashboard/src/components/team/teamModelAccess.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
export const ALL_PROXY_MODELS = "all-proxy-models";
|
||||
export const NO_DEFAULT_MODELS = "no-default-models";
|
||||
|
||||
export interface TeamAccessGroupModelGrant {
|
||||
access_group_id: string;
|
||||
access_group_name: string;
|
||||
models: string[];
|
||||
}
|
||||
|
||||
export type TeamModelBadgeKind = "all-proxy" | "no-default" | "direct" | "access-group";
|
||||
|
||||
export interface TeamModelBadge {
|
||||
label: string;
|
||||
kind: TeamModelBadgeKind;
|
||||
tooltip: string;
|
||||
}
|
||||
|
||||
export function normalizeTeamModelSelection(models: string[] | undefined): string[] {
|
||||
return models && models.length > 0 ? models : [NO_DEFAULT_MODELS];
|
||||
}
|
||||
|
||||
const describeGroups = (names: string[]): string =>
|
||||
names.length > 1 ? `access groups ${names.join(", ")}` : `access group ${names[0]}`;
|
||||
|
||||
export function computeTeamModelBadges(
|
||||
models: string[],
|
||||
accessGroupModels: string[],
|
||||
accessGroupDetails: TeamAccessGroupModelGrant[] | undefined,
|
||||
): TeamModelBadge[] {
|
||||
const grants = accessGroupDetails ?? [];
|
||||
const groupNamesFor = (model: string): string[] =>
|
||||
grants.filter((g) => g.models.includes(model)).map((g) => g.access_group_name);
|
||||
const viaGroups = (model: string): string => {
|
||||
const names = groupNamesFor(model);
|
||||
return names.length > 0 ? describeGroups(names) : "an access group";
|
||||
};
|
||||
|
||||
const allProxy = models.length === 0 || models.includes(ALL_PROXY_MODELS);
|
||||
const directModels = allProxy ? [] : models.filter((m) => m !== NO_DEFAULT_MODELS);
|
||||
const groupModels = [...new Set(grants.length > 0 ? grants.flatMap((g) => g.models) : accessGroupModels)].filter(
|
||||
(m) => !directModels.includes(m),
|
||||
);
|
||||
|
||||
const allProxyBadge: TeamModelBadge = {
|
||||
label: "All proxy models",
|
||||
kind: "all-proxy",
|
||||
tooltip: models.includes(ALL_PROXY_MODELS)
|
||||
? "Granted by the All Proxy Models entry in the team's model list"
|
||||
: "The team's model list is empty, so it can access every model on the proxy",
|
||||
};
|
||||
const noDefaultBadge: TeamModelBadge = {
|
||||
label: "No default models",
|
||||
kind: "no-default",
|
||||
tooltip: "No models are granted directly. Access comes only from access groups",
|
||||
};
|
||||
const headBadge = (): TeamModelBadge[] => {
|
||||
if (allProxy) return [allProxyBadge];
|
||||
if (models.includes(NO_DEFAULT_MODELS)) return [noDefaultBadge];
|
||||
return [];
|
||||
};
|
||||
|
||||
return [
|
||||
...headBadge(),
|
||||
...directModels.map(
|
||||
(m): TeamModelBadge => ({
|
||||
label: m,
|
||||
kind: "direct",
|
||||
tooltip:
|
||||
groupNamesFor(m).length > 0
|
||||
? `Granted directly in the team's model list, and also via ${viaGroups(m)}`
|
||||
: "Granted directly in the team's model list",
|
||||
}),
|
||||
),
|
||||
...groupModels.map(
|
||||
(m): TeamModelBadge => ({
|
||||
label: m,
|
||||
kind: "access-group",
|
||||
tooltip: `Granted via ${viaGroups(m)}`,
|
||||
}),
|
||||
),
|
||||
];
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue