scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-22 00:31:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 51

feat: add ADEPT deterministic template routing for adaptive SLM delegation in agentic workflows

Browse source
This commit is contained in:
UltimateGamingShack 2026-09-10 01:26:31 +05:30
parent 252c71c0b2
commit 64d7db125d
28 changed files with 3861 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

163
litellm/router.py
View file

@ -300,6 +300,9 @@ if TYPE_CHECKING:
from litellm.router_strategy.adaptive_router.adaptive_router import (
AdaptiveRouter,
)
from litellm.router_strategy.adept_router.adept_router import (
AdeptRouter,
)
from litellm.router_strategy.auto_router.auto_router import (
AutoRouter,
PreRoutingHookResponse,
@ -924,6 +927,7 @@ class Router:
self.adaptive_routers: dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]] = {}
self.quality_routers: dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]] = {}
self.routing_plugins: list[RoutingPlugin] = list(plugins) if plugins else []
self.adept_routers: dict[str, AdeptRouter] = {} # mutable-ok: registry filled per deploy sync
# Initialize model_group_alias early since it's used in set_model_list
self.model_group_alias: dict[str, str | RouterModelGroupAliasItem] = (
@ -9199,6 +9203,26 @@ class Router:
if self._unregister_pre_routing_strategy(self.adaptive_routers, model_name, tags):
self._sync_adaptive_router_hooks()
def _release_adept_router_for_deleted_deployment(self, item: object) -> None:
"""Drop an ADEPT deployment's in-memory router, unregister its callbacks, and release
its shared PG client when no sibling deployment still references the same URL."""
model_name: Final = (
item.model_name
if isinstance(item, Deployment)
else (item.get("model_name") if isinstance(item, dict) else None)
)
if model_name is None:
return
removed_adept: Final = self.adept_routers.pop(model_name, None)
if removed_adept is None:
return
litellm.logging_callback_manager.remove_callback_from_all_lists(removed_adept)
removed_adept.close()
if not any(r.pg_url == removed_adept.pg_url for r in self.adept_routers.values()):
from litellm.router_strategy.adept_router.store.implementation.prisma import schedule_disconnect
schedule_disconnect(removed_adept.pg_url)
def _finalize_adaptive_router_if_configured(self) -> None:
"""Locate every adaptive-router deployment in the finalized model_list and
build an AdaptiveRouter for each. Safe no-op when none are configured.
@ -9374,6 +9398,97 @@ class Router:
strategy_label="Quality-router",
)
def _is_adept_router_deployment(self, litellm_params: LiteLLM_Params) -> bool:
"""Returns True when the model prefix is 'adept/'."""
return litellm_params.model.startswith("adept/")
def init_adept_router_deployment(self, deployment: Deployment) -> None:
"""Initialize an ADEPT router deployment and register it in self.adept_routers.
Idempotent: repeat calls with identical params are a no-op; a param change rebuilds
the in-memory router so the new value takes effect without a proxy restart.
"""
from litellm.router_strategy.adept_router.adept_router import AdeptRouter
from litellm.router_strategy.adept_router.config import (
DEFAULT_CONVERSATIONS_THRESHOLD,
)
lp: Final = deployment.litellm_params
default_model: Final = lp.adept_router_default_model
if default_model is None:
raise ValueError("adept_router_default_model is required for ADEPT router deployments.")
if not lp.adept_router_pg_host:
raise ValueError(
"adept_router_pg_host is required for ADEPT router deployments. "
"Configure a PostgreSQL database so the trainer pipeline can access the data."
)
from urllib.parse import quote_plus
password: Final = lp.adept_router_pg_password or ""
port: Final = lp.adept_router_pg_port or 5432
user: Final = lp.adept_router_pg_user or ""
database: Final = lp.adept_router_pg_database or ""
ssl_mode: Final = lp.adept_router_pg_ssl_mode or "prefer"
pg_url: Final = f"postgresql://{quote_plus(user)}:{quote_plus(password)}@{lp.adept_router_pg_host}:{port}/{database}?sslmode={ssl_mode}"
trainer_url: Final = lp.adept_router_trainer_url
if trainer_url:
from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
try:
validate_url(trainer_url)
except SSRFError as ssrf_err:
raise ValueError(
f"adept_router_trainer_url {trainer_url!r} rejected by SSRF guard: {ssrf_err}. "
"Add the host to `general_settings.user_url_allowed_hosts` if it is an internal "
"trainer the operator has explicitly cleared."
) from ssrf_err
threshold: Final = lp.adept_router_conversations_threshold or DEFAULT_CONVERSATIONS_THRESHOLD
tag_prefix: Final = lp.adept_router_tag_prefix or ""
existing: Final = self.adept_routers.get(deployment.model_name)
if existing is not None:
params_changed: Final = (
existing.default_model != default_model
or existing.pg_url != pg_url
or existing.template_router.trainer_url != trainer_url
or existing.template_router.conversations_threshold != threshold
or existing.template_router.tag_prefix != tag_prefix
)
if not params_changed:
verbose_router_logger.debug(
"AdeptRouter: '%s' already registered with matching params — skipping re-init.",
deployment.model_name,
)
return
verbose_router_logger.info(
"AdeptRouter: '%s' params changed — rebuilding in-memory router.", deployment.model_name
)
litellm.logging_callback_manager.remove_callback_from_all_lists(existing)
existing.close()
if existing.pg_url != pg_url and not any(
name != deployment.model_name and r.pg_url == existing.pg_url for name, r in self.adept_routers.items()
):
from litellm.router_strategy.adept_router.store.implementation.prisma import schedule_disconnect
schedule_disconnect(existing.pg_url)
adept_router: Final = AdeptRouter(
model_name=deployment.model_name,
default_model=default_model,
litellm_router_instance=self,
pg_url=pg_url,
tag_prefix=tag_prefix,
conversations_threshold=threshold,
trainer_url=trainer_url,
seed_config=lp.adept_router_seed_config,
)
self.adept_routers[deployment.model_name] = adept_router
litellm.logging_callback_manager.add_litellm_callback(adept_router)
def deployment_is_active_for_environment(self, deployment: Deployment) -> bool:
"""
Function to check if a llm deployment is active for a given environment. Allows using the same config.yaml across multople environments
@ -9403,6 +9518,11 @@ class Router:
self.complexity_routers = {}
self.auto_routers = {}
self._provider_unresolved_deployments = ()
for stale_adept in tuple(self.adept_routers.values()):
litellm.logging_callback_manager.remove_callback_from_all_lists(stale_adept)
stale_adept.close()
pre_reload_adept_urls: Final = frozenset(r.pg_url for r in self.adept_routers.values())
self.adept_routers = {} # mutable-ok: registry reset on model_list reload; populated incrementally per deployment sync
self._invalidate_model_group_info_cache()
self._invalidate_access_groups_cache()
# we add api_base/api_key each model so load balancing between azure/gpt on api_base1 and api_base2 works
@ -9480,6 +9600,15 @@ class Router:
# deployments have been registered.
self._finalize_adaptive_router_if_configured()
# Deferred until after re-init so we never disconnect a client a rebuilt deployment reused.
post_reload_adept_urls: Final = frozenset(r.pg_url for r in self.adept_routers.values())
orphaned_adept_urls: Final = pre_reload_adept_urls - post_reload_adept_urls
if orphaned_adept_urls:
from litellm.router_strategy.adept_router.store.implementation.prisma import schedule_disconnect
for orphaned_url in orphaned_adept_urls:
schedule_disconnect(orphaned_url)
def _add_deployment(self, deployment: Deployment) -> Deployment:
import os
@ -9499,9 +9628,10 @@ class Router:
if split_litellm_model in litellm._known_custom_logger_compatible_callbacks:
is_prompt_management_model = True
if is_prompt_management_model:
# For prompt management models, skip LLM provider validation
# The actual model will be resolved at runtime from the prompt file
# ADEPT deployments skip standard get_llm_provider validation; init_adept_router_deployment handles them.
is_adept_router_model: Final = self._is_adept_router_deployment(litellm_params=deployment.litellm_params)
if is_prompt_management_model or is_adept_router_model:
_model = litellm_model
custom_llm_provider = None
dynamic_api_key = None
@ -9606,6 +9736,9 @@ class Router:
if self._is_quality_router_deployment(litellm_params=deployment.litellm_params):
self.init_quality_router_deployment(deployment=deployment)
if self._is_adept_router_deployment(litellm_params=deployment.litellm_params):
self.init_adept_router_deployment(deployment=deployment)
return deployment
def _initialize_deployment_for_pass_through(self, deployment: Deployment, custom_llm_provider: str):
@ -10097,6 +10230,7 @@ class Router:
"the deployment is out of the model_list and its indices are repaired",
id,
)
self._release_adept_router_for_deleted_deployment(item)
return item
else:
return None
@ -13511,6 +13645,14 @@ class Router:
model=registered_model_name, request_kwargs=request_kwargs
)
if selected_strategy is None:
if registered_model_name in self.adept_routers:
return await self.adept_routers[registered_model_name].async_pre_routing_hook(
model=registered_model_name,
request_kwargs=request_kwargs,
messages=messages,
input=input,
specific_deployment=specific_deployment,
)
self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
self._stamp_or_clear_metadata_key(
request_kwargs=request_kwargs, key=SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, value=None
@ -13617,6 +13759,15 @@ class Router:
if newly_forwarded:
request_kwargs.update(((_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, tuple(key for key, _ in newly_forwarded)),))
if registered_model_name in self.adept_routers:
return await self.adept_routers[registered_model_name].async_pre_routing_hook(
model=registered_model_name,
request_kwargs=request_kwargs,
messages=messages,
input=input,
specific_deployment=specific_deployment,
)
return pre_routing_hook_response
def _forwardable_alias_marker_params(
@ -13790,6 +13941,12 @@ class Router:
"usage-based-routing-v2, cost-based-routing, latency-based-routing, least-busy), "
"or remove `plugins` from the Router config."
)
if self.adept_routers:
raise ValueError(
"An ADEPT router is configured but this call resolved to the synchronous "
"deployment-selection path, which never runs the async pre-routing hook. "
"Use an async Router method."
)
# users need to explicitly call a specific deployment, by setting `specific_deployment = True` as completion()/embedding() kwarg
# When this was no explicit we had several issues with fallbacks timing out

0
litellm/router_strategy/adept_router/__init__.py Normal file
View file

308
litellm/router_strategy/adept_router/adept_router.py Normal file
View file

@ -0,0 +1,308 @@
"""
ADEPT (Adaptive Deployment via Prompt Templates) Router.
Extracts a structural skeleton from each single-turn prompt, hashes it with the system prompt
for per-tool isolation, and routes to a task-specific SLM once one has been trained. Until then,
traffic falls back to the default model while conversations accumulate as training data.
"""
import asyncio
import datetime
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Final, TypeAlias
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm._logging import verbose_router_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.router_strategy.adept_router.config import DEFAULT_CONVERSATIONS_THRESHOLD
from litellm.router_strategy.adept_router.template.implementation.adept_template_router import (
AdeptTemplateRouter,
)
from litellm.types.utils import ModelResponse, Usage
if TYPE_CHECKING:
from litellm.router import Router
from litellm.types.router import PreRoutingHookResponse
else:
Router: Final = object
PreRoutingHookResponse: Final = object
class _MessageContentBlock(BaseModel):
model_config = ConfigDict(extra="ignore")
type: str | None = None
text: str | None = None
class _UsageEnvelope(BaseModel):
model_config = ConfigDict(extra="ignore")
usage: Usage | None = None
_MessageList: TypeAlias = list[dict[str, object]]
_MESSAGES_ADAPTER: Final = TypeAdapter(_MessageList)
_CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(list[_MessageContentBlock])
_METADATA_ADAPTER: Final = TypeAdapter(dict[str, object])
class AdeptRouter(CustomLogger):
def __init__(
self,
model_name: str,
default_model: str,
litellm_router_instance: "Router",
pg_url: str,
tag_prefix: str = "",
conversations_threshold: int = DEFAULT_CONVERSATIONS_THRESHOLD,
trainer_url: str | None = None,
seed_config: Sequence[Mapping[str, object]] | None = None,
) -> None:
self.model_name = model_name
self.default_model = default_model
self.pg_url = pg_url
self.litellm_router_instance = litellm_router_instance
self.template_router = AdeptTemplateRouter(
model_name=model_name,
litellm_router_instance=litellm_router_instance,
pg_url=pg_url,
tag_prefix=tag_prefix,
conversations_threshold=conversations_threshold,
trainer_url=trainer_url,
)
self._seed_config = seed_config
self._seeded = not seed_config
self._seed_lock = asyncio.Lock()
self._seed_task: asyncio.Task[None] | None = None # mutable-ok: task handle rotates on restart
def _kick_off_seed(self) -> None:
if self._seeded:
return
existing: Final = self._seed_task
if existing is not None and not existing.done():
return
try:
loop: Final = asyncio.get_running_loop()
except RuntimeError:
return
self._seed_task = loop.create_task(self._run_seed())
def close(self) -> None:
"""Cancel background tasks so a retired router cannot mutate state or write to the DB."""
if self._seed_task is not None:
self._seed_task.cancel()
self._seed_task = None
self.template_router.stop_refresh()
async def _run_seed(self) -> None:
async with self._seed_lock:
if self._seeded:
return
for entry in self._seed_config or ():
description = entry.get("description", "")
target_model = entry.get("target_model", self.default_model)
if not description:
verbose_router_logger.warning(
"AdeptRouter: seed_config entry missing 'description', skipping: %s", str(entry)[:100]
)
continue
if await self.template_router.seed_template(str(description), str(target_model)):
verbose_router_logger.info("AdeptRouter: seeded template for target_model=%s", target_model)
self._seeded = True
async def async_pre_routing_hook(
self,
model: str,
request_kwargs: Mapping[str, object],
messages: _MessageList | None = None,
input: str | Sequence[object] | None = None,
specific_deployment: bool | None = False,
) -> "PreRoutingHookResponse | None":
from litellm.types.router import PreRoutingHookResponse
if messages is None:
return None
self._kick_off_seed()
message_content: Final = self._extract_user_text(messages)
if not message_content:
await self._authorize_routed_model(request_kwargs, self.default_model)
return PreRoutingHookResponse(model=self.default_model, messages=messages)
system_prompt: Final = self._extract_system_prompt(messages)
template_match: Final = await self.template_router.route(message_content, system_prompt)
target: Final = template_match.get("target_model") if template_match is not None else None
routed_model: Final = target or self.default_model
routed_to_slm: Final = bool(target)
await self._authorize_routed_model(request_kwargs, routed_model)
if template_match is not None:
verbose_router_logger.info(
"AdeptRouter: matched template %s, routing to %s",
template_match.get("template_id"),
routed_model,
)
else:
verbose_router_logger.info("AdeptRouter: no template match, falling back to %s", self.default_model)
for md_key in ("metadata", "litellm_metadata"):
candidate = request_kwargs.get(md_key)
if isinstance(candidate, dict):
candidate["adept_routed_to_slm"] = routed_to_slm
break
return PreRoutingHookResponse(model=routed_model, messages=messages)
async def _authorize_routed_model(self, request_kwargs: Mapping[str, object], routed_model: str) -> None:
"""Re-run proxy model-access check against the swapped target so a caller cannot be silently
upgraded from the ADEPT alias to a SLM they lack access to."""
if routed_model == self.model_name:
return
auth_from_litellm_params: Final = self._read_request_metadata(
request_kwargs.get("litellm_params"), "user_api_key_auth"
)
auth_obj: Final = (
auth_from_litellm_params
if auth_from_litellm_params is not None
else self._extract_user_api_key_auth(request_kwargs)
)
if auth_obj is None:
return
try:
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
except ImportError:
return
if not isinstance(auth_obj, UserAPIKeyAuth):
return
await can_key_call_resolved_model(
model=routed_model,
llm_model_list=self.litellm_router_instance.model_list,
valid_token=auth_obj,
llm_router=self.litellm_router_instance,
)
@staticmethod
def _extract_user_api_key_auth(request_kwargs: Mapping[str, object]) -> object:
for md_key in ("metadata", "litellm_metadata"):
candidate = request_kwargs.get(md_key)
if isinstance(candidate, Mapping):
value = candidate.get("user_api_key_auth")
if value is not None:
return value
return None
@staticmethod
def _read_request_metadata(litellm_params: object, key: str) -> object:
try:
params: Final = _METADATA_ADAPTER.validate_python(litellm_params)
except ValidationError:
return None
for md_key in ("metadata", "litellm_metadata"):
try:
nested = _METADATA_ADAPTER.validate_python(params.get(md_key)) # rebind-ok: loop rebinds per iteration
except ValidationError:
continue
value = nested.get(key) # rebind-ok: loop rebinds per iteration
if value is not None:
return value
return None
async def async_log_success_event(
self,
kwargs: Mapping[str, object],
response_obj: ModelResponse,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None:
# Callback fires for every request; gate on model_group to only log our own.
lp_raw: Final = kwargs.get("litellm_params")
request_model_group: Final = self._read_request_metadata(lp_raw, "model_group")
if request_model_group != self.model_name:
return
try:
messages: Final = _MESSAGES_ADAPTER.validate_python(kwargs.get("messages"))
except ValidationError:
return
if not messages:
return
if messages[-1].get("role") == "tool":
return
try:
prompt_text: Final = self._extract_user_text(messages)
if not prompt_text:
return
usage: Final = _UsageEnvelope.model_validate(response_obj, from_attributes=True).usage
if usage is None:
return
response_content: Final = self._response_text(response_obj)
if response_content is None:
return
token_usage: Final[dict[str, object]] = { # mutable-ok: JSON payload persisted to a Postgres JSON column
"prompt_tokens": usage.prompt_tokens,
"completion_tokens": usage.completion_tokens,
"total_tokens": usage.total_tokens,
}
cost_raw: Final = kwargs.get("response_cost")
cost_usd: Final = cost_raw if isinstance(cost_raw, (int, float)) else None
latency_ms: Final = (end_time - start_time).total_seconds() * 1000
system_prompt: Final = self._extract_system_prompt(messages)
routed_to_slm_raw: Final = self._read_request_metadata(lp_raw, "adept_routed_to_slm")
routed_to_slm: Final = routed_to_slm_raw if isinstance(routed_to_slm_raw, bool) else None
actual_model: Final = str(kwargs.get("model", "unknown"))
await self.template_router.store_conversation(
prompt_text,
response_content,
actual_model,
token_usage,
cost_usd,
latency_ms,
system_prompt,
routed_to_slm,
)
verbose_router_logger.info("AdeptRouter: stored interaction.")
except (AttributeError, KeyError, TypeError, ValueError):
verbose_router_logger.exception("AdeptRouter: failed to log success event")
@staticmethod
def _response_text(response_obj: ModelResponse) -> str | None:
if not response_obj.choices:
return None
choice: Final = response_obj.choices[0]
return choice.message.content
@staticmethod
def _content_to_text(content: object) -> str:
if content is None:
return ""
if isinstance(content, str):
return content
try:
blocks: Final = _CONTENT_BLOCKS_ADAPTER.validate_python(content)
except ValidationError:
return str(content)
return " ".join(block.text or "" for block in blocks if block.type == "text")
@staticmethod
def _extract_system_prompt(messages: Sequence[Mapping[str, object]]) -> str | None:
for msg in messages:
if msg.get("role") == "system":
content = msg.get("content")
return str(content) if content else None
return None
@staticmethod
def _extract_user_text(messages: Sequence[Mapping[str, object]]) -> str:
for msg in reversed(messages):
if msg.get("role") == "user":
return AdeptRouter._content_to_text(msg.get("content"))
return ""

3
litellm/router_strategy/adept_router/config.py Normal file
View file

@ -0,0 +1,3 @@
from typing import Final
DEFAULT_CONVERSATIONS_THRESHOLD: Final = 1000

0
litellm/router_strategy/adept_router/store/__init__.py Normal file
View file

0
litellm/router_strategy/adept_router/store/implementation/__init__.py Normal file
View file

326
litellm/router_strategy/adept_router/store/implementation/prisma.py Normal file
View file

@ -0,0 +1,326 @@
import asyncio
import json
from collections.abc import AsyncGenerator, Mapping, Sequence
from contextlib import asynccontextmanager
from datetime import datetime
from typing import Final
from urllib.parse import urlsplit, urlunsplit
from prisma import Prisma
from prisma.errors import PrismaError
from prisma.types import DatasourceOverride
from pydantic import BaseModel, ConfigDict, TypeAdapter, field_validator
from litellm._logging import verbose_router_logger
from litellm.router_strategy.adept_router.store.store_template import (
AdeptTemplateStore,
StoredTemplate,
)
_JSON_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_DRAIN_TIMEOUT_SECONDS: Final = 30.0
class _IdRow(BaseModel):
id: str
class _CountRow(BaseModel):
c: int
class _TemplateRow(BaseModel):
model_config = ConfigDict(extra="ignore")
id: str
template: str
template_hash: str | None = None
router_id: str
target_model: str | None = None
additional_information: Mapping[str, object] | None = None
created_at: datetime | None = None
@field_validator("additional_information", mode="before")
@classmethod
def _coerce_json(cls, value: object) -> object:
if isinstance(value, str):
return _JSON_ADAPTER.validate_json(value)
return value
def _redact_url(url: str) -> str:
"""Return the URL with any embedded password replaced by '***'."""
try:
split: Final = urlsplit(url)
except ValueError:
return "<redacted>"
if split.password is None:
return url
user: Final = split.username or ""
host: Final = split.hostname or ""
port_suffix: Final = f":{split.port}" if split.port is not None else ""
netloc: Final = f"{user}:***@{host}{port_suffix}" if user else f":***@{host}{port_suffix}"
return urlunsplit((split.scheme, netloc, split.path, split.query, split.fragment))
async def _create_tables(client: Prisma) -> None:
await client.execute_raw(
"CREATE TABLE IF NOT EXISTS templates ("
"id TEXT PRIMARY KEY, template TEXT NOT NULL, template_hash VARCHAR(64) NOT NULL, "
"router_id TEXT NOT NULL, target_model TEXT, additional_information JSONB, "
"created_at TIMESTAMPTZ NOT NULL DEFAULT now())"
)
await client.execute_raw(
"CREATE UNIQUE INDEX IF NOT EXISTS ix_templates_router_hash ON templates (router_id, template_hash)"
)
await client.execute_raw(
"CREATE TABLE IF NOT EXISTS conversations ("
"id SERIAL PRIMARY KEY, template_id TEXT NOT NULL REFERENCES templates(id), "
"prompt TEXT NOT NULL, response TEXT NOT NULL, additional_information JSONB, "
"created_at TIMESTAMPTZ NOT NULL DEFAULT now())"
)
await client.execute_raw("CREATE INDEX IF NOT EXISTS ix_conversations_template_id ON conversations (template_id)")
def _json_or_none(payload: Mapping[str, object] | None) -> str | None:
return json.dumps(payload) if payload is not None else None
class _ClientHandle:
"""Wraps a shared Prisma client with an in-flight refcount so `disconnect_client` can drain
active operations before tearing the socket down."""
__slots__ = ("_closed", "_drained", "_inflight", "_lock", "client")
def __init__(self, client: Prisma) -> None:
self.client: Final = client
self._lock: Final = asyncio.Lock()
self._inflight = 0 # mutable-ok: refcount for graceful drain
self._drained: Final = asyncio.Event()
self._drained.set()
self._closed = False # mutable-ok: one-way close flag rejecting new borrows
@asynccontextmanager
async def borrow(self) -> AsyncGenerator[Prisma]:
async with self._lock:
if self._closed:
raise PrismaError("Prisma client has been disconnected")
self._inflight += 1 # rebind-ok: refcount increment guarded by _lock
self._drained.clear()
try:
yield self.client
finally:
async with self._lock:
self._inflight -= 1 # rebind-ok: refcount decrement guarded by _lock
if self._inflight == 0:
self._drained.set()
async def close(self) -> None:
async with self._lock:
self._closed = True # rebind-ok: one-way transition guarded by _lock
already_drained: Final = self._inflight == 0
if already_drained:
return
try:
await asyncio.wait_for(self._drained.wait(), timeout=_DRAIN_TIMEOUT_SECONDS)
except asyncio.TimeoutError:
verbose_router_logger.warning(
"AdeptPrismaRepo: drain timed out with %s in-flight operations; disconnecting anyway.",
self._inflight,
)
_CLIENTS: Final[dict[str, _ClientHandle]] = {} # mutable-ok: connection registry keyed by database URL
_REGISTRY_LOCK: Final = asyncio.Lock()
async def _get_handle(db_url: str) -> _ClientHandle:
cached: Final = _CLIENTS.get(db_url)
if cached is not None:
return cached
async with _REGISTRY_LOCK:
existing: Final = _CLIENTS.get(db_url)
if existing is not None:
return existing
client: Final = Prisma(datasource=DatasourceOverride(url=db_url))
await client.connect()
await _create_tables(client)
handle: Final = _ClientHandle(client)
_CLIENTS[db_url] = handle
return handle
async def disconnect_client(db_url: str) -> None:
async with _REGISTRY_LOCK:
existing: Final = _CLIENTS.pop(db_url, None)
if existing is None:
return
await existing.close()
try:
await existing.client.disconnect()
except Exception as e: # noqa: BLE001 # best-effort: a hung PG socket must not block the rebuild path
verbose_router_logger.warning("AdeptPrismaRepo: disconnect failed for %s: %s", _redact_url(db_url), e)
def schedule_disconnect(db_url: str) -> None:
try:
loop: Final = asyncio.get_running_loop()
except RuntimeError:
return
loop.create_task(disconnect_client(db_url))
class AdeptPrismaRepo(AdeptTemplateStore):
def __init__(self, db_url: str) -> None:
if not db_url:
raise ValueError(
"A PostgreSQL connection URL is required. Example: postgresql://user:password@host:5432/dbname"
)
self._db_url = db_url
async def match_by_hash(self, template_hash: str, router_id: str) -> str | None:
try:
handle: Final = await _get_handle(self._db_url)
async with handle.borrow() as client:
rows: Final = await client.query_raw(
"SELECT id FROM templates WHERE router_id = $1 AND template_hash = $2 LIMIT 1",
router_id,
template_hash,
model=_IdRow,
)
except PrismaError as e:
verbose_router_logger.error("Error matching template by hash: %s", e)
return None
else:
return rows[0].id if rows else None
async def load_all_for_router(self, router_id: str, limit: int) -> Sequence[StoredTemplate]:
try:
handle: Final = await _get_handle(self._db_url)
async with handle.borrow() as client:
rows: Final = await client.query_raw(
"SELECT id, template, template_hash, router_id, target_model, additional_information, created_at "
"FROM templates WHERE router_id = $1 ORDER BY created_at DESC LIMIT $2",
router_id,
limit,
model=_TemplateRow,
)
except PrismaError as e:
verbose_router_logger.error("Error loading templates for router %s: %s", router_id, e)
return ()
return tuple(
StoredTemplate(
id=row.id,
template=row.template,
template_hash=row.template_hash,
router_id=row.router_id,
target_model=row.target_model,
additional_information=row.additional_information,
created_at=row.created_at,
)
for row in rows
)
async def store_conversation(
self,
prompt: str,
response: str,
template_id: str | None = None,
additional_information: Mapping[str, object] | None = None,
) -> bool:
if not template_id:
verbose_router_logger.error("template_id is required to store a conversation.")
return False
try:
handle: Final = await _get_handle(self._db_url)
async with handle.borrow() as client:
await client.execute_raw(
"INSERT INTO conversations (template_id, prompt, response, additional_information) "
"VALUES ($1, $2, $3, $4::jsonb)",
template_id,
prompt,
response,
_json_or_none(additional_information),
)
except PrismaError as e:
verbose_router_logger.error("Error storing conversation: %s", e)
return False
return True
async def store_template(
self,
template_id: str,
template: str,
template_hash: str,
target_model: str,
router_id: str,
additional_information: Mapping[str, object] | None = None,
) -> str | None:
"""Insert a new template row, returning the surviving id (ours or a concurrent insert's).
ON CONFLICT DO NOTHING on the (router_id, template_hash) unique index makes concurrent
inserts safe: the loser no-ops and we re-read the winner.
"""
try:
handle: Final = await _get_handle(self._db_url)
async with handle.borrow() as client:
await client.execute_raw(
"INSERT INTO templates (id, template, template_hash, target_model, router_id, additional_information) "
"VALUES ($1, $2, $3, $4, $5, $6::jsonb) ON CONFLICT (router_id, template_hash) DO NOTHING",
template_id,
template,
template_hash,
target_model,
router_id,
_json_or_none(additional_information),
)
rows: Final = await client.query_raw(
"SELECT id FROM templates WHERE router_id = $1 AND template_hash = $2 LIMIT 1",
router_id,
template_hash,
model=_IdRow,
)
except PrismaError as e:
verbose_router_logger.error("AdeptRouter: error storing template: %s", e)
return None
return rows[0].id if rows else template_id
async def get_template(self, template_id: str) -> StoredTemplate | None:
try:
handle: Final = await _get_handle(self._db_url)
async with handle.borrow() as client:
rows: Final = await client.query_raw(
"SELECT id, template, template_hash, router_id, target_model, additional_information, created_at "
"FROM templates WHERE id = $1 LIMIT 1",
template_id,
model=_TemplateRow,
)
except PrismaError as e:
verbose_router_logger.error("Error retrieving template: %s", e)
return None
if not rows:
return None
row: Final = rows[0]
return StoredTemplate(
id=row.id,
template=row.template,
template_hash=row.template_hash,
router_id=row.router_id,
target_model=row.target_model,
additional_information=row.additional_information,
created_at=row.created_at,
)
async def count_conversation_by_template_id(self, template_id: str) -> int | None:
try:
handle: Final = await _get_handle(self._db_url)
async with handle.borrow() as client:
rows: Final = await client.query_raw(
"SELECT count(*)::int AS c FROM conversations WHERE template_id = $1",
template_id,
model=_CountRow,
)
except PrismaError as e:
verbose_router_logger.error("Error counting conversations: %s", e)
return None
return rows[0].c if rows else 0

71
litellm/router_strategy/adept_router/store/store_template.py Normal file
View file

@ -0,0 +1,71 @@
from abc import ABC, abstractmethod
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime
@dataclass(frozen=True, slots=True)
class StoredTemplate:
"""A template row read back from the store."""
id: str
template: str
template_hash: str | None
router_id: str
target_model: str | None
additional_information: Mapping[str, object] | None
created_at: datetime | None
class AdeptTemplateStore(ABC):
"""Abstract interface for storing and retrieving ADEPT prompt templates and conversations.
Implementations talk to the user's own database, so every method is async.
"""
@abstractmethod
async def match_by_hash(self, template_hash: str, router_id: str) -> str | None:
"""Look up a template ID by the SHA-256 hash of its masked template string."""
...
@abstractmethod
async def load_all_for_router(self, router_id: str, limit: int) -> Sequence[StoredTemplate]:
"""Bulk-load recent templates for a router so the request path can serve from memory."""
...
@abstractmethod
async def store_conversation(
self,
prompt: str,
response: str,
template_id: str | None = None,
additional_information: Mapping[str, object] | None = None,
) -> bool:
"""Store a prompt-response pair linked to a template."""
...
@abstractmethod
async def store_template(
self,
template_id: str,
template: str,
template_hash: str,
target_model: str,
router_id: str,
additional_information: Mapping[str, object] | None = None,
) -> str | None:
"""
Store a new template row. Returns the surviving template_id (ours or a concurrent
insert's) so the caller can use it without a follow-up query.
"""
...
@abstractmethod
async def get_template(self, template_id: str) -> StoredTemplate | None:
"""Retrieve metadata for a specific template by ID."""
...
@abstractmethod
async def count_conversation_by_template_id(self, template_id: str) -> int | None:
"""Count conversations associated with a template."""
...

0
litellm/router_strategy/adept_router/template/__init__.py Normal file
View file

0
litellm/router_strategy/adept_router/template/implementation/__init__.py Normal file
View file

305
litellm/router_strategy/adept_router/template/implementation/adept_template_router.py Normal file
View file

@ -0,0 +1,305 @@
import asyncio
import hashlib
import re
import time
from collections import OrderedDict
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Final, TypeAlias
from uuid import uuid4
import httpx
from litellm._logging import verbose_router_logger
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.router_strategy.adept_router.config import DEFAULT_CONVERSATIONS_THRESHOLD
from litellm.router_strategy.adept_router.store.store_template import (
AdeptTemplateStore,
StoredTemplate,
)
from litellm.router_strategy.adept_router.template.router_template import (
AdeptTemplateMatch,
BaseTemplateRouter,
)
from litellm.types.llms.custom_http import httpxSpecialProvider
if TYPE_CHECKING:
from litellm.router import Router
else:
Router: Final = object
_TEMPLATE_CACHE_MAX_SIZE: Final = 1024
_TEMPLATE_CACHE_TTL_SECONDS: Final = 60.0
_REFRESH_INTERVAL_SECONDS: Final = 60.0
_TRAINER_HTTP_TIMEOUT_SECONDS: Final = 10.0
_TemplateCache: TypeAlias = OrderedDict[tuple[str, str], tuple[float, StoredTemplate]]
class AdeptTemplateRouter(BaseTemplateRouter):
ID_RE = re.compile(r"\b[A-Z]{2,}-\d{3,}\b")
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
URL_RE = re.compile(r"https?://\S+|www\.\S+")
# UUID must be masked before NUM — UUID hex digits partially match NUM_RE.
UUID_RE = re.compile(
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b"
)
NUM_RE = re.compile(r"\b\d{1,4}([/-]\d{1,2}([/-]\d{1,4})?)?\b")
NORMALIZE_RE = re.compile(r"\s+")
def __init__(
self,
model_name: str,
litellm_router_instance: "Router",
pg_url: str,
tag_prefix: str = "",
conversations_threshold: int = DEFAULT_CONVERSATIONS_THRESHOLD,
trainer_url: str | None = None,
) -> None:
from litellm.router_strategy.adept_router.store.implementation.prisma import (
AdeptPrismaRepo,
)
self.model_name = model_name
self.litellm_router_instance = litellm_router_instance
self.tag_prefix = tag_prefix
self.conversations_threshold = conversations_threshold
self.trainer_url = trainer_url
self._router_id_cache: str | None = None
self._template_cache: _TemplateCache = OrderedDict() # mutable-ok: LRU cache, bounded and TTL-gated
self._refresh_task: asyncio.Task[None] | None = None # mutable-ok: task handle rotates on restart
escaped_prefix: Final = re.escape(self.tag_prefix)
self.TAG_CONTENT_RE = re.compile(
r"<" + escaped_prefix + r"([a-zA-Z0-9_ ]+)>([^<]*)</" + escaped_prefix + r"\1>"
)
self.TAG_REPLACEMENT = r"<" + escaped_prefix + r"\1></" + escaped_prefix + r"\1>"
self.template_store: AdeptTemplateStore = AdeptPrismaRepo(pg_url)
def get_router_id(self) -> str:
if self._router_id_cache is None:
self._router_id_cache = self.litellm_router_instance.get_model_ids(model_name=self.model_name)[0]
return self._router_id_cache
def _normalize_text(self, text: str) -> str:
return self.NORMALIZE_RE.sub(" ", text.strip())
def _mask_text(self, text: str) -> str:
ids: Final = self.ID_RE.sub("{ID}", text)
emails: Final = self.EMAIL_RE.sub("{EMAIL}", ids)
urls: Final = self.URL_RE.sub("{URL}", emails)
uuids: Final = self.UUID_RE.sub("{UUID}", urls)
return self.NUM_RE.sub("{NUM}", uuids)
def _extract_tag_content(self, text: str) -> Sequence[tuple[str, str]]:
return tuple((match.group(1), match.group(2)) for match in self.TAG_CONTENT_RE.finditer(text))
def _extract_template(self, prompt: str) -> tuple[str, Sequence[tuple[str, str]]]:
normalized: Final = self._normalize_text(prompt)
extractions: Final = self._extract_tag_content(normalized)
skeleton: Final = self.TAG_CONTENT_RE.sub(self.TAG_REPLACEMENT, normalized)
masked_template: Final = self._mask_text(skeleton)
verbose_router_logger.debug("Extracted template with %s tag(s)", len(extractions))
return masked_template, extractions
@staticmethod
def _hash_template(masked_template: str, system_prompt: str | None = None) -> str:
# Prepending the system prompt isolates two tools with identical user-message
# structure but different task definitions.
if system_prompt:
normalized_sys: Final = re.sub(r"\s+", " ", system_prompt.strip())
payload: Final = normalized_sys + " | " + masked_template
return hashlib.sha256(payload.encode()).hexdigest()
return hashlib.sha256(masked_template.encode()).hexdigest()
async def seed_template(self, description: str, target_model: str) -> bool:
masked: Final = self._mask_text(self._normalize_text(description))
template_hash: Final = self._hash_template(masked)
router_id: Final = self.get_router_id()
if await self.template_store.match_by_hash(template_hash, router_id) is not None:
return False
await self.template_store.store_template(
template_id=str(uuid4()),
template=masked,
template_hash=template_hash,
target_model=target_model,
router_id=router_id,
)
return True
async def route(self, prompt: str, system_prompt: str | None = None) -> AdeptTemplateMatch | None:
try:
self._ensure_refresh_running()
masked_template, _ = self._extract_template(prompt)
template_hash: Final = self._hash_template(masked_template, system_prompt)
router_id: Final = self.get_router_id()
cached: Final = self._cache_get((router_id, template_hash))
if cached is None:
verbose_router_logger.debug("No matching template in cache")
return None
verbose_router_logger.debug("Template cache hit for hash %s", template_hash[:8])
return AdeptTemplateMatch(
template_id=cached.id,
template=cached.template,
target_model=cached.target_model,
metadata=cached.additional_information,
)
except (AttributeError, IndexError, KeyError, TypeError, ValueError) as e:
verbose_router_logger.exception("Error matching template: %s", e)
return None
def _ensure_refresh_running(self) -> None:
existing: Final = self._refresh_task
if existing is not None and not existing.done():
return
try:
loop: Final = asyncio.get_running_loop()
except RuntimeError:
return
self._refresh_task = loop.create_task(self._refresh_loop())
def stop_refresh(self) -> None:
if self._refresh_task is not None:
self._refresh_task.cancel()
self._refresh_task = None
async def _refresh_loop(self) -> None:
while True:
try:
await self.refresh_cache()
except (AttributeError, IndexError, KeyError, TypeError, ValueError) as e:
verbose_router_logger.warning("AdeptRouter: cache refresh failed: %s", e)
await asyncio.sleep(_REFRESH_INTERVAL_SECONDS)
async def refresh_cache(self) -> None:
router_id: Final = self.get_router_id()
templates: Final = await self.template_store.load_all_for_router(router_id, limit=_TEMPLATE_CACHE_MAX_SIZE)
now: Final = time.monotonic()
rebuilt: Final[_TemplateCache] = OrderedDict( # mutable-ok: cache rebuild populated in one pass then swapped in
((router_id, t.template_hash), (now, t)) for t in templates if t.template_hash is not None
)
self._template_cache = rebuilt # rebind-ok: full cache swap after background refresh
def _cache_get(self, key: tuple[str, str]) -> StoredTemplate | None:
entry: Final = self._template_cache.get(key)
if entry is None:
return None
inserted_at, stored = entry
if time.monotonic() - inserted_at > _TEMPLATE_CACHE_TTL_SECONDS:
self._template_cache.pop(key, None)
return None
self._template_cache.move_to_end(key)
return stored
def _cache_put(self, key: tuple[str, str], stored: StoredTemplate) -> None:
self._template_cache[key] = (time.monotonic(), stored)
self._template_cache.move_to_end(key)
while len(self._template_cache) > _TEMPLATE_CACHE_MAX_SIZE:
self._template_cache.popitem(last=False)
def _cache_invalidate(self, key: tuple[str, str]) -> None:
self._template_cache.pop(key, None)
async def _resolve_template_id(
self, masked_template: str, template_hash: str, router_id: str, system_prompt: str | None
) -> str:
matched_id: Final = await self.template_store.match_by_hash(template_hash, router_id)
if matched_id is not None:
return matched_id
verbose_router_logger.info("No existing template found, storing new template.")
sys_prompt_payload: Final = {"system_prompt": system_prompt} # mutable-ok: JSON column payload
template_additional_info: Final[Mapping[str, object] | None] = sys_prompt_payload if system_prompt else None
stored_id: Final = await self.template_store.store_template(
template_id=str(uuid4()),
template=masked_template,
template_hash=template_hash,
target_model="",
router_id=router_id,
additional_information=template_additional_info,
)
self._cache_invalidate((router_id, template_hash))
return stored_id or str(uuid4())
async def store_conversation(
self,
prompt: str,
response: str,
model: str | None = None,
token_usage: Mapping[str, object] | None = None,
cost_usd: float | None = None,
latency_ms: float | None = None,
system_prompt: str | None = None,
routed_to_slm: bool | None = None,
) -> None:
try:
masked_template, extractions = self._extract_template(prompt)
template_hash: Final = self._hash_template(masked_template, system_prompt)
router_id: Final = self.get_router_id()
template_id: Final = await self._resolve_template_id(
masked_template, template_hash, router_id, system_prompt
)
additional_info: Final[dict[str, object]] = {"extractions": extractions} # mutable-ok: JSON column payload
if model is not None:
additional_info["model"] = model
if token_usage is not None:
additional_info["token_usage"] = token_usage
if cost_usd is not None:
additional_info["cost_usd"] = cost_usd
if latency_ms is not None:
additional_info["latency_ms"] = round(latency_ms, 2)
if routed_to_slm is not None:
additional_info["routed_to_slm"] = routed_to_slm
await self.template_store.store_conversation(
prompt=prompt,
response=response,
template_id=template_id,
additional_information=additional_info,
)
conversation_count: Final = await self.template_store.count_conversation_by_template_id(template_id)
if (
conversation_count is not None
and conversation_count >= self.conversations_threshold
and conversation_count % self.conversations_threshold == 0
):
# Trainer runs may flip target_model, so drop the cached row before firing.
self._cache_invalidate((router_id, template_hash))
self._trigger_trainer(template_id)
verbose_router_logger.info("Stored interaction for template %s", template_id)
except (AttributeError, IndexError, KeyError, TypeError, ValueError) as e:
verbose_router_logger.exception("Error storing interaction: %s", e)
def _trigger_trainer(self, template_id: str) -> None:
if not self.trainer_url:
verbose_router_logger.info(
"AdeptRouter: threshold reached for template %s but no trainer_url configured — skipping notification.",
template_id,
)
return
asyncio.create_task(self._trainer_post(f"{self.trainer_url}/run-workflow/{template_id}", template_id))
@staticmethod
async def _trainer_post(url: str, template_id: str) -> None:
from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
try:
pinned_url, host_header = validate_url(url)
except SSRFError as ssrf_err:
verbose_router_logger.warning(
"AdeptRouter: refusing trainer POST for template %s — %s", template_id, ssrf_err
)
return
client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
try:
await client.post(
url=pinned_url,
headers={"Host": host_header}, # mutable-ok: one-shot per-request headers dict handed to httpx
timeout=_TRAINER_HTTP_TIMEOUT_SECONDS,
)
verbose_router_logger.info("Triggered trainer for template %s", template_id)
except httpx.HTTPError as e:
verbose_router_logger.warning("Failed to trigger trainer: %s", e)

41
litellm/router_strategy/adept_router/template/router_template.py Normal file
View file

@ -0,0 +1,41 @@
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing_extensions import ReadOnly, TypedDict
class AdeptTemplateMatch(TypedDict):
"""Result of matching a prompt to a stored template."""
template_id: ReadOnly[str]
template: ReadOnly[str]
target_model: ReadOnly[str | None]
metadata: ReadOnly[Mapping[str, object] | None]
class BaseTemplateRouter(ABC):
"""Abstract base class for template-based prompt routing."""
@abstractmethod
async def route(self, prompt: str, system_prompt: str | None = None) -> AdeptTemplateMatch | None:
"""
Match a prompt to a stored template.
Returns a dict with template details if matched, None otherwise.
"""
...
@abstractmethod
async def store_conversation(
self,
prompt: str,
response: str,
model: str | None = None,
token_usage: Mapping[str, object] | None = None,
cost_usd: float | None = None,
latency_ms: float | None = None,
system_prompt: str | None = None,
routed_to_slm: bool | None = None,
) -> None:
"""Persist a prompt-response pair with its template and per-call metrics."""
...

13
litellm/types/router.py
View file

@ -387,6 +387,19 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
quality_router_config: dict | None = None
quality_router_default_model: str | None = None
# adept-router params
adept_router_default_model: str | None = None
adept_router_tag_prefix: str | None = None
adept_router_seed_config: list[dict] | None = None # mutable-ok: pydantic config field
adept_router_conversations_threshold: int | None = None
adept_router_trainer_url: str | None = None
adept_router_pg_host: str | None = None
adept_router_pg_port: int | None = None
adept_router_pg_database: str | None = None
adept_router_pg_user: str | None = None
adept_router_pg_password: str | None = None
adept_router_pg_ssl_mode: str | None = None
# Vector Store Params
vector_store_id: str | None = None
milvus_text_field: str | None = None

13
litellm/types/utils.py
View file

@ -3852,6 +3852,19 @@ all_litellm_params = (
"adaptive_router_default_model",
"quality_router_config",
"quality_router_default_model",
"adept_router",
"adept_router_default_model",
"adept_router_pg_host",
"adept_router_pg_port",
"adept_router_pg_user",
"adept_router_pg_password",
"adept_router_pg_database",
"adept_router_tag_prefix",
"adept_router_conversations_threshold",
"adept_router_trainer_url",
"adept_router_seed_config",
"adept_router_pg_ssl_mode",
"adept_routed_to_slm",
]
+ list(StandardCallbackDynamicParams.__annotations__.keys())
+ list(CustomPricingLiteLLMParams.model_fields.keys())

1
tests/code_coverage_tests/router_code_coverage.py
View file

@ -88,6 +88,7 @@ ignored_function_names = [
"_resolve_claude_code_session_router", # Tested through Claude Code session routing in test_router.py
"_get_claude_code_session_router_binding", # Tested through the two-worker session routing test in test_router.py
"_apply_updated_routing_strategy_args", # Tested via update_settings in test_lowest_latency.py (file lacks "router" in name)
"_release_adept_router_for_deleted_deployment", # Tested indirectly via delete_deployment in test_adept_router.py (the two test_delete_deployment_* tests)
]

48
tests/test_litellm/litellm_core_utils/test_get_supported_openai_params.py
View file

@ -1,7 +1,5 @@
import pytest
from litellm.litellm_core_utils.get_supported_openai_params import (
get_supported_openai_params,
)
@ -33,9 +31,7 @@ def test_base_model_label_alone_lacks_bedrock_tools():
"""The label by itself does not advertise tools; this is what made the union
necessary. Guards against the discrepancy disappearing (and the regression test
above silently passing for the wrong reason)."""
params = get_supported_openai_params(
model=BEDROCK_LABEL, custom_llm_provider="bedrock"
)
params = get_supported_openai_params(model=BEDROCK_LABEL, custom_llm_provider="bedrock")
assert params is not None
assert "tools" not in params
@ -46,14 +42,8 @@ def test_base_model_is_additive_not_replacement():
Bedrock: real id supports ``tools`` but not the label's reasoning hint; the union
must contain the real model's ``tools`` regardless of the label being a subset."""
real_only = set(
get_supported_openai_params(
model=BEDROCK_REAL_MODEL, custom_llm_provider="bedrock"
)
)
label_only = set(
get_supported_openai_params(model=BEDROCK_LABEL, custom_llm_provider="bedrock")
)
real_only = set(get_supported_openai_params(model=BEDROCK_REAL_MODEL, custom_llm_provider="bedrock"))
label_only = set(get_supported_openai_params(model=BEDROCK_LABEL, custom_llm_provider="bedrock"))
combined = set(
get_supported_openai_params(
model=BEDROCK_REAL_MODEL,
@ -67,17 +57,13 @@ def test_base_model_is_additive_not_replacement():
assert real_only <= combined
def test_base_model_adds_capabilities_the_real_model_lacks():
def test_base_model_adds_capabilities_the_real_model_lacks(local_model_cost_map):
"""Regression for #27717 (the behavior the union must preserve).
``gemini-3.1-pro`` isn't in the cost map so it advertises no reasoning support,
but the registered ``gemini-3.1-pro-preview`` base_model does. The hint must add
``reasoning_effort``/``thinking`` without the call erroring."""
real_only = set(
get_supported_openai_params(
model="gemini-3.1-pro", custom_llm_provider="gemini"
)
)
real_only = set(get_supported_openai_params(model="gemini-3.1-pro", custom_llm_provider="gemini"))
assert "reasoning_effort" not in real_only
combined = set(
@ -93,21 +79,15 @@ def test_base_model_adds_capabilities_the_real_model_lacks():
def test_no_base_model_is_unchanged():
"""Omitting ``base_model`` must resolve purely from ``model``."""
with_none = get_supported_openai_params(
model=BEDROCK_REAL_MODEL, custom_llm_provider="bedrock", base_model=None
)
plain = get_supported_openai_params(
model=BEDROCK_REAL_MODEL, custom_llm_provider="bedrock"
)
with_none = get_supported_openai_params(model=BEDROCK_REAL_MODEL, custom_llm_provider="bedrock", base_model=None)
plain = get_supported_openai_params(model=BEDROCK_REAL_MODEL, custom_llm_provider="bedrock")
assert with_none == plain
def test_base_model_equal_to_model_is_unchanged():
"""A ``base_model`` identical to ``model`` must not double-resolve or reorder."""
plain = get_supported_openai_params(
model=BEDROCK_REAL_MODEL, custom_llm_provider="bedrock"
)
plain = get_supported_openai_params(model=BEDROCK_REAL_MODEL, custom_llm_provider="bedrock")
same = get_supported_openai_params(
model=BEDROCK_REAL_MODEL,
custom_llm_provider="bedrock",
@ -152,14 +132,10 @@ def test_bedrock_converse_alias_resolves_like_bedrock():
params saw no Bedrock capabilities for a Converse model invoked via the alias."""
anthropic_model = "bedrock/converse/us.anthropic.claude-sonnet-4-6"
via_alias = get_supported_openai_params(
model=anthropic_model, custom_llm_provider="bedrock_converse"
)
via_alias = get_supported_openai_params(model=anthropic_model, custom_llm_provider="bedrock_converse")
assert via_alias is not None
assert via_alias == get_supported_openai_params(
model=anthropic_model, custom_llm_provider="bedrock"
)
assert via_alias == get_supported_openai_params(model=anthropic_model, custom_llm_provider="bedrock")
assert "web_search_options" not in via_alias
assert "tools" in via_alias
@ -167,9 +143,7 @@ def test_bedrock_converse_alias_resolves_like_bedrock():
def test_bedrock_converse_alias_keeps_nova_web_search_options():
"""Nova on the ``bedrock_converse`` alias still advertises web_search_options, proving the
alias routes through the model-aware config rather than a blanket Bedrock default."""
nova_params = get_supported_openai_params(
model="amazon.nova-pro-v1:0", custom_llm_provider="bedrock_converse"
)
nova_params = get_supported_openai_params(model="amazon.nova-pro-v1:0", custom_llm_provider="bedrock_converse")
assert nova_params is not None
assert "web_search_options" in nova_params

42
tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py
View file

@ -22,7 +22,6 @@ from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation
from litellm.types.llms.vertex_ai import VertexAIBatchEmbeddingsResponseObject
from litellm.types.utils import EmbeddingResponse
IMAGE_DATA_URI = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
GCS_URL = "gs://my-bucket/image.png"
@ -74,9 +73,7 @@ class TestBuildPartForInput:
assert part["file_data"]["file_uri"] == GCS_URL
def test_file_reference_resolved(self):
resolved = {
"files/abc": {"mime_type": "image/jpeg", "uri": "https://example.com/abc"}
}
resolved = {"files/abc": {"mime_type": "image/jpeg", "uri": "https://example.com/abc"}}
part = _build_part_for_input("files/abc", resolved_files=resolved)
assert part["file_data"] is not None
assert part["file_data"]["mime_type"] == "image/jpeg"
@ -115,10 +112,7 @@ class TestTransformOpenaiInputGeminiContent:
)
assert len(result["requests"]) == 2
# First request is text
assert (
result["requests"][0]["content"]["parts"][0]["text"]
== "The food was delicious"
)
assert result["requests"][0]["content"]["parts"][0]["text"] == "The food was delicious"
# Second request is image
assert result["requests"][1]["content"]["parts"][0]["inline_data"] is not None
@ -217,9 +211,7 @@ class TestProcessResponse:
"""Test that process_response sets correct indices."""
def test_single_embedding_index(self):
predictions: VertexAIBatchEmbeddingsResponseObject = {
"embeddings": [{"values": [0.1, 0.2]}]
}
predictions: VertexAIBatchEmbeddingsResponseObject = {"embeddings": [{"values": [0.1, 0.2]}]}
model_response = EmbeddingResponse()
result = process_response(
input="hello",
@ -270,9 +262,7 @@ class TestProcessResponse:
def test_nested_input_token_counting(self):
"""Nested list: only plain-text sub-elements should be counted."""
predictions: VertexAIBatchEmbeddingsResponseObject = {
"embeddings": [{"values": [0.1, 0.2]}]
}
predictions: VertexAIBatchEmbeddingsResponseObject = {"embeddings": [{"values": [0.1, 0.2]}]}
result = process_response(
input=[["a red shoe", IMAGE_DATA_URI]],
model_response=EmbeddingResponse(),
@ -307,7 +297,7 @@ class TestProcessEmbedContentResponseUsage:
MODEL = "gemini-embedding-2"
def test_multimodal_image_preserves_usage_metadata(self):
def test_multimodal_image_preserves_usage_metadata(self, local_model_cost_map):
response_json = {
"embedding": {"values": [0.1, 0.2, 0.3]},
"usageMetadata": {
@ -374,9 +364,7 @@ class TestProcessEmbedContentResponseUsage:
response_json=response_json,
)
assert result.usage.prompt_tokens == 516
assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx(
2.0
)
assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx(2.0)
assert result.usage.prompt_tokens_details.text_tokens == 1
def test_missing_usage_metadata_does_not_estimate_from_base64(self):
@ -400,7 +388,7 @@ class TestProcessEmbedContentResponseUsage:
)
assert result.usage.prompt_tokens > 0
def test_file_reference_image_billed_per_image_not_text(self):
def test_file_reference_image_billed_per_image_not_text(self, local_model_cost_map):
"""files/... image refs must bill per-image, not at the text token rate."""
response_json = {
"embedding": {"values": [0.1, 0.2, 0.3]},
@ -432,7 +420,7 @@ class TestProcessEmbedContentResponseUsage:
)
assert prompt_cost == pytest.approx(0.00012)
def test_file_reference_non_image_not_counted_as_image(self):
def test_file_reference_non_image_not_counted_as_image(self, local_model_cost_map):
"""A files/... ref resolving to a non-image mime must not be image-counted."""
response_json = {
"embedding": {"values": [0.1, 0.2]},
@ -456,9 +444,7 @@ class TestProcessEmbedContentResponseUsage:
)
assert result.usage.prompt_tokens_details.image_count == 0
assert result.usage.prompt_tokens_details.audio_tokens == 64
assert result.usage.prompt_tokens_details.audio_length_seconds == pytest.approx(
2.0
)
assert result.usage.prompt_tokens_details.audio_length_seconds == pytest.approx(2.0)
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
@ -467,7 +453,7 @@ class TestProcessEmbedContentResponseUsage:
)
assert prompt_cost == pytest.approx(2.0 * 0.00016)
def test_video_plus_audio_does_not_double_bill_text(self):
def test_video_plus_audio_does_not_double_bill_text(self, local_model_cost_map):
"""Video+audio responses must not get video tokens reassigned to text."""
response_json = {
"embedding": {"values": [0.1]},
@ -487,12 +473,8 @@ class TestProcessEmbedContentResponseUsage:
response_json=response_json,
)
assert result.usage.prompt_tokens_details.text_tokens == 1
assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx(
2.0
)
assert result.usage.prompt_tokens_details.audio_length_seconds == pytest.approx(
2.0
)
assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx(2.0)
assert result.usage.prompt_tokens_details.audio_length_seconds == pytest.approx(2.0)
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,

1792
tests/test_litellm/test_adept_router.py Normal file
View file

File diff suppressed because it is too large Load diff

2
ui/litellm-dashboard/eslint-suppressions.json
View file

@ -2455,4 +2455,4 @@
"count": 1
}
}
}
}

1
ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx
View file

@ -128,6 +128,7 @@ describe("ModelsAndEndpointsPage", () => {
"All Models",
"Add Model",
"Auto-Routers Beta",
"ADEPT Routers",
"LLM Credentials",
"Pass-Through Endpoints",
"Health Status",

7
ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx
View file

@ -17,6 +17,7 @@ import { useModelDashboardData } from "@/app/(dashboard)/models-and-endpoints/us
import AllModelsPanel from "@/app/(dashboard)/models-and-endpoints/panels/AllModelsPanel";
import AutoRoutersTabPanel from "@/app/(dashboard)/models-and-endpoints/panels/AutoRoutersTabPanel";
import AddModelPanel from "@/app/(dashboard)/models-and-endpoints/panels/AddModelPanel";
import AddAdeptRouterPanel from "@/app/(dashboard)/models-and-endpoints/panels/AddAdeptRouterPanel";
import LlmCredentialsPanel from "@/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel";
import PassThroughPanel from "@/app/(dashboard)/models-and-endpoints/panels/PassThroughPanel";
import HealthStatusPanel from "@/app/(dashboard)/models-and-endpoints/panels/HealthStatusPanel";
@ -30,6 +31,7 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
type ModelTabSlug =
| "add"
| "auto-routers"
| "adept-routers"
| "llm-credentials"
| "pass-through"
| "health"
@ -43,6 +45,7 @@ const BASE_TAB_KEY = "all-models";
const TAB_LABELS: Record<ModelTabSlug, string> = {
add: "Add Model",
"auto-routers": "Auto-Routers",
"adept-routers": "ADEPT Routers",
"llm-credentials": "LLM Credentials",
"pass-through": "Pass-Through Endpoints",
health: "Health Status",
@ -58,6 +61,8 @@ const renderPanel = (key: string) => {
return <AllModelsPanel />;
case "auto-routers":
return <AutoRoutersTabPanel />;
case "adept-routers":
return <AddAdeptRouterPanel />;
case "add":
return <AddModelPanel />;
case "llm-credentials":
@ -105,7 +110,7 @@ export default function ModelsAndEndpointsPage() {
() => [
"",
...(canCreate ? (["add"] as const) : []),
...(isAdmin || canCreate ? (["auto-routers"] as const) : []),
...(isAdmin || canCreate ? (["auto-routers", "adept-routers"] as const) : []),
// effectiveSessionRole reports proxy_admin_viewer as "Admin", so isAdmin alone would show a
// viewer these write-only panels; only the raw-role isViewOnly separates them. Health Status
// stays: it is the bucket's one read view, and viewers keep read parity with admins.

40
ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddAdeptRouterPanel.tsx Normal file
View file

@ -0,0 +1,40 @@
"use client";
import { useQueryClient } from "@tanstack/react-query";
import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams";
import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings";
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
import { internalUserRoles } from "@/utils/roles";
import { modelCreationScope } from "@/utils/modelPermissions";
import AddAdeptRouterTab from "@/components/add_model/AddAdeptRouterTab";
/**
* Owns the permission decision for the ADEPT Routers tab. Creating an ADEPT router is a
* POST /model/new like Add Model and Auto Router, so it takes the same audience rule:
* a proxy admin, or a team admin who scopes it to a team.
*/
export default function AddAdeptRouterPanel() {
const { accessToken, userRole, userId: userID, isViewOnly } = useAuthorized();
const { data: teams } = useTeams();
const { data: uiSettings } = useUISettings();
const queryClient = useQueryClient();
const isInternalUser = userRole != null && internalUserRoles.includes(userRole);
const scope = modelCreationScope(
{ userRole, userID, isViewOnly },
{
teams: teams ?? null,
disabledForInternalUsers: isInternalUser && uiSettings?.values?.disable_model_add_for_internal_users === true,
},
);
return (
<AddAdeptRouterTab
handleOk={() => queryClient.invalidateQueries({ queryKey: ["models", "list"] })}
accessToken={accessToken ?? ""}
userRole={userRole ?? ""}
userId={userID ?? null}
createScope={scope}
/>
);
}

291
ui/litellm-dashboard/src/components/add_model/AddAdeptRouterTab.tsx Normal file
View file

@ -0,0 +1,291 @@
"use client";
import React, { useEffect, useState } from "react";
import { useForm, useWatch, type UseFormReturn } from "react-hook-form";
import { toast } from "@/lib/toast";
import { FieldGroup } from "@/components/ui/field";
import { FormField } from "@/components/shared/form/FormField";
import { Button } from "@/components/ui/button";
import { Card } from "@/components/ui/card";
import { Input } from "@/components/ui/input";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { fetchAvailableModels, type ModelGroup } from "@/components/llm_calls/fetch_models";
import { modelAvailableCall } from "../networking";
import AccessGroupTagsCombobox from "./AccessGroupTagsCombobox";
import ModelChoiceCombobox from "./ModelChoiceCombobox";
import TeamDropdown from "../common_components/team_dropdown";
import { type ModelWriteScope } from "@/utils/modelPermissions";
import { handleAddAdeptRouterSubmit, type AddAdeptRouterValues } from "./HandleAddAdeptRouterSubmit";
// Fixed sslmode options mirror libpq's set. Kept in sync with the backend `LiteLLM_Params.adept_router_pg_ssl_mode`.
const PG_SSL_MODES = ["disable", "allow", "prefer", "require", "verify-ca", "verify-full"] as const;
const EMPTY_FORM_VALUES: AddAdeptRouterValues = {
adept_router_name: "",
adept_router_default_model: "",
adept_router_tag_prefix: "",
adept_router_conversations_threshold: null,
adept_router_trainer_url: "",
adept_router_pg_host: "",
adept_router_pg_port: 5432,
adept_router_pg_database: "",
adept_router_pg_user: "",
adept_router_pg_password: "",
adept_router_pg_ssl_mode: "prefer",
team_id: "",
model_access_group: [],
};
interface AddAdeptRouterTabProps {
handleOk?: () => void;
accessToken: string;
userRole: string;
userId?: string | null;
createScope?: ModelWriteScope;
}
const AddAdeptRouterTab: React.FC<AddAdeptRouterTabProps> = ({
handleOk,
accessToken,
userRole,
createScope = "unscoped-ok",
}) => {
const requiresTeamScope = createScope === "team-required";
const form: UseFormReturn<AddAdeptRouterValues> = useForm<AddAdeptRouterValues>({
defaultValues: EMPTY_FORM_VALUES,
});
const [modelAccessGroups, setModelAccessGroups] = useState<string[]>([]);
const [modelInfo, setModelInfo] = useState<ModelGroup[]>([]);
const watchedName = useWatch({ control: form.control, name: "adept_router_name" });
const watchedDefaultModel = useWatch({ control: form.control, name: "adept_router_default_model" });
const watchedTeamId = useWatch({ control: form.control, name: "team_id" });
useEffect(() => {
const loadAccessGroups = async () => {
try {
const response = await modelAvailableCall(accessToken, "", "", false, null, true, true);
setModelAccessGroups(response["data"].map((model: { id: string }) => model["id"]));
} catch {
// access groups unavailable; the combobox falls back to a plain text entry
}
};
loadAccessGroups();
}, [accessToken]);
useEffect(() => {
const loadModels = async () => {
try {
setModelInfo(await fetchAvailableModels(accessToken));
} catch {
// model list unavailable; the combobox still renders a free-text entry
}
};
loadModels();
}, [accessToken]);
const modelChoices = Array.from(new Set(modelInfo.map((m) => m.model_group))).map((g) => ({ value: g, label: g }));
const computeSubmitBlockedReason = (): string | null => {
if (!watchedName?.trim()) return "Enter a router name";
if (!watchedDefaultModel?.trim()) return "Select a default fallback model";
if (requiresTeamScope && !watchedTeamId?.trim()) return "Select a team to create this router under";
return null;
};
const submitBlockedReason: string | null = computeSubmitBlockedReason();
const onSubmit = async (values: AddAdeptRouterValues) => {
if (submitBlockedReason !== null) {
toast.fromError(submitBlockedReason);
return;
}
await handleAddAdeptRouterSubmit(values, accessToken, () => form.reset(EMPTY_FORM_VALUES), handleOk);
};
return (
<Card className="block p-6">
<div className="mb-4">
<h3 className="text-lg font-medium">Add ADEPT Router</h3>
<p className="text-sm text-muted-foreground">
Route XML-tagged agent prompts to task-specific SLMs. Requests fall back to the default model until a
template&apos;s <code>target_model</code> is trained.
</p>
</div>
<form onSubmit={form.handleSubmit(onSubmit)} noValidate>
<FieldGroup>
<FormField
control={form.control}
name="adept_router_name"
label="Router Name"
description="Model group name; the underlying alias is adept/<name>."
>
{({ ref, ...field }) => (
<Input {...field} ref={ref} placeholder="e.g., adept_router_prod" value={field.value ?? ""} />
)}
</FormField>
<FormField
control={form.control}
name="adept_router_default_model"
label="Default Model"
description="Model that serves requests until a template is trained."
>
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
<ModelChoiceCombobox
id={id}
value={value ?? ""}
onChange={onChange}
choices={modelChoices}
placeholder="Pick a fallback model"
ariaInvalid={ariaInvalid}
ariaDescribedBy={ariaDescribedBy}
/>
)}
</FormField>
<FormField
control={form.control}
name="adept_router_tag_prefix"
label="XML Tag Prefix"
description="Optional prefix for the XML tags the router masks (e.g. 'var' matches <var invoice_id>)."
>
{({ ref, ...field }) => <Input {...field} ref={ref} placeholder="var" value={field.value ?? ""} />}
</FormField>
<FormField
control={form.control}
name="adept_router_conversations_threshold"
label="Conversations Threshold"
description="Trigger the trainer at every multiple of this count (default: 1000)."
>
{({ ref, value, onChange, ...field }) => (
<Input
{...field}
ref={ref}
type="number"
min={1}
value={value ?? ""}
onChange={(event) => onChange(event.target.value === "" ? null : event.target.valueAsNumber)}
/>
)}
</FormField>
<FormField
control={form.control}
name="adept_router_trainer_url"
label="Trainer URL"
description="External training pipeline webhook. Must be http:// or https:// and pass any allowed-hosts filter."
>
{({ ref, ...field }) => (
<Input
{...field}
ref={ref}
type="url"
placeholder="https://trainer.internal/hooks"
value={field.value ?? ""}
/>
)}
</FormField>
<FormField
control={form.control}
name="adept_router_pg_host"
label="PostgreSQL Host"
description="ADEPT stores templates + conversations in its own Postgres, separate from the proxy DB."
>
{({ ref, ...field }) => <Input {...field} ref={ref} placeholder="db.internal" value={field.value ?? ""} />}
</FormField>
<FormField control={form.control} name="adept_router_pg_port" label="PostgreSQL Port">
{({ ref, value, onChange, ...field }) => (
<Input
{...field}
ref={ref}
type="number"
min={1}
max={65535}
value={value ?? ""}
onChange={(event) => onChange(event.target.value === "" ? null : event.target.valueAsNumber)}
/>
)}
</FormField>
<FormField control={form.control} name="adept_router_pg_database" label="PostgreSQL Database">
{({ ref, ...field }) => <Input {...field} ref={ref} placeholder="adept" value={field.value ?? ""} />}
</FormField>
<FormField control={form.control} name="adept_router_pg_user" label="PostgreSQL User">
{({ ref, ...field }) => <Input {...field} ref={ref} placeholder="adept_rw" value={field.value ?? ""} />}
</FormField>
<FormField control={form.control} name="adept_router_pg_password" label="PostgreSQL Password">
{({ ref, ...field }) => (
<Input {...field} ref={ref} type="password" value={field.value ?? ""} autoComplete="new-password" />
)}
</FormField>
<FormField
control={form.control}
name="adept_router_pg_ssl_mode"
label="PostgreSQL SSL Mode"
description="libpq sslmode. Defaults to 'prefer'; use 'verify-full' when you have a CA configured."
>
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
<Select value={value ?? "prefer"} onValueChange={onChange}>
<SelectTrigger id={id} aria-invalid={ariaInvalid} aria-describedby={ariaDescribedBy}>
<SelectValue placeholder="prefer" />
</SelectTrigger>
<SelectContent>
{PG_SSL_MODES.map((mode) => (
<SelectItem key={mode} value={mode}>
{mode}
</SelectItem>
))}
</SelectContent>
</Select>
)}
</FormField>
{requiresTeamScope && (
<FormField
control={form.control}
name="team_id"
label="Team"
description="Team admins must pick the team that owns this router."
>
{({ id, value, onChange }) => (
<TeamDropdown id={id} value={value ?? ""} onChange={(next) => onChange(next ?? "")} />
)}
</FormField>
)}
<FormField
control={form.control}
name="model_access_group"
label="Model Access Groups"
description="Optional access groups virtual keys use to gate this router."
>
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
<AccessGroupTagsCombobox
id={id}
value={value ?? []}
onChange={onChange}
options={modelAccessGroups}
ariaInvalid={ariaInvalid}
ariaDescribedBy={ariaDescribedBy}
/>
)}
</FormField>
</FieldGroup>
<div className="mt-6 flex justify-end">
<Button type="submit" disabled={submitBlockedReason !== null} title={submitBlockedReason ?? undefined}>
{submitBlockedReason ?? "Create ADEPT Router"}
</Button>
</div>
</form>
</Card>
);
};
export default AddAdeptRouterTab;

66
ui/litellm-dashboard/src/components/add_model/HandleAddAdeptRouterSubmit.tsx Normal file
View file

@ -0,0 +1,66 @@
import { modelCreateCall, Model } from "../networking";
import { toast } from "@/lib/toast";
export interface AddAdeptRouterValues {
adept_router_name: string;
adept_router_default_model: string;
adept_router_tag_prefix?: string;
adept_router_conversations_threshold?: number | null;
adept_router_trainer_url?: string;
adept_router_pg_host?: string;
adept_router_pg_port?: number | null;
adept_router_pg_database?: string;
adept_router_pg_user?: string;
adept_router_pg_password?: string;
adept_router_pg_ssl_mode?: string;
team_id?: string;
model_access_group?: string[];
}
// Drops undefined / empty-string entries so `litellm_params` on the backend only carries the
// keys the operator actually set; keeps the create payload aligned with the Pydantic defaults.
const dropEmpty = <T extends Record<string, unknown>>(values: T): Partial<T> =>
Object.fromEntries(
Object.entries(values).filter(([, value]) => value !== undefined && value !== "" && value !== null),
) as Partial<T>;
export const handleAddAdeptRouterSubmit = async (
values: AddAdeptRouterValues,
accessToken: string,
resetForm: () => void,
callback?: () => void,
) => {
try {
const rawLitellmParams = {
model: `adept/${values.adept_router_name}`,
adept_router_default_model: values.adept_router_default_model,
adept_router_tag_prefix: values.adept_router_tag_prefix,
adept_router_conversations_threshold: values.adept_router_conversations_threshold ?? undefined,
adept_router_trainer_url: values.adept_router_trainer_url,
adept_router_pg_host: values.adept_router_pg_host,
adept_router_pg_port: values.adept_router_pg_port ?? undefined,
adept_router_pg_database: values.adept_router_pg_database,
adept_router_pg_user: values.adept_router_pg_user,
adept_router_pg_password: values.adept_router_pg_password,
adept_router_pg_ssl_mode: values.adept_router_pg_ssl_mode,
};
const litellmParams = dropEmpty(rawLitellmParams);
const adeptConfig = {
model_name: values.adept_router_name,
litellm_params: litellmParams,
model_info: {
...(values.team_id ? { team_id: values.team_id } : {}),
...(values.model_access_group?.length ? { access_groups: values.model_access_group } : {}),
},
};
await modelCreateCall(accessToken, adeptConfig as unknown as Model);
toast.success(`Successfully created ADEPT Router: ${values.adept_router_name}`);
resetForm();
callback?.();
} catch (error) {
console.error("Failed to add ADEPT router:", error);
toast.fromError("Failed to add ADEPT router: " + error);
}
};

46
ui/litellm-dashboard/src/components/edit_adept_router/AdeptRouterEditControl.tsx Normal file
View file

@ -0,0 +1,46 @@
"use client";
import { useState } from "react";
import { Button } from "@/components/ui/button";
import EditAdeptRouterModal, { type AdeptRouterModelData } from "./EditAdeptRouterModal";
interface AdeptRouterEditControlProps {
canEdit: boolean;
isEditing: boolean;
modelData: AdeptRouterModelData;
accessToken: string;
onUpdated: (updated: AdeptRouterModelData) => void;
}
/** Composes the "Edit ADEPT Router" trigger + modal into one unit so model_info_view.tsx does
* not have to know about the ADEPT-specific state, mirroring how EditAutoRouterModal is wired
* next to it but without adding 15+ lines of view-model glue to the parent. */
export const AdeptRouterEditControl: React.FC<AdeptRouterEditControlProps> = ({
canEdit,
isEditing,
modelData,
accessToken,
onUpdated,
}) => {
const [isOpen, setIsOpen] = useState(false);
const isAdept = (modelData?.litellm_params as { model?: string } | undefined)?.model?.startsWith("adept/") ?? false;
if (!isAdept) return null;
return (
<>
{canEdit && !isEditing && (
<Button onClick={() => setIsOpen(true)} className="flex items-center">
Edit ADEPT Router
</Button>
)}
<EditAdeptRouterModal
isVisible={isOpen}
onCancel={() => setIsOpen(false)}
onSuccess={onUpdated}
modelData={modelData}
accessToken={accessToken}
/>
</>
);
};
export default AdeptRouterEditControl;

297
ui/litellm-dashboard/src/components/edit_adept_router/EditAdeptRouterModal.tsx Normal file
View file

@ -0,0 +1,297 @@
"use client";
import React, { useEffect, useState } from "react";
import { useForm } from "react-hook-form";
import { toast } from "@/lib/toast";
import { FieldGroup } from "@/components/ui/field";
import { FormField } from "@/components/shared/form/FormField";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog";
import { fetchAvailableModels, type ModelGroup } from "@/components/llm_calls/fetch_models";
import { modelPatchUpdateCall } from "../networking";
import ModelChoiceCombobox from "../add_model/ModelChoiceCombobox";
const PG_SSL_MODES = ["disable", "allow", "prefer", "require", "verify-ca", "verify-full"] as const;
interface AdeptRouterLitellmParams {
adept_router_default_model?: string | null;
adept_router_tag_prefix?: string | null;
adept_router_conversations_threshold?: number | null;
adept_router_trainer_url?: string | null;
adept_router_pg_host?: string | null;
adept_router_pg_port?: number | null;
adept_router_pg_database?: string | null;
adept_router_pg_user?: string | null;
adept_router_pg_password?: string | null;
adept_router_pg_ssl_mode?: string | null;
}
export interface AdeptRouterModelData {
model_name: string;
litellm_params?: AdeptRouterLitellmParams;
model_info: { id: string };
}
interface EditAdeptRouterModalProps {
isVisible: boolean;
onCancel: () => void;
onSuccess: (updatedModel: AdeptRouterModelData) => void;
modelData: AdeptRouterModelData;
accessToken: string;
}
interface EditAdeptRouterFormValues {
adept_router_name: string;
adept_router_default_model: string;
adept_router_tag_prefix: string;
adept_router_conversations_threshold: number | null;
adept_router_trainer_url: string;
adept_router_pg_host: string;
adept_router_pg_port: number | null;
adept_router_pg_database: string;
adept_router_pg_user: string;
adept_router_pg_password: string;
adept_router_pg_ssl_mode: string;
}
const toFormValues = (modelData: AdeptRouterModelData): EditAdeptRouterFormValues => {
const lp = modelData.litellm_params ?? {};
return {
adept_router_name: modelData.model_name,
adept_router_default_model: lp.adept_router_default_model ?? "",
adept_router_tag_prefix: lp.adept_router_tag_prefix ?? "",
adept_router_conversations_threshold: lp.adept_router_conversations_threshold ?? null,
adept_router_trainer_url: lp.adept_router_trainer_url ?? "",
adept_router_pg_host: lp.adept_router_pg_host ?? "",
adept_router_pg_port: lp.adept_router_pg_port ?? 5432,
adept_router_pg_database: lp.adept_router_pg_database ?? "",
adept_router_pg_user: lp.adept_router_pg_user ?? "",
adept_router_pg_password: "",
adept_router_pg_ssl_mode: lp.adept_router_pg_ssl_mode ?? "prefer",
};
};
const EditAdeptRouterModal: React.FC<EditAdeptRouterModalProps> = ({
isVisible,
onCancel,
onSuccess,
modelData,
accessToken,
}) => {
const form = useForm<EditAdeptRouterFormValues>({ defaultValues: toFormValues(modelData) });
const [modelInfo, setModelInfo] = useState<ModelGroup[]>([]);
const [saving, setSaving] = useState(false);
const pgPasswordAlreadySet = !!modelData.litellm_params?.adept_router_pg_password;
useEffect(() => {
if (isVisible) {
form.reset(toFormValues(modelData));
}
}, [isVisible, modelData, form]);
useEffect(() => {
if (!isVisible) return;
const loadModels = async () => {
try {
setModelInfo(await fetchAvailableModels(accessToken));
} catch {
// model list unavailable; combobox still accepts free-text
}
};
loadModels();
}, [isVisible, accessToken]);
const modelChoices = Array.from(new Set(modelInfo.map((m) => m.model_group))).map((g) => ({ value: g, label: g }));
const handleSave = async (values: EditAdeptRouterFormValues) => {
setSaving(true);
try {
const updatedLitellmParams: AdeptRouterLitellmParams & { model: string } = {
model: `adept/${values.adept_router_name}`,
adept_router_default_model: values.adept_router_default_model || null,
adept_router_tag_prefix: values.adept_router_tag_prefix || null,
adept_router_conversations_threshold: values.adept_router_conversations_threshold ?? null,
adept_router_trainer_url: values.adept_router_trainer_url || null,
adept_router_pg_host: values.adept_router_pg_host || null,
adept_router_pg_port: values.adept_router_pg_port ?? null,
adept_router_pg_database: values.adept_router_pg_database || null,
adept_router_pg_user: values.adept_router_pg_user || null,
adept_router_pg_ssl_mode: values.adept_router_pg_ssl_mode || null,
};
// Only overwrite the password when the operator typed a new one; otherwise the
// stored value is preserved by omission.
if (values.adept_router_pg_password) {
updatedLitellmParams.adept_router_pg_password = values.adept_router_pg_password;
}
const patchPayload = {
model_name: values.adept_router_name,
litellm_params: updatedLitellmParams,
};
await modelPatchUpdateCall(accessToken, patchPayload, modelData.model_info.id);
toast.success(`Updated ADEPT Router: ${values.adept_router_name}`);
const updatedModel: AdeptRouterModelData = {
...modelData,
model_name: values.adept_router_name,
litellm_params: {
...(modelData.litellm_params ?? {}),
...updatedLitellmParams,
},
};
onSuccess(updatedModel);
onCancel();
} catch (error) {
console.error("Failed to update ADEPT router:", error);
toast.fromError("Failed to update ADEPT router: " + error);
} finally {
setSaving(false);
}
};
return (
<Dialog open={isVisible} onOpenChange={(open) => !open && onCancel()}>
<DialogContent className="max-h-[90vh] overflow-y-auto sm:max-w-2xl">
<DialogHeader>
<DialogTitle>Edit ADEPT Router</DialogTitle>
<DialogDescription>
Update the default model, trainer, or Postgres connection. The password is only written on save when you
enter a new value.
</DialogDescription>
</DialogHeader>
<form onSubmit={form.handleSubmit(handleSave)} noValidate>
<FieldGroup>
<FormField
control={form.control}
name="adept_router_name"
label="Router Name"
description="Changing this rewrites the model group and the alias (adept/<name>)."
>
{({ ref, ...field }) => <Input {...field} ref={ref} value={field.value ?? ""} />}
</FormField>
<FormField control={form.control} name="adept_router_default_model" label="Default Model">
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
<ModelChoiceCombobox
id={id}
value={value ?? ""}
onChange={onChange}
choices={modelChoices}
placeholder="Pick a fallback model"
ariaInvalid={ariaInvalid}
ariaDescribedBy={ariaDescribedBy}
/>
)}
</FormField>
<FormField control={form.control} name="adept_router_tag_prefix" label="XML Tag Prefix">
{({ ref, ...field }) => <Input {...field} ref={ref} value={field.value ?? ""} />}
</FormField>
<FormField
control={form.control}
name="adept_router_conversations_threshold"
label="Conversations Threshold"
>
{({ ref, value, onChange, ...field }) => (
<Input
{...field}
ref={ref}
type="number"
min={1}
value={value ?? ""}
onChange={(event) => onChange(event.target.value === "" ? null : event.target.valueAsNumber)}
/>
)}
</FormField>
<FormField control={form.control} name="adept_router_trainer_url" label="Trainer URL">
{({ ref, ...field }) => <Input {...field} ref={ref} type="url" value={field.value ?? ""} />}
</FormField>
<FormField control={form.control} name="adept_router_pg_host" label="PostgreSQL Host">
{({ ref, ...field }) => <Input {...field} ref={ref} value={field.value ?? ""} />}
</FormField>
<FormField control={form.control} name="adept_router_pg_port" label="PostgreSQL Port">
{({ ref, value, onChange, ...field }) => (
<Input
{...field}
ref={ref}
type="number"
min={1}
max={65535}
value={value ?? ""}
onChange={(event) => onChange(event.target.value === "" ? null : event.target.valueAsNumber)}
/>
)}
</FormField>
<FormField control={form.control} name="adept_router_pg_database" label="PostgreSQL Database">
{({ ref, ...field }) => <Input {...field} ref={ref} value={field.value ?? ""} />}
</FormField>
<FormField control={form.control} name="adept_router_pg_user" label="PostgreSQL User">
{({ ref, ...field }) => <Input {...field} ref={ref} value={field.value ?? ""} />}
</FormField>
<FormField
control={form.control}
name="adept_router_pg_password"
label="PostgreSQL Password"
description={pgPasswordAlreadySet ? "Leave blank to keep the current password." : undefined}
>
{({ ref, ...field }) => (
<Input
{...field}
ref={ref}
type="password"
value={field.value ?? ""}
placeholder={pgPasswordAlreadySet ? "Password stored" : ""}
autoComplete="new-password"
/>
)}
</FormField>
<FormField control={form.control} name="adept_router_pg_ssl_mode" label="PostgreSQL SSL Mode">
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
<Select value={value ?? "prefer"} onValueChange={onChange}>
<SelectTrigger id={id} aria-invalid={ariaInvalid} aria-describedby={ariaDescribedBy}>
<SelectValue placeholder="prefer" />
</SelectTrigger>
<SelectContent>
{PG_SSL_MODES.map((mode) => (
<SelectItem key={mode} value={mode}>
{mode}
</SelectItem>
))}
</SelectContent>
</Select>
)}
</FormField>
</FieldGroup>
<DialogFooter>
<Button variant="outline" type="button" onClick={onCancel} disabled={saving}>
Cancel
</Button>
<Button type="submit" disabled={saving}>
{saving ? "Saving..." : "Save"}
</Button>
</DialogFooter>
</form>
</DialogContent>
</Dialog>
);
};
export default EditAdeptRouterModal;

9
ui/litellm-dashboard/src/components/model_info_view.tsx
View file

@ -26,6 +26,7 @@ import { canModifyModel } from "@/utils/modelPermissions";
import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams";
import DeleteResourceModal from "./common_components/DeleteResourceModal";
import EditAutoRouterModal from "./edit_auto_router/edit_auto_router_modal";
import AdeptRouterEditControl from "./edit_adept_router/AdeptRouterEditControl";
import ReuseCredentialsModal from "./model_add/reuse_credentials";
import { toast } from "@/lib/toast";
import {
@ -578,6 +579,7 @@ export default function ModelInfoView({
onModelUpdate(updatedModel);
}
};
const isWildcardModel = modelData.litellm_model_name.includes("*");
const wildcardProvider = modelData.litellm_model_name.split("/")[0];
const healthCheckModelOptions =
@ -748,6 +750,13 @@ export default function ModelInfoView({
Edit Auto Router
</Button>
)}
<AdeptRouterEditControl
canEdit={canEditModel}
isEditing={isEditing}
modelData={localModelData || modelData}
accessToken={accessToken || ""}
onUpdated={handleAutoRouterUpdate}
/>
{canEditModel ? (
!isEditing && (
<Button onClick={() => setIsEditing(true)} className="flex items-center">

48
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -29625,6 +29625,30 @@ export interface components {
} | null;
/** Adaptive Router Default Model */
adaptive_router_default_model?: string | null;
/** Adept Router Conversations Threshold */
adept_router_conversations_threshold?: number | null;
/** Adept Router Default Model */
adept_router_default_model?: string | null;
/** Adept Router Pg Database */
adept_router_pg_database?: string | null;
/** Adept Router Pg Host */
adept_router_pg_host?: string | null;
/** Adept Router Pg Password */
adept_router_pg_password?: string | null;
/** Adept Router Pg Port */
adept_router_pg_port?: number | null;
/** Adept Router Pg Ssl Mode */
adept_router_pg_ssl_mode?: string | null;
/** Adept Router Pg User */
adept_router_pg_user?: string | null;
/** Adept Router Seed Config */
adept_router_seed_config?: {
[key: string]: unknown;
}[] | null;
/** Adept Router Tag Prefix */
adept_router_tag_prefix?: string | null;
/** Adept Router Trainer Url */
adept_router_trainer_url?: string | null;
/**
* Allow Client Keepalive Override
* @default false
@ -39851,6 +39875,30 @@ export interface components {
} | null;
/** Adaptive Router Default Model */
adaptive_router_default_model?: string | null;
/** Adept Router Conversations Threshold */
adept_router_conversations_threshold?: number | null;
/** Adept Router Default Model */
adept_router_default_model?: string | null;
/** Adept Router Pg Database */
adept_router_pg_database?: string | null;
/** Adept Router Pg Host */
adept_router_pg_host?: string | null;
/** Adept Router Pg Password */
adept_router_pg_password?: string | null;
/** Adept Router Pg Port */
adept_router_pg_port?: number | null;
/** Adept Router Pg Ssl Mode */
adept_router_pg_ssl_mode?: string | null;
/** Adept Router Pg User */
adept_router_pg_user?: string | null;
/** Adept Router Seed Config */
adept_router_seed_config?: {
[key: string]: unknown;
}[] | null;
/** Adept Router Tag Prefix */
adept_router_tag_prefix?: string | null;
/** Adept Router Trainer Url */
adept_router_trainer_url?: string | null;
/**
* Allow Client Keepalive Override
* @default false