mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-06 07:48:06 +00:00
Merge branch 'litellm_internal_staging' into litellm_guardrails_monitor_table_shadcn_0812
This commit is contained in:
yuneng-jiang
GitHub
commit
0bc737f60d
33 changed files with 1564 additions and 108 deletions
|
|
@ -29,7 +29,7 @@ End-to-end tests belong in `tests/e2e/` and must follow the harness conventions
|
|||
|
||||
When creating PRs, don't set base to `main`. `litellm_internal_staging` is the default base branch and serves that purpose for both internal and external / OSS contributions
|
||||
|
||||
When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
|
||||
When writing a PR body, treat the comments and imperative instructions inside .github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
|
||||
|
||||
Same applies for filing bug reports and feature requests, with .github/ISSUE_TEMPLATE/bug_report.yml and .github/ISSUE_TEMPLATE/feature_request.yml, respectively
|
||||
|
||||
|
|
|
|||
|
|
@ -90,7 +90,6 @@ class LangfuseOtelLogger(OpenTelemetry):
|
|||
"generation_name": LangfuseSpanAttributes.GENERATION_NAME,
|
||||
"generation_id": LangfuseSpanAttributes.GENERATION_ID,
|
||||
"parent_observation_id": LangfuseSpanAttributes.PARENT_OBSERVATION_ID,
|
||||
"version": LangfuseSpanAttributes.GENERATION_VERSION,
|
||||
"mask_input": LangfuseSpanAttributes.MASK_INPUT,
|
||||
"mask_output": LangfuseSpanAttributes.MASK_OUTPUT,
|
||||
"trace_user_id": LangfuseSpanAttributes.TRACE_USER_ID,
|
||||
|
|
@ -99,13 +98,18 @@ class LangfuseOtelLogger(OpenTelemetry):
|
|||
"trace_name": LangfuseSpanAttributes.TRACE_NAME,
|
||||
"trace_id": LangfuseSpanAttributes.TRACE_ID,
|
||||
"trace_metadata": LangfuseSpanAttributes.TRACE_METADATA,
|
||||
"trace_version": LangfuseSpanAttributes.TRACE_VERSION,
|
||||
"trace_release": LangfuseSpanAttributes.TRACE_RELEASE,
|
||||
"trace_release": LangfuseSpanAttributes.RELEASE,
|
||||
"existing_trace_id": LangfuseSpanAttributes.EXISTING_TRACE_ID,
|
||||
"update_trace_keys": LangfuseSpanAttributes.UPDATE_TRACE_KEYS,
|
||||
"debug_langfuse": LangfuseSpanAttributes.DEBUG_LANGFUSE,
|
||||
}
|
||||
|
||||
version: Final = (
|
||||
metadata.get("trace_version") if metadata.get("trace_version") is not None else metadata.get("version")
|
||||
)
|
||||
if version is not None:
|
||||
safe_set_attribute(span, LangfuseSpanAttributes.VERSION.value, version)
|
||||
|
||||
for key, enum_attr in mapping.items():
|
||||
if key in metadata and metadata[key] is not None:
|
||||
value = metadata[key]
|
||||
|
|
|
|||
|
|
@ -42,9 +42,7 @@ def langfuse_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str,
|
|||
public_key: Final = params.get("langfuse_public_key")
|
||||
secret_key: Final = params.get("langfuse_secret_key")
|
||||
if public_key and secret_key:
|
||||
return {
|
||||
"Authorization": _V1Langfuse._get_langfuse_authorization_header(
|
||||
public_key=public_key, secret_key=secret_key
|
||||
)
|
||||
}
|
||||
return _V1Langfuse._build_langfuse_otel_headers(
|
||||
_V1Langfuse._get_langfuse_authorization_header(public_key=public_key, secret_key=secret_key)
|
||||
)
|
||||
return {}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# What is this?
|
||||
## Helper utilities
|
||||
import copy
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal
|
||||
|
||||
import httpx
|
||||
|
|
@ -181,7 +181,7 @@ def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metad
|
|||
|
||||
|
||||
def get_metadata_variable_name_from_kwargs(
|
||||
kwargs: dict,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> Literal["metadata", "litellm_metadata"]:
|
||||
"""
|
||||
Helper to return what the "metadata" field should be called in the request data
|
||||
|
|
|
|||
|
|
@ -26120,11 +26120,12 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"groq/llama-3.1-8b-instant": {
|
||||
"deprecation_date": "2026-08-16",
|
||||
"input_cost_per_token": 5e-08,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 8e-08,
|
||||
"supports_function_calling": true,
|
||||
|
|
@ -26132,9 +26133,10 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"groq/llama-3.3-70b-versatile": {
|
||||
"deprecation_date": "2026-08-16",
|
||||
"input_cost_per_token": 5.9e-07,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 128000,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"mode": "chat",
|
||||
|
|
@ -26155,7 +26157,28 @@
|
|||
"supports_response_schema": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"groq/meta-llama/llama-prompt-guard-2-22m": {
|
||||
"input_cost_per_token": 3e-08,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 512,
|
||||
"max_output_tokens": 512,
|
||||
"max_tokens": 512,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-08,
|
||||
"source": "https://console.groq.com/docs/models"
|
||||
},
|
||||
"groq/meta-llama/llama-prompt-guard-2-86m": {
|
||||
"input_cost_per_token": 4e-08,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 512,
|
||||
"max_output_tokens": 512,
|
||||
"max_tokens": 512,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4e-08,
|
||||
"source": "https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m"
|
||||
},
|
||||
"groq/meta-llama/llama-guard-4-12b": {
|
||||
"deprecation_date": "2026-03-05",
|
||||
"input_cost_per_token": 2e-07,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 8192,
|
||||
|
|
@ -26165,6 +26188,7 @@
|
|||
"output_cost_per_token": 2e-07
|
||||
},
|
||||
"groq/meta-llama/llama-4-maverick-17b-128e-instruct": {
|
||||
"deprecation_date": "2026-03-09",
|
||||
"input_cost_per_token": 2e-07,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -26178,6 +26202,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"groq/meta-llama/llama-4-scout-17b-16e-instruct": {
|
||||
"deprecation_date": "2026-07-17",
|
||||
"input_cost_per_token": 1.1e-07,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -26191,6 +26216,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"groq/moonshotai/kimi-k2-instruct-0905": {
|
||||
"deprecation_date": "2026-04-15",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"output_cost_per_token": 3e-06,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -26208,8 +26234,8 @@
|
|||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32766,
|
||||
"max_tokens": 32766,
|
||||
"max_output_tokens": 65536,
|
||||
"max_tokens": 65536,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"search_context_cost_per_query": {
|
||||
|
|
@ -26229,8 +26255,8 @@
|
|||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"max_output_tokens": 65536,
|
||||
"max_tokens": 65536,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-07,
|
||||
"search_context_cost_per_query": {
|
||||
|
|
@ -26265,7 +26291,26 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_web_search": true
|
||||
},
|
||||
"groq/canopylabs/orpheus-v1-english": {
|
||||
"input_cost_per_character": 2.2e-05,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 4000,
|
||||
"max_output_tokens": 50000,
|
||||
"max_tokens": 50000,
|
||||
"mode": "audio_speech",
|
||||
"source": "https://console.groq.com/docs/model/canopylabs/orpheus-v1-english"
|
||||
},
|
||||
"groq/canopylabs/orpheus-arabic-saudi": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 4000,
|
||||
"max_output_tokens": 50000,
|
||||
"max_tokens": 50000,
|
||||
"mode": "audio_speech",
|
||||
"source": "https://console.groq.com/docs/models"
|
||||
},
|
||||
"groq/playai-tts": {
|
||||
"deprecation_date": "2025-12-31",
|
||||
"input_cost_per_character": 5e-05,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 10000,
|
||||
|
|
@ -26273,7 +26318,23 @@
|
|||
"max_tokens": 10000,
|
||||
"mode": "audio_speech"
|
||||
},
|
||||
"groq/qwen/qwen3.6-27b": {
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 16384,
|
||||
"max_tokens": 16384,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-06,
|
||||
"source": "https://console.groq.com/docs/model/qwen/qwen3.6-27b",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"groq/qwen/qwen3-32b": {
|
||||
"deprecation_date": "2026-07-17",
|
||||
"input_cost_per_token": 2.9e-07,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 131000,
|
||||
|
|
@ -45826,11 +45887,15 @@
|
|||
},
|
||||
"bedrock_mantle/openai.gpt-5.6-sol": {
|
||||
"input_cost_per_token": 5.5e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 1.1e-05,
|
||||
"cache_creation_input_token_cost": 6.875e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 1.375e-05,
|
||||
"cache_read_input_token_cost": 5.5e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
|
||||
"output_cost_per_token": 3.3e-05,
|
||||
"output_cost_per_token_above_272k_tokens": 4.95e-05,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"max_input_tokens": 272000,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "responses",
|
||||
|
|
@ -45854,11 +45919,15 @@
|
|||
},
|
||||
"bedrock_mantle/openai.gpt-5.6-terra": {
|
||||
"input_cost_per_token": 2.2e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 4.4e-06,
|
||||
"cache_creation_input_token_cost": 2.75e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5.5e-06,
|
||||
"cache_read_input_token_cost": 2.2e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 4.4e-07,
|
||||
"output_cost_per_token": 1.32e-05,
|
||||
"output_cost_per_token_above_272k_tokens": 1.98e-05,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"max_input_tokens": 272000,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "responses",
|
||||
|
|
@ -45882,11 +45951,15 @@
|
|||
},
|
||||
"bedrock_mantle/openai.gpt-5.6-luna": {
|
||||
"input_cost_per_token": 2.2e-07,
|
||||
"input_cost_per_token_above_272k_tokens": 4.4e-07,
|
||||
"cache_creation_input_token_cost": 2.75e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5.5e-07,
|
||||
"cache_read_input_token_cost": 2.2e-08,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 4.4e-08,
|
||||
"output_cost_per_token": 1.32e-06,
|
||||
"output_cost_per_token_above_272k_tokens": 1.98e-06,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"max_input_tokens": 272000,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "responses",
|
||||
|
|
|
|||
|
|
@ -845,6 +845,22 @@ def cleanup_router_config_variables():
|
|||
prisma_client = None
|
||||
|
||||
|
||||
async def _flush_spend_logs_queue_on_shutdown() -> None:
|
||||
if prisma_client is None:
|
||||
return
|
||||
|
||||
try:
|
||||
from litellm.proxy.utils import drain_spend_logs_queue
|
||||
|
||||
await drain_spend_logs_queue(
|
||||
prisma_client=prisma_client,
|
||||
db_writer_client=db_writer_client,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # shutdown must continue even if the drain fails
|
||||
verbose_proxy_logger.exception("Error flushing spend logs queue on shutdown: %s", e)
|
||||
|
||||
|
||||
async def proxy_shutdown_event():
|
||||
global prisma_client, master_key, user_custom_auth, user_custom_key_generate, user_custom_key_update
|
||||
verbose_proxy_logger.info("Shutting down LiteLLM Proxy Server")
|
||||
|
|
@ -1255,6 +1271,8 @@ async def proxy_startup_event(app: FastAPI):
|
|||
except Exception as e:
|
||||
verbose_proxy_logger.error("Error stopping DB health watchdog task: %s", e)
|
||||
|
||||
await _flush_spend_logs_queue_on_shutdown()
|
||||
|
||||
await proxy_config.stop_config_sync_subscriber()
|
||||
|
||||
await proxy_config.stop_auth_cache_invalidation_subscriber()
|
||||
|
|
@ -8731,14 +8749,14 @@ class ProxyStartupEvent:
|
|||
if general_settings.get("disable_spend_logs", False) is False:
|
||||
from litellm.proxy.utils import _monitor_spend_logs_queue
|
||||
|
||||
# Start background task to monitor spend logs queue size
|
||||
asyncio.create_task(
|
||||
monitor_task: Final = asyncio.create_task(
|
||||
_monitor_spend_logs_queue(
|
||||
prisma_client=prisma_client,
|
||||
db_writer_client=db_writer_client,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
)
|
||||
prisma_client.spend_logs_queue_monitor_task = monitor_task # rebind-ok: the client owns its monitor handle
|
||||
|
||||
### ADD NEW MODELS ###
|
||||
store_model_in_db = get_secret_bool("STORE_MODEL_IN_DB", store_model_in_db) or store_model_in_db
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import contextlib
|
||||
import copy
|
||||
import hashlib
|
||||
import inspect
|
||||
|
|
@ -3006,6 +3007,7 @@ async def prefetch_config_params(prisma_client: "PrismaClient | None", param_nam
|
|||
class PrismaClient:
|
||||
spend_log_transactions: list = []
|
||||
_spend_log_transactions_lock = asyncio.Lock()
|
||||
spend_logs_queue_monitor_task: "asyncio.Task[None] | None" = None
|
||||
tool_usage_transactions: list["ToolUsageTransaction"] = []
|
||||
_tool_usage_transactions_lock = asyncio.Lock()
|
||||
autorouter_turn_transactions: ClassVar[
|
||||
|
|
@ -5722,13 +5724,22 @@ async def update_spend_logs_job(
|
|||
logs_to_process: Final = prisma_client.spend_log_transactions[:MAX_LOGS_PER_INTERVAL]
|
||||
prisma_client.spend_log_transactions = prisma_client.spend_log_transactions[len(logs_to_process) :]
|
||||
|
||||
await ProxyUpdateSpend.update_spend_logs(
|
||||
n_retry_times=n_retry_times,
|
||||
prisma_client=prisma_client,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
db_writer_client=db_writer_client,
|
||||
logs_to_process=logs_to_process,
|
||||
)
|
||||
try:
|
||||
await ProxyUpdateSpend.update_spend_logs(
|
||||
n_retry_times=n_retry_times,
|
||||
prisma_client=prisma_client,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
db_writer_client=db_writer_client,
|
||||
logs_to_process=logs_to_process,
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
async with prisma_client._spend_log_transactions_lock:
|
||||
prisma_client.spend_log_transactions[:0] = logs_to_process
|
||||
verbose_proxy_logger.warning(
|
||||
"Spend tracking - spend log write cancelled, requeued %d rows for the next flush",
|
||||
len(logs_to_process),
|
||||
)
|
||||
raise
|
||||
|
||||
# Guardrail/policy usage tracking (same batch, outside spend-logs update)
|
||||
try:
|
||||
|
|
@ -5787,6 +5798,39 @@ async def update_spend_logs_job(
|
|||
)
|
||||
|
||||
|
||||
MAX_SPEND_LOG_DRAIN_ITERATIONS: Final = 20
|
||||
|
||||
|
||||
async def drain_spend_logs_queue(
|
||||
prisma_client: PrismaClient,
|
||||
db_writer_client: "AsyncHTTPHandler | None",
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
) -> None:
|
||||
monitor_task: Final = prisma_client.spend_logs_queue_monitor_task
|
||||
if monitor_task is not None:
|
||||
monitor_task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await monitor_task
|
||||
prisma_client.spend_logs_queue_monitor_task = None # rebind-ok: the client owns its monitor handle
|
||||
|
||||
for _ in range(MAX_SPEND_LOG_DRAIN_ITERATIONS):
|
||||
if await _total_queued_spend_transactions(prisma_client) == 0:
|
||||
return
|
||||
await update_spend_logs_job(
|
||||
prisma_client=prisma_client,
|
||||
db_writer_client=db_writer_client,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
remaining: Final = await _total_queued_spend_transactions(prisma_client)
|
||||
if remaining > 0:
|
||||
spend_log_error(
|
||||
"Spend tracking - %d spend log rows still queued after %d drain passes",
|
||||
remaining,
|
||||
MAX_SPEND_LOG_DRAIN_ITERATIONS,
|
||||
)
|
||||
|
||||
|
||||
async def _monitor_spend_logs_queue(
|
||||
prisma_client: PrismaClient,
|
||||
db_writer_client: AsyncHTTPHandler | None,
|
||||
|
|
|
|||
|
|
@ -96,6 +96,7 @@ from litellm.router_utils.add_retry_fallback_headers import (
|
|||
response_in_flight_token_count,
|
||||
)
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
AUTO_ROUTER_MODEL_PREFIX,
|
||||
classify_strategy_router_model,
|
||||
)
|
||||
from litellm.router_utils.batch_utils import (
|
||||
|
|
@ -318,6 +319,8 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None
|
|||
|
||||
_PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT")
|
||||
|
||||
_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"})
|
||||
|
||||
|
||||
def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) -> bool:
|
||||
for chunk in chunks:
|
||||
|
|
@ -10712,6 +10715,14 @@ class Router:
|
|||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _is_strategy_marker_deployment(deployment: Mapping[str, object]) -> bool:
|
||||
litellm_params: Final = deployment.get("litellm_params")
|
||||
if not isinstance(litellm_params, Mapping):
|
||||
return False
|
||||
deployment_model: Final = litellm_params.get("model")
|
||||
return isinstance(deployment_model, str) and classify_strategy_router_model(deployment_model) is not None
|
||||
|
||||
def _common_checks_available_deployment(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -10839,7 +10850,12 @@ class Router:
|
|||
model
|
||||
] # update the model to the actual value if an alias has been passed in
|
||||
|
||||
return model, healthy_deployments
|
||||
marker_flags: Final = tuple(self._is_strategy_marker_deployment(d) for d in healthy_deployments)
|
||||
if all(marker_flags) or not any(marker_flags):
|
||||
return model, healthy_deployments
|
||||
return model, [ # mutable-ok: matches this function's list contract expected by downstream filters
|
||||
d for d, is_marker in zip(healthy_deployments, marker_flags, strict=True) if not is_marker
|
||||
]
|
||||
|
||||
def _filter_deployments_by_model_access_groups(
|
||||
self,
|
||||
|
|
@ -11352,6 +11368,10 @@ class Router:
|
|||
|
||||
return filtered
|
||||
|
||||
def _model_name_has_plain_deployments(self, model: str) -> bool:
|
||||
indices: Final = self.model_name_to_deployment_indices.get(model) or ()
|
||||
return any(not self._is_strategy_marker_deployment(self.model_list[idx]) for idx in indices)
|
||||
|
||||
def _select_pre_routing_strategy(
|
||||
self, model: str, request_kwargs: dict
|
||||
) -> "TaggedPreRoutingStrategy[PreRoutingStrategy] | None":
|
||||
|
|
@ -11360,7 +11380,14 @@ class Router:
|
|||
that share a `model_name` by matching the request's tags against each
|
||||
registered strategy's tags before falling back to the first registered.
|
||||
Returns the tagged registry entry so the caller can tell whether the
|
||||
request's tags were what selected it.
|
||||
request's tags were what selected it, and can locate the marker
|
||||
deployment the strategy was registered from via its (model_name, tags)
|
||||
pair.
|
||||
|
||||
With tag filtering enabled, strategies that all carry real tags matching
|
||||
none of the request's do not capture it when the name also has plain
|
||||
deployments: returning None hands the request to ordinary tag-aware
|
||||
deployment selection.
|
||||
"""
|
||||
candidates: Final[list[TaggedPreRoutingStrategy[PreRoutingStrategy]]] = [
|
||||
*self.auto_routers.get(model, []),
|
||||
|
|
@ -11370,8 +11397,6 @@ class Router:
|
|||
]
|
||||
if not candidates:
|
||||
return None
|
||||
if len(candidates) == 1:
|
||||
return candidates[0]
|
||||
|
||||
request_tags: Final = _get_tags_from_request_kwargs(request_kwargs)
|
||||
if request_tags:
|
||||
|
|
@ -11383,6 +11408,12 @@ class Router:
|
|||
for tagged in candidates:
|
||||
if "default" in tagged.tags:
|
||||
return tagged
|
||||
if (
|
||||
self.enable_tag_filtering
|
||||
and all(tagged.tags for tagged in candidates)
|
||||
and self._model_name_has_plain_deployments(model)
|
||||
):
|
||||
return None
|
||||
return candidates[0]
|
||||
|
||||
async def async_pre_routing_hook(
|
||||
|
|
@ -11445,25 +11476,47 @@ class Router:
|
|||
)
|
||||
|
||||
# `model` (the alias, e.g. "smart-router") is never the deployment actually
|
||||
# called - apply the alias's own litellm_params (besides `model` itself,
|
||||
# which is just the alias marker) to the request, since the tier/route
|
||||
# deployment the hook selected won't have them. Router-only fields
|
||||
# (tpm, rpm, weight, complexity_router_config, ...) are excluded from the
|
||||
# actual outbound LLM call downstream by litellm.types.utils.all_litellm_params,
|
||||
# called - apply the router marker's own litellm_params to the request,
|
||||
# since the tier/route deployment the hook selected won't have them. The
|
||||
# marker entry is looked up by its `auto_router/` model prefix and the
|
||||
# selected strategy's tags, never by list position: plain deployments may
|
||||
# share the alias `model_name` and must not leak their params (`api_base`,
|
||||
# `api_key`, ...) onto the routed call. Router-only fields (tpm, rpm,
|
||||
# weight, complexity_router_config, ...) are excluded from the actual
|
||||
# outbound LLM call downstream by litellm.types.utils.all_litellm_params,
|
||||
# not here. Custom pricing fields ARE call params, so they must be
|
||||
# excluded here: they price the alias, not the deployment the hook
|
||||
# selected, and forwarding them re-registers the routed deployment at
|
||||
# the alias's price (an explicit 0 makes every alias request bill $0).
|
||||
if pre_routing_hook_response is not None:
|
||||
alias_index: Final = self.model_name_to_deployment_indices.get(model, [])
|
||||
if alias_index:
|
||||
alias_litellm_params: Final = self.model_list[alias_index[0]].get("litellm_params", {})
|
||||
for key, value in alias_litellm_params.items():
|
||||
if key != "model" and key not in CustomPricingLiteLLMParams.model_fields and value is not None:
|
||||
request_kwargs.setdefault(key, value)
|
||||
for key, value in self._forwardable_alias_marker_params(model=model, strategy_tags=selected_strategy.tags):
|
||||
request_kwargs.setdefault(key, value)
|
||||
|
||||
return pre_routing_hook_response
|
||||
|
||||
def _forwardable_alias_marker_params(
|
||||
self, model: str, strategy_tags: tuple[str, ...]
|
||||
) -> tuple[tuple[str, object], ...]:
|
||||
marker_params: Final = tuple(
|
||||
litellm_params
|
||||
for idx in self.model_name_to_deployment_indices.get(model, ())
|
||||
if isinstance(litellm_params := self.model_list[idx].get("litellm_params", {}), dict)
|
||||
and str(litellm_params.get("model", "")).startswith(AUTO_ROUTER_MODEL_PREFIX)
|
||||
)
|
||||
tag_matched: Final = tuple(
|
||||
params for params in marker_params if tuple(params.get("tags") or ()) == strategy_tags
|
||||
)
|
||||
selected: Final = tag_matched[0] if tag_matched else (marker_params[0] if marker_params else None)
|
||||
if selected is None:
|
||||
return ()
|
||||
return tuple(
|
||||
(key, value)
|
||||
for key, value in selected.items()
|
||||
if key not in _ALIAS_PARAMS_NEVER_FORWARDED
|
||||
and key not in CustomPricingLiteLLMParams.model_fields
|
||||
and value is not None
|
||||
)
|
||||
|
||||
def _consumed_request_tags_stamp(
|
||||
self,
|
||||
selected_strategy: "TaggedPreRoutingStrategy[PreRoutingStrategy]",
|
||||
|
|
|
|||
|
|
@ -584,8 +584,26 @@ async def get_deployments_for_tag(
|
|||
return healthy_deployments
|
||||
|
||||
|
||||
def _tags_in_metadata(metadata: object) -> list[str]:
|
||||
"""
|
||||
Tags out of a metadata bucket the caller controls the shape of.
|
||||
|
||||
A request can send its metadata (and its ``tags``) as anything the JSON body
|
||||
allowed, an unparsed string or null included, so any shape that is not a list
|
||||
of string tags carries no tags rather than raising.
|
||||
"""
|
||||
if not isinstance(metadata, Mapping):
|
||||
return []
|
||||
typed_metadata: Final[Mapping[str, object]] = metadata
|
||||
tags: Final = typed_metadata.get("tags")
|
||||
if isinstance(tags, str) or not isinstance(tags, Sequence):
|
||||
return []
|
||||
typed_tags: Final[Sequence[object]] = tags
|
||||
return [tag for tag in typed_tags if isinstance(tag, str)]
|
||||
|
||||
|
||||
def _get_tags_from_request_kwargs(
|
||||
request_kwargs: dict[Any, Any] | None = None,
|
||||
request_kwargs: Mapping[Any, Any] | None = None,
|
||||
metadata_variable_name: Literal["metadata", "litellm_metadata"] | None = None,
|
||||
) -> list[str]:
|
||||
"""
|
||||
|
|
@ -604,12 +622,11 @@ def _get_tags_from_request_kwargs(
|
|||
return []
|
||||
resolved_variable_name: Final = metadata_variable_name or get_metadata_variable_name_from_kwargs(request_kwargs)
|
||||
if resolved_variable_name in request_kwargs:
|
||||
metadata: Final = request_kwargs[resolved_variable_name] or {}
|
||||
tags = metadata.get("tags", [])
|
||||
return tags if tags is not None else []
|
||||
elif "litellm_params" in request_kwargs:
|
||||
litellm_params: Final = request_kwargs["litellm_params"] or {}
|
||||
_metadata: Final = litellm_params.get(resolved_variable_name, {}) or {}
|
||||
tags = _metadata.get("tags", [])
|
||||
return tags if tags is not None else []
|
||||
return _tags_in_metadata(request_kwargs[resolved_variable_name])
|
||||
if "litellm_params" in request_kwargs:
|
||||
litellm_params: Final = request_kwargs["litellm_params"]
|
||||
if not isinstance(litellm_params, Mapping):
|
||||
return []
|
||||
typed_litellm_params: Final[Mapping[str, object]] = litellm_params
|
||||
return _tags_in_metadata(typed_litellm_params.get(resolved_variable_name))
|
||||
return []
|
||||
|
|
|
|||
|
|
@ -16,12 +16,13 @@ class LangfuseOtelConfig(BaseModel):
|
|||
|
||||
class LangfuseSpanAttributes(str, Enum):
|
||||
LANGFUSE_ENVIRONMENT = "langfuse.environment"
|
||||
VERSION = "langfuse.version"
|
||||
RELEASE = "langfuse.release"
|
||||
|
||||
# ---- Generation-level metadata ----
|
||||
GENERATION_NAME = "langfuse.generation.name"
|
||||
GENERATION_ID = "langfuse.generation.id"
|
||||
PARENT_OBSERVATION_ID = "langfuse.generation.parent_observation_id"
|
||||
GENERATION_VERSION = "langfuse.generation.version"
|
||||
MASK_INPUT = "langfuse.generation.mask_input"
|
||||
MASK_OUTPUT = "langfuse.generation.mask_output"
|
||||
|
||||
|
|
@ -36,8 +37,6 @@ class LangfuseSpanAttributes(str, Enum):
|
|||
TRACE_NAME = "langfuse.trace.name"
|
||||
TRACE_ID = "langfuse.trace.id"
|
||||
TRACE_METADATA = "langfuse.trace.metadata"
|
||||
TRACE_VERSION = "langfuse.trace.version"
|
||||
TRACE_RELEASE = "langfuse.trace.release"
|
||||
EXISTING_TRACE_ID = "langfuse.trace.existing_id"
|
||||
UPDATE_TRACE_KEYS = "langfuse.trace.update_keys"
|
||||
|
||||
|
|
|
|||
|
|
@ -26120,11 +26120,12 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"groq/llama-3.1-8b-instant": {
|
||||
"deprecation_date": "2026-08-16",
|
||||
"input_cost_per_token": 5e-08,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 8e-08,
|
||||
"supports_function_calling": true,
|
||||
|
|
@ -26132,9 +26133,10 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"groq/llama-3.3-70b-versatile": {
|
||||
"deprecation_date": "2026-08-16",
|
||||
"input_cost_per_token": 5.9e-07,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 128000,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"mode": "chat",
|
||||
|
|
@ -26155,7 +26157,28 @@
|
|||
"supports_response_schema": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"groq/meta-llama/llama-prompt-guard-2-22m": {
|
||||
"input_cost_per_token": 3e-08,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 512,
|
||||
"max_output_tokens": 512,
|
||||
"max_tokens": 512,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-08,
|
||||
"source": "https://console.groq.com/docs/models"
|
||||
},
|
||||
"groq/meta-llama/llama-prompt-guard-2-86m": {
|
||||
"input_cost_per_token": 4e-08,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 512,
|
||||
"max_output_tokens": 512,
|
||||
"max_tokens": 512,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4e-08,
|
||||
"source": "https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m"
|
||||
},
|
||||
"groq/meta-llama/llama-guard-4-12b": {
|
||||
"deprecation_date": "2026-03-05",
|
||||
"input_cost_per_token": 2e-07,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 8192,
|
||||
|
|
@ -26165,6 +26188,7 @@
|
|||
"output_cost_per_token": 2e-07
|
||||
},
|
||||
"groq/meta-llama/llama-4-maverick-17b-128e-instruct": {
|
||||
"deprecation_date": "2026-03-09",
|
||||
"input_cost_per_token": 2e-07,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -26178,6 +26202,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"groq/meta-llama/llama-4-scout-17b-16e-instruct": {
|
||||
"deprecation_date": "2026-07-17",
|
||||
"input_cost_per_token": 1.1e-07,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -26191,6 +26216,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"groq/moonshotai/kimi-k2-instruct-0905": {
|
||||
"deprecation_date": "2026-04-15",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"output_cost_per_token": 3e-06,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
|
|
@ -26208,8 +26234,8 @@
|
|||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32766,
|
||||
"max_tokens": 32766,
|
||||
"max_output_tokens": 65536,
|
||||
"max_tokens": 65536,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"search_context_cost_per_query": {
|
||||
|
|
@ -26229,8 +26255,8 @@
|
|||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"max_output_tokens": 65536,
|
||||
"max_tokens": 65536,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-07,
|
||||
"search_context_cost_per_query": {
|
||||
|
|
@ -26265,7 +26291,26 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_web_search": true
|
||||
},
|
||||
"groq/canopylabs/orpheus-v1-english": {
|
||||
"input_cost_per_character": 2.2e-05,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 4000,
|
||||
"max_output_tokens": 50000,
|
||||
"max_tokens": 50000,
|
||||
"mode": "audio_speech",
|
||||
"source": "https://console.groq.com/docs/model/canopylabs/orpheus-v1-english"
|
||||
},
|
||||
"groq/canopylabs/orpheus-arabic-saudi": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 4000,
|
||||
"max_output_tokens": 50000,
|
||||
"max_tokens": 50000,
|
||||
"mode": "audio_speech",
|
||||
"source": "https://console.groq.com/docs/models"
|
||||
},
|
||||
"groq/playai-tts": {
|
||||
"deprecation_date": "2025-12-31",
|
||||
"input_cost_per_character": 5e-05,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 10000,
|
||||
|
|
@ -26273,7 +26318,23 @@
|
|||
"max_tokens": 10000,
|
||||
"mode": "audio_speech"
|
||||
},
|
||||
"groq/qwen/qwen3.6-27b": {
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 16384,
|
||||
"max_tokens": 16384,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-06,
|
||||
"source": "https://console.groq.com/docs/model/qwen/qwen3.6-27b",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"groq/qwen/qwen3-32b": {
|
||||
"deprecation_date": "2026-07-17",
|
||||
"input_cost_per_token": 2.9e-07,
|
||||
"litellm_provider": "groq",
|
||||
"max_input_tokens": 131000,
|
||||
|
|
@ -45826,11 +45887,15 @@
|
|||
},
|
||||
"bedrock_mantle/openai.gpt-5.6-sol": {
|
||||
"input_cost_per_token": 5.5e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 1.1e-05,
|
||||
"cache_creation_input_token_cost": 6.875e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 1.375e-05,
|
||||
"cache_read_input_token_cost": 5.5e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
|
||||
"output_cost_per_token": 3.3e-05,
|
||||
"output_cost_per_token_above_272k_tokens": 4.95e-05,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"max_input_tokens": 272000,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "responses",
|
||||
|
|
@ -45854,11 +45919,15 @@
|
|||
},
|
||||
"bedrock_mantle/openai.gpt-5.6-terra": {
|
||||
"input_cost_per_token": 2.2e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 4.4e-06,
|
||||
"cache_creation_input_token_cost": 2.75e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5.5e-06,
|
||||
"cache_read_input_token_cost": 2.2e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 4.4e-07,
|
||||
"output_cost_per_token": 1.32e-05,
|
||||
"output_cost_per_token_above_272k_tokens": 1.98e-05,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"max_input_tokens": 272000,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "responses",
|
||||
|
|
@ -45882,11 +45951,15 @@
|
|||
},
|
||||
"bedrock_mantle/openai.gpt-5.6-luna": {
|
||||
"input_cost_per_token": 2.2e-07,
|
||||
"input_cost_per_token_above_272k_tokens": 4.4e-07,
|
||||
"cache_creation_input_token_cost": 2.75e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5.5e-07,
|
||||
"cache_read_input_token_cost": 2.2e-08,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 4.4e-08,
|
||||
"output_cost_per_token": 1.32e-06,
|
||||
"output_cost_per_token_above_272k_tokens": 1.98e-06,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"max_input_tokens": 272000,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "responses",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
"""Per-request multi-tenant credential routing (V1 parity)."""
|
||||
|
||||
import base64
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
|
@ -42,6 +43,17 @@ def test_langfuse_dynamic_headers_need_both_keys():
|
|||
assert headers is not None and "Authorization" in headers
|
||||
|
||||
|
||||
def test_langfuse_dynamic_headers_carry_v4_ingestion_version():
|
||||
headers = dynamic_otlp_headers(
|
||||
"langfuse_otel", {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}
|
||||
)
|
||||
expected_auth = "Basic " + base64.b64encode(b"pk:sk").decode()
|
||||
assert headers == {
|
||||
"Authorization": expected_auth,
|
||||
"x-langfuse-ingestion-version": "4",
|
||||
}
|
||||
|
||||
|
||||
def test_weave_dynamic_headers():
|
||||
headers = dynamic_otlp_headers(
|
||||
"weave_otel", {"wandb_api_key": "w", "weave_project_id": "p"}
|
||||
|
|
|
|||
|
|
@ -211,7 +211,7 @@ class TestLangfuseOtelIntegration:
|
|||
LangfuseSpanAttributes.GENERATION_NAME.value: "gen-name",
|
||||
LangfuseSpanAttributes.GENERATION_ID.value: "gen-id",
|
||||
LangfuseSpanAttributes.PARENT_OBSERVATION_ID.value: "parent-id",
|
||||
LangfuseSpanAttributes.GENERATION_VERSION.value: "v1",
|
||||
LangfuseSpanAttributes.VERSION.value: "t-ver",
|
||||
LangfuseSpanAttributes.MASK_INPUT.value: True,
|
||||
LangfuseSpanAttributes.MASK_OUTPUT.value: False,
|
||||
LangfuseSpanAttributes.TRACE_USER_ID.value: "user-123",
|
||||
|
|
@ -221,8 +221,7 @@ class TestLangfuseOtelIntegration:
|
|||
LangfuseSpanAttributes.TRACE_NAME.value: "trace-name",
|
||||
LangfuseSpanAttributes.TRACE_ID.value: "traceid", # stripped dashes
|
||||
LangfuseSpanAttributes.TRACE_METADATA.value: json.dumps({"k": "v"}),
|
||||
LangfuseSpanAttributes.TRACE_VERSION.value: "t-ver",
|
||||
LangfuseSpanAttributes.TRACE_RELEASE.value: "rel-1",
|
||||
LangfuseSpanAttributes.RELEASE.value: "rel-1",
|
||||
LangfuseSpanAttributes.EXISTING_TRACE_ID.value: "existing-id",
|
||||
LangfuseSpanAttributes.UPDATE_TRACE_KEYS.value: json.dumps(
|
||||
["key1", "key2"]
|
||||
|
|
@ -240,6 +239,52 @@ class TestLangfuseOtelIntegration:
|
|||
actual == expected
|
||||
), "Mismatch between expected and actual OTEL attribute mapping."
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"metadata, expected_version",
|
||||
[
|
||||
(
|
||||
{"version": "v-observation", "trace_version": "v-trace"},
|
||||
"v-trace",
|
||||
),
|
||||
({"trace_version": "v-trace"}, "v-trace"),
|
||||
({"version": "v-observation"}, "v-observation"),
|
||||
({"version": "v-observation", "trace_version": ""}, ""),
|
||||
({}, None),
|
||||
],
|
||||
ids=[
|
||||
"trace-version-wins-as-documented",
|
||||
"trace-only",
|
||||
"observation-version-is-the-fallback",
|
||||
"empty-trace-version-is-not-absent",
|
||||
"neither-key-emits-nothing",
|
||||
],
|
||||
)
|
||||
def test_version_emitted_on_langfuse_v4_key(self, metadata, expected_version):
|
||||
kwargs = {"litellm_params": {"metadata": {"trace_release": "rel-9", **metadata}}}
|
||||
|
||||
with patch(
|
||||
"litellm.integrations.arize._utils.safe_set_attribute"
|
||||
) as mock_safe_set_attribute:
|
||||
LangfuseOtelLogger._set_langfuse_specific_attributes(
|
||||
MagicMock(), kwargs, None
|
||||
)
|
||||
|
||||
emitted = {
|
||||
call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list
|
||||
}
|
||||
|
||||
if expected_version is None:
|
||||
assert "langfuse.version" not in emitted
|
||||
else:
|
||||
assert emitted["langfuse.version"] == expected_version
|
||||
assert emitted["langfuse.release"] == "rel-9"
|
||||
for retired_key in (
|
||||
"langfuse.generation.version",
|
||||
"langfuse.trace.version",
|
||||
"langfuse.trace.release",
|
||||
):
|
||||
assert retired_key not in emitted
|
||||
|
||||
def test_set_langfuse_specific_attributes_with_content(self):
|
||||
"""Test that _set_langfuse_specific_attributes correctly sets observation.output with regular content response."""
|
||||
from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes
|
||||
|
|
|
|||
|
|
@ -55,17 +55,10 @@ class TestGoogleInteractionsCreate:
|
|||
print(f"Usage: {response.usage}")
|
||||
|
||||
def test_create_with_content_list(self, api_key):
|
||||
"""Test creating an interaction with a structured content list (Turn format)."""
|
||||
"""Test creating an interaction with a structured content list (Content[] input)."""
|
||||
response = interactions.create(
|
||||
model="gemini/gemini-2.5-flash",
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is the capital of France?"}
|
||||
],
|
||||
}
|
||||
],
|
||||
input=[{"type": "text", "text": "What is the capital of France?"}],
|
||||
api_key=api_key,
|
||||
)
|
||||
|
||||
|
|
@ -169,25 +162,25 @@ class TestGoogleInteractionsStreaming:
|
|||
|
||||
|
||||
class TestGoogleInteractionsMultiTurn:
|
||||
"""Tests for multi-turn conversations using Turn[] input."""
|
||||
"""Tests for multi-turn conversations using Step[] input."""
|
||||
|
||||
def test_multi_turn_conversation(self, api_key):
|
||||
"""Test a multi-turn conversation per OpenAPI spec (Turn[] format)."""
|
||||
"""Test a multi-turn conversation per OpenAPI spec (Step[] format)."""
|
||||
response = interactions.create(
|
||||
model="gemini/gemini-2.5-flash",
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"type": "user_input",
|
||||
"content": [{"type": "text", "text": "My name is Alice."}],
|
||||
},
|
||||
{
|
||||
"role": "model",
|
||||
"type": "model_output",
|
||||
"content": [
|
||||
{"type": "text", "text": "Hello Alice! Nice to meet you."}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"type": "user_input",
|
||||
"content": [{"type": "text", "text": "What is my name?"}],
|
||||
},
|
||||
],
|
||||
|
|
|
|||
|
|
@ -167,17 +167,39 @@ class TestRequestCompliance:
|
|||
assert text_schema["properties"]["type"].get("const") == "text"
|
||||
print("✓ TextContent schema is correct")
|
||||
|
||||
def test_turn_schema(self, spec_dict):
|
||||
"""Verify Turn schema for multi-turn conversations."""
|
||||
turn_schema = spec_dict["components"]["schemas"]["Turn"]
|
||||
def test_step_schema(self, spec_dict):
|
||||
"""Verify step-based multi-turn input.
|
||||
|
||||
assert "role" in turn_schema["properties"]
|
||||
assert "content" in turn_schema["properties"]
|
||||
Google replaced the role-carrying `Turn` schema with typed steps
|
||||
(spec update of Aug 13, 2026): conversation history is now a `Step[]`
|
||||
where `UserInputStep`/`ModelOutputStep` pin `type` values that our
|
||||
transformations read to recover the role. Assert exactly what our code
|
||||
depends on: `InteractionsInput` accepts a Step array, both step kinds
|
||||
are part of the `Step` union, each pins its `type` const, and each
|
||||
carries a `Content[]` content field.
|
||||
"""
|
||||
input_schema = spec_dict["components"]["schemas"]["InteractionsInput"]
|
||||
step_array_items = [
|
||||
option["items"]["$ref"].split("/")[-1]
|
||||
for option in input_schema["oneOf"]
|
||||
if option.get("type") == "array" and "$ref" in option.get("items", {})
|
||||
]
|
||||
assert "Step" in step_array_items, f"InteractionsInput should accept Step[], got arrays of {step_array_items}"
|
||||
|
||||
# Content can be string or Content[]
|
||||
content_prop = turn_schema["properties"]["content"]
|
||||
assert "oneOf" in content_prop
|
||||
print("✓ Turn schema supports role + content")
|
||||
step_variants = {
|
||||
option["$ref"].split("/")[-1]
|
||||
for option in spec_dict["components"]["schemas"]["Step"]["oneOf"]
|
||||
if "$ref" in option
|
||||
}
|
||||
assert {"UserInputStep", "ModelOutputStep"} <= step_variants, f"Step union is missing role steps: {step_variants}"
|
||||
|
||||
for step_name, type_value in [("UserInputStep", "user_input"), ("ModelOutputStep", "model_output")]:
|
||||
step_schema = spec_dict["components"]["schemas"][step_name]
|
||||
assert step_schema["properties"]["type"].get("const") == type_value
|
||||
assert "type" in step_schema["required"]
|
||||
content_items = step_schema["properties"]["content"]["items"]
|
||||
assert content_items["$ref"].split("/")[-1] == "Content"
|
||||
print(f"✓ {step_name} pins type '{type_value}' with Content[] content")
|
||||
|
||||
|
||||
class TestResponseCompliance:
|
||||
|
|
|
|||
|
|
@ -485,6 +485,70 @@ def test_generic_cost_per_token_minimax_m3_above_512k_tokens():
|
|||
assert round(completion_cost, 10) == round(expected_completion, 10)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"bedrock_mantle/openai.gpt-5.6-sol",
|
||||
"bedrock_mantle/openai.gpt-5.6-terra",
|
||||
"bedrock_mantle/openai.gpt-5.6-luna",
|
||||
],
|
||||
)
|
||||
def test_generic_cost_per_token_bedrock_mantle_gpt56_long_context(model):
|
||||
"""Bedrock GPT-5.6 supports a 1M context window, billed at the long-context rates above 272K."""
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
model_cost_map = litellm.model_cost[model]
|
||||
assert model_cost_map["max_input_tokens"] == 1000000
|
||||
|
||||
cached_tokens = 100000
|
||||
completion_tokens = 1000
|
||||
|
||||
short_prompt_tokens = 272000
|
||||
short_usage = Usage(
|
||||
prompt_tokens=short_prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=short_prompt_tokens + completion_tokens,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens),
|
||||
)
|
||||
short_prompt_cost, short_completion_cost = generic_cost_per_token(
|
||||
model=model,
|
||||
usage=short_usage,
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
)
|
||||
assert round(short_prompt_cost, 10) == round(
|
||||
model_cost_map["input_cost_per_token"] * (short_prompt_tokens - cached_tokens)
|
||||
+ model_cost_map["cache_read_input_token_cost"] * cached_tokens,
|
||||
10,
|
||||
)
|
||||
assert round(short_completion_cost, 10) == round(
|
||||
model_cost_map["output_cost_per_token"] * completion_tokens, 10
|
||||
)
|
||||
|
||||
long_prompt_tokens = 900000
|
||||
long_usage = Usage(
|
||||
prompt_tokens=long_prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=long_prompt_tokens + completion_tokens,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens),
|
||||
)
|
||||
long_prompt_cost, long_completion_cost = generic_cost_per_token(
|
||||
model=model,
|
||||
usage=long_usage,
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
)
|
||||
assert round(long_prompt_cost, 10) == round(
|
||||
model_cost_map["input_cost_per_token_above_272k_tokens"]
|
||||
* (long_prompt_tokens - cached_tokens)
|
||||
+ model_cost_map["cache_read_input_token_cost_above_272k_tokens"]
|
||||
* cached_tokens,
|
||||
10,
|
||||
)
|
||||
assert round(long_completion_cost, 10) == round(
|
||||
model_cost_map["output_cost_per_token_above_272k_tokens"] * completion_tokens, 10
|
||||
)
|
||||
|
||||
|
||||
def test_generic_cost_per_token_honors_non_standard_above_threshold():
|
||||
"""Regression for #30344: get_model_info must keep arbitrary
|
||||
input/output_cost_per_token_above_<N>_tokens thresholds, not only the hard-coded
|
||||
|
|
|
|||
|
|
@ -1526,7 +1526,11 @@ class TestBedrockMantleResponsesPricing:
|
|||
assert info["cache_creation_input_token_cost"] == pytest.approx(cache_creation_cost)
|
||||
assert info["cache_read_input_token_cost"] == pytest.approx(cache_read_cost)
|
||||
assert info["output_cost_per_token"] == pytest.approx(output_cost)
|
||||
assert info["max_input_tokens"] == 272000
|
||||
assert info["max_input_tokens"] == 1000000
|
||||
assert info["input_cost_per_token_above_272k_tokens"] == pytest.approx(input_cost * 2)
|
||||
assert info["cache_creation_input_token_cost_above_272k_tokens"] == pytest.approx(cache_creation_cost * 2)
|
||||
assert info["cache_read_input_token_cost_above_272k_tokens"] == pytest.approx(cache_read_cost * 2)
|
||||
assert info["output_cost_per_token_above_272k_tokens"] == pytest.approx(output_cost * 1.5)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, input_cost, output_cost",
|
||||
|
|
|
|||
|
|
@ -205,6 +205,50 @@ async def test_proxy_shutdown_event_prisma_disconnect_raises_error(monkeypatch):
|
|||
await proxy_shutdown_event()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _flush_spend_logs_queue_on_shutdown
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flush_spend_logs_queue_on_shutdown_drains_before_disconnect(monkeypatch):
|
||||
fake_prisma = MagicMock()
|
||||
monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False)
|
||||
monkeypatch.setattr(ps, "db_writer_client", None, raising=False)
|
||||
|
||||
drain = AsyncMock()
|
||||
import litellm.proxy.utils as utils_mod
|
||||
|
||||
monkeypatch.setattr(utils_mod, "drain_spend_logs_queue", drain)
|
||||
|
||||
await ps._flush_spend_logs_queue_on_shutdown()
|
||||
|
||||
observed = {
|
||||
"drain_calls": drain.await_count,
|
||||
"drain_prisma": drain.await_args.kwargs["prisma_client"] is fake_prisma,
|
||||
}
|
||||
assert observed == {
|
||||
"drain_calls": 1,
|
||||
"drain_prisma": True,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flush_spend_logs_queue_on_shutdown_swallows_drain_errors(monkeypatch):
|
||||
monkeypatch.setattr(ps, "prisma_client", MagicMock(), raising=False)
|
||||
monkeypatch.setattr(ps, "db_writer_client", None, raising=False)
|
||||
|
||||
import litellm.proxy.utils as utils_mod
|
||||
|
||||
monkeypatch.setattr(
|
||||
utils_mod,
|
||||
"drain_spend_logs_queue",
|
||||
AsyncMock(side_effect=RuntimeError("db gone")),
|
||||
)
|
||||
|
||||
await ps._flush_spend_logs_queue_on_shutdown()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _initialize_shared_aiohttp_session
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -128,6 +128,7 @@ def mock_prisma_client() -> MagicMock:
|
|||
client.proxy_logging_obj.failure_handler = AsyncMock()
|
||||
client.spend_log_transactions = []
|
||||
client._spend_log_transactions_lock = asyncio.Lock()
|
||||
client.spend_logs_queue_monitor_task = None
|
||||
client.tool_usage_transactions = []
|
||||
client._tool_usage_transactions_lock = asyncio.Lock()
|
||||
client.jsonify_object = lambda data: dict(data)
|
||||
|
|
|
|||
|
|
@ -17,8 +17,10 @@ from unittest.mock import AsyncMock, MagicMock
|
|||
import pytest
|
||||
|
||||
from litellm.proxy.utils import (
|
||||
MAX_SPEND_LOG_DRAIN_ITERATIONS,
|
||||
_monitor_spend_logs_queue,
|
||||
_raise_failed_update_spend_exception,
|
||||
drain_spend_logs_queue,
|
||||
update_daily_tag_spend,
|
||||
update_spend,
|
||||
update_spend_logs_job,
|
||||
|
|
@ -263,6 +265,198 @@ async def test_update_spend_logs_job_processes_and_clears_queue(
|
|||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_spend_logs_job_requeues_popped_rows_when_write_cancelled(
|
||||
mock_prisma_client: Any, make_spend_log_row: Any
|
||||
) -> None:
|
||||
proxy_logging = MagicMock()
|
||||
proxy_logging.failure_handler = AsyncMock()
|
||||
mock_prisma_client.spend_log_transactions = [
|
||||
make_spend_log_row(request_id="r1"),
|
||||
make_spend_log_row(request_id="r2"),
|
||||
]
|
||||
|
||||
row_arriving_mid_flush = make_spend_log_row(request_id="r3")
|
||||
|
||||
async def _cancel_mid_write(*args: Any, **kwargs: Any) -> None:
|
||||
mock_prisma_client.spend_log_transactions.append(row_arriving_mid_flush)
|
||||
raise asyncio.CancelledError()
|
||||
|
||||
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(
|
||||
side_effect=_cancel_mid_write
|
||||
)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await update_spend_logs_job(
|
||||
prisma_client=mock_prisma_client,
|
||||
db_writer_client=None,
|
||||
proxy_logging_obj=proxy_logging,
|
||||
)
|
||||
|
||||
assert [
|
||||
row["request_id"] for row in mock_prisma_client.spend_log_transactions
|
||||
] == ["r1", "r2", "r3"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_spend_logs_job_does_not_requeue_when_cancelled_after_write(
|
||||
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Rows are already committed once guardrail tracking runs, so replaying
|
||||
them would double-count the non-idempotent daily guardrail increments.
|
||||
"""
|
||||
import litellm.proxy.guardrails.usage_tracking as guard_mod
|
||||
|
||||
proxy_logging = MagicMock()
|
||||
proxy_logging.failure_handler = AsyncMock()
|
||||
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
|
||||
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock()
|
||||
|
||||
monkeypatch.setattr(
|
||||
guard_mod,
|
||||
"process_spend_logs_guardrail_usage",
|
||||
AsyncMock(side_effect=asyncio.CancelledError()),
|
||||
raising=False,
|
||||
)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await update_spend_logs_job(
|
||||
prisma_client=mock_prisma_client,
|
||||
db_writer_client=None,
|
||||
proxy_logging_obj=proxy_logging,
|
||||
)
|
||||
|
||||
assert mock_prisma_client.spend_log_transactions == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drain_spend_logs_queue_flushes_rows_queued_while_draining(
|
||||
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
import litellm.proxy.db.spend_log_tool_index as tool_mod
|
||||
import litellm.proxy.guardrails.usage_tracking as guard_mod
|
||||
|
||||
monkeypatch.setattr(
|
||||
guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False
|
||||
)
|
||||
|
||||
proxy_logging = MagicMock()
|
||||
proxy_logging.failure_handler = AsyncMock()
|
||||
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
|
||||
|
||||
written: list[str] = []
|
||||
|
||||
async def _write(*args: Any, **kwargs: Any) -> None:
|
||||
written.extend(row["request_id"] for row in kwargs["data"])
|
||||
if len(written) == 1:
|
||||
mock_prisma_client.spend_log_transactions.append(
|
||||
make_spend_log_row(request_id="r2")
|
||||
)
|
||||
|
||||
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write)
|
||||
|
||||
await drain_spend_logs_queue(
|
||||
prisma_client=mock_prisma_client,
|
||||
db_writer_client=None,
|
||||
proxy_logging_obj=proxy_logging,
|
||||
)
|
||||
|
||||
assert written == ["r1", "r2"]
|
||||
assert mock_prisma_client.spend_log_transactions == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drain_spend_logs_queue_stops_monitor_and_keeps_its_popped_rows(
|
||||
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
import litellm.proxy.db.spend_log_tool_index as tool_mod
|
||||
import litellm.proxy.guardrails.usage_tracking as guard_mod
|
||||
|
||||
monkeypatch.setattr(
|
||||
guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False
|
||||
)
|
||||
|
||||
proxy_logging = MagicMock()
|
||||
proxy_logging.failure_handler = AsyncMock()
|
||||
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
|
||||
|
||||
write_started = asyncio.Event()
|
||||
written: list[str] = []
|
||||
write_calls = {"n": 0}
|
||||
|
||||
async def _write(*args: Any, **kwargs: Any) -> None:
|
||||
write_calls["n"] += 1
|
||||
if write_calls["n"] == 1:
|
||||
write_started.set()
|
||||
await asyncio.Event().wait()
|
||||
written.extend(row["request_id"] for row in kwargs["data"])
|
||||
|
||||
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write)
|
||||
|
||||
async def _monitor() -> None:
|
||||
await update_spend_logs_job(
|
||||
prisma_client=mock_prisma_client,
|
||||
db_writer_client=None,
|
||||
proxy_logging_obj=proxy_logging,
|
||||
)
|
||||
|
||||
mock_prisma_client.spend_logs_queue_monitor_task = asyncio.create_task(_monitor())
|
||||
await write_started.wait()
|
||||
|
||||
await drain_spend_logs_queue(
|
||||
prisma_client=mock_prisma_client,
|
||||
db_writer_client=None,
|
||||
proxy_logging_obj=proxy_logging,
|
||||
)
|
||||
|
||||
assert written == ["r1"]
|
||||
assert mock_prisma_client.spend_log_transactions == []
|
||||
assert mock_prisma_client.spend_logs_queue_monitor_task is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drain_spend_logs_queue_gives_up_after_max_passes(
|
||||
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
import litellm.proxy.db.spend_log_tool_index as tool_mod
|
||||
import litellm.proxy.guardrails.usage_tracking as guard_mod
|
||||
|
||||
monkeypatch.setattr(
|
||||
guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False
|
||||
)
|
||||
|
||||
proxy_logging = MagicMock()
|
||||
proxy_logging.failure_handler = AsyncMock()
|
||||
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
|
||||
|
||||
async def _write_and_refill(*args: Any, **kwargs: Any) -> None:
|
||||
mock_prisma_client.spend_log_transactions.append(make_spend_log_row())
|
||||
|
||||
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(
|
||||
side_effect=_write_and_refill
|
||||
)
|
||||
|
||||
await drain_spend_logs_queue(
|
||||
prisma_client=mock_prisma_client,
|
||||
db_writer_client=None,
|
||||
proxy_logging_obj=proxy_logging,
|
||||
)
|
||||
|
||||
assert (
|
||||
mock_prisma_client.db.litellm_spendlogs.create_many.await_count
|
||||
== MAX_SPEND_LOG_DRAIN_ITERATIONS
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_monitor_spend_logs_queue_invokes_job_when_queue_nonempty(
|
||||
mock_prisma_client: Any,
|
||||
|
|
|
|||
|
|
@ -1176,6 +1176,41 @@ class TestPreRoutingStrategyRegistry:
|
|||
}
|
||||
assert router._select_pre_routing_strategy("smart", {}).strategy is cn
|
||||
|
||||
@staticmethod
|
||||
def _router_with_plain_smart_deployment(enable_tag_filtering: bool) -> Router:
|
||||
return Router(
|
||||
model_list=[{"model_name": "smart", "litellm_params": {"model": "openai/gpt-4o-mini"}}],
|
||||
enable_tag_filtering=enable_tag_filtering,
|
||||
)
|
||||
|
||||
def test_select_falls_through_to_plain_deployments_when_no_tag_matches_under_tag_filtering(self):
|
||||
router = self._router_with_plain_smart_deployment(enable_tag_filtering=True)
|
||||
cn, us = object(), object()
|
||||
|
||||
router.complexity_routers = {"smart": [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]}
|
||||
assert router._select_pre_routing_strategy("smart", {}) is None
|
||||
assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["cn"]}}).strategy is cn
|
||||
|
||||
router.complexity_routers = {
|
||||
"smart": [
|
||||
TaggedPreRoutingStrategy(tags=("cn",), strategy=cn),
|
||||
TaggedPreRoutingStrategy(tags=("us",), strategy=us),
|
||||
]
|
||||
}
|
||||
assert router._select_pre_routing_strategy("smart", {}) is None
|
||||
assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["row"]}}) is None
|
||||
assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["us"]}}).strategy is us
|
||||
|
||||
router.complexity_routers["router-only"] = [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]
|
||||
assert router._select_pre_routing_strategy("router-only", {}).strategy is cn
|
||||
|
||||
def test_select_keeps_capturing_when_tag_filtering_is_disabled(self):
|
||||
router = self._router_with_plain_smart_deployment(enable_tag_filtering=False)
|
||||
cn = object()
|
||||
|
||||
router.complexity_routers = {"smart": [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]}
|
||||
assert router._select_pre_routing_strategy("smart", {}).strategy is cn
|
||||
|
||||
|
||||
class TestAsyncPreRoutingHookMultiFormat:
|
||||
"""Test async_pre_routing_hook with multiple input formats."""
|
||||
|
|
@ -2079,14 +2114,16 @@ class TestRouterPreRoutingAliasOverrides:
|
|||
assert field not in request_kwargs
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alias_overrides_exclude_only_model(self):
|
||||
"""`model` (the alias marker, e.g. auto_router/complexity_router) is
|
||||
excluded since it's never a real provider model. Router-only fields
|
||||
like complexity_router_config DO flow through into request_kwargs at
|
||||
this layer - they're filtered from the actual outbound LLM call
|
||||
downstream by litellm.types.utils.all_litellm_params instead, not by
|
||||
the router's pre-routing hook. See test_router_init_only_params_are_
|
||||
never_sent_to_a_provider for the guard on that downstream filter."""
|
||||
async def test_alias_overrides_exclude_only_marker_and_connection_params(self):
|
||||
"""`model` (the alias marker, e.g. auto_router/complexity_router) and
|
||||
provider-connection params (api_base/api_key/api_version) are excluded
|
||||
since they never describe the tier deployment actually called.
|
||||
Router-only fields like complexity_router_config DO flow through into
|
||||
request_kwargs at this layer - they're filtered from the actual
|
||||
outbound LLM call downstream by litellm.types.utils.all_litellm_params
|
||||
instead, not by the router's pre-routing hook. See
|
||||
test_router_init_only_params_are_never_sent_to_a_provider for the
|
||||
guard on that downstream filter."""
|
||||
router = self._make_router()
|
||||
request_kwargs: Dict = {}
|
||||
|
||||
|
|
@ -2106,9 +2143,10 @@ class TestRouterPreRoutingAliasOverrides:
|
|||
assert request_kwargs["complexity_router_default_model"] == "gpt-4o"
|
||||
|
||||
def test_router_init_only_params_are_never_sent_to_a_provider(self):
|
||||
"""The router's pre-routing hook only excludes `model` (see
|
||||
test_alias_overrides_exclude_only_model above) - every other alias
|
||||
litellm_param, including router-init-only fields like
|
||||
"""The router's pre-routing hook only excludes `model` and
|
||||
provider-connection params (see test_alias_overrides_exclude_only_
|
||||
marker_and_connection_params above) - every other alias litellm_param,
|
||||
including router-init-only fields like
|
||||
complexity_router_config, flows into request_kwargs unfiltered. That's
|
||||
only safe because litellm.completion()/acompletion() itself strips
|
||||
anything listed in all_litellm_params before building the provider
|
||||
|
|
@ -2201,6 +2239,154 @@ class TestRouterPreRoutingAliasOverrides:
|
|||
assert request_kwargs["drop_params"] is True
|
||||
|
||||
|
||||
class TestRouterPreRoutingSharedAliasName:
|
||||
"""
|
||||
Regression tests for https://github.com/BerriAI/litellm/issues/36619.
|
||||
|
||||
A plain deployment and an `auto_router/` marker can share a `model_name`.
|
||||
The alias-param forwarding after a pre-routing rewrite must read the
|
||||
marker entry, never whichever same-name entry happens to sit first in
|
||||
`model_list` - otherwise the plain entry's api_base/api_key get grafted
|
||||
onto the routed tier's call (a Gemini path under api.openai.com, 404).
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _plain_entry() -> dict:
|
||||
return {
|
||||
"model_name": "gpt4o",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o",
|
||||
"api_key": "sk-plain-entry",
|
||||
"api_base": "https://plain-entry.example/v1",
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _marker_entry() -> dict:
|
||||
return {
|
||||
"model_name": "gpt4o",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"drop_params": True,
|
||||
"complexity_router_config": {"tiers": {"SIMPLE": "gemini-flash", "MEDIUM": "gemini-flash"}},
|
||||
"complexity_router_default_model": "gemini-flash",
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _tier_entry() -> dict:
|
||||
return {
|
||||
"model_name": "gemini-flash",
|
||||
"litellm_params": {"model": "gemini/gemini-3.6-flash", "api_key": "sk-tier"},
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("plain_entry_first", [True, False], ids=["plain_entry_first", "marker_entry_first"])
|
||||
async def test_marker_params_forwarded_regardless_of_model_list_order(self, plain_entry_first):
|
||||
"""In either config order the routed call gets the marker's own params
|
||||
(drop_params) and never the plain sibling's api_base/api_key."""
|
||||
shared_name_entries = (
|
||||
[self._plain_entry(), self._marker_entry()]
|
||||
if plain_entry_first
|
||||
else [self._marker_entry(), self._plain_entry()]
|
||||
)
|
||||
router = Router(model_list=[*shared_name_entries, self._tier_entry()])
|
||||
request_kwargs: Dict = {}
|
||||
|
||||
result = await router.async_pre_routing_hook(
|
||||
model="gpt4o",
|
||||
request_kwargs=request_kwargs,
|
||||
messages=[{"role": "user", "content": "What is the capital of France?"}],
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.model == "gemini-flash"
|
||||
assert "api_base" not in request_kwargs
|
||||
assert "api_key" not in request_kwargs
|
||||
assert request_kwargs["drop_params"] is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connection_params_on_the_marker_itself_are_not_forwarded(self):
|
||||
"""Even when the marker entry carries api_base/api_key/api_version,
|
||||
they describe no real deployment and must not reach the routed call,
|
||||
while the marker's other params still do."""
|
||||
marker_with_connection_params = {
|
||||
"model_name": "smart",
|
||||
"litellm_params": {
|
||||
**self._marker_entry()["litellm_params"],
|
||||
"api_key": "sk-marker",
|
||||
"api_base": "https://marker.example/v1",
|
||||
"api_version": "2024-01-01",
|
||||
},
|
||||
}
|
||||
router = Router(model_list=[marker_with_connection_params, self._tier_entry()])
|
||||
request_kwargs: Dict = {}
|
||||
|
||||
result = await router.async_pre_routing_hook(
|
||||
model="smart",
|
||||
request_kwargs=request_kwargs,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert "api_base" not in request_kwargs
|
||||
assert "api_key" not in request_kwargs
|
||||
assert "api_version" not in request_kwargs
|
||||
assert request_kwargs["drop_params"] is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_scoped_markers_forward_the_selected_markers_params(self):
|
||||
"""With two tag-scoped markers under one name, the forwarded params
|
||||
come from the marker whose tags matched the request, not from the
|
||||
first marker in the list."""
|
||||
|
||||
def tagged_marker(routed_model: str, tags: list, drop_params: bool | None) -> dict:
|
||||
return {
|
||||
"model_name": "smart",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_default_model": routed_model,
|
||||
"complexity_router_config": {"tiers": {"SIMPLE": [routed_model], "MEDIUM": [routed_model]}},
|
||||
"tags": tags,
|
||||
**({"drop_params": drop_params} if drop_params is not None else {}),
|
||||
},
|
||||
}
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
tagged_marker("gpt-cn", ["cn"], None),
|
||||
tagged_marker("gpt-us", ["us"], True),
|
||||
]
|
||||
)
|
||||
|
||||
us_kwargs: Dict = {"metadata": {"tags": ["us"]}}
|
||||
us_result = await router.async_pre_routing_hook(
|
||||
model="smart",
|
||||
request_kwargs=us_kwargs,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
assert us_result is not None and us_result.model == "gpt-us"
|
||||
assert us_kwargs["drop_params"] is True
|
||||
|
||||
cn_kwargs: Dict = {"metadata": {"tags": ["cn"]}}
|
||||
cn_result = await router.async_pre_routing_hook(
|
||||
model="smart",
|
||||
request_kwargs=cn_kwargs,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
assert cn_result is not None and cn_result.model == "gpt-cn"
|
||||
assert "drop_params" not in cn_kwargs
|
||||
|
||||
def test_forwardable_alias_marker_params_reads_the_marker_entry_only(self):
|
||||
router = Router(model_list=[self._plain_entry(), self._marker_entry(), self._tier_entry()])
|
||||
|
||||
forwarded = dict(router._forwardable_alias_marker_params(model="gpt4o", strategy_tags=()))
|
||||
|
||||
assert forwarded["drop_params"] is True
|
||||
assert "api_key" not in forwarded and "api_base" not in forwarded
|
||||
assert router._forwardable_alias_marker_params(model="gemini-flash", strategy_tags=()) == ()
|
||||
|
||||
|
||||
class TestAdaptiveSoftFloors:
|
||||
def test_adaptive_defaults_use_cost_weighted_cold_policy(self):
|
||||
config = ComplexityRouterConfig(
|
||||
|
|
|
|||
|
|
@ -423,6 +423,33 @@ def test_get_tags_from_request_kwargs_various_inputs():
|
|||
assert _get_tags_from_request_kwargs({"foo": "bar"}) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_kwargs",
|
||||
[
|
||||
{"metadata": "not-a-dict"},
|
||||
{"litellm_metadata": "not-a-dict"},
|
||||
{"litellm_metadata": ["not", "a", "dict"]},
|
||||
{"litellm_params": "not-a-dict"},
|
||||
{"litellm_params": {"metadata": "not-a-dict"}},
|
||||
{"metadata": {"tags": "free"}},
|
||||
{"metadata": {"tags": {"free": "paid"}}},
|
||||
],
|
||||
)
|
||||
def test_get_tags_from_request_kwargs_reads_no_tags_from_a_non_dict_shape(request_kwargs):
|
||||
"""Metadata and `tags` are request-controlled, so a client can send either as a
|
||||
string, a list or null. Every shape that cannot hold string tags reads as untagged
|
||||
instead of raising, because callers run on the hot request path."""
|
||||
from litellm.router_strategy.tag_based_routing import _get_tags_from_request_kwargs
|
||||
|
||||
assert _get_tags_from_request_kwargs(request_kwargs) == []
|
||||
|
||||
|
||||
def test_get_tags_from_request_kwargs_keeps_only_string_tags():
|
||||
from litellm.router_strategy.tag_based_routing import _get_tags_from_request_kwargs
|
||||
|
||||
assert _get_tags_from_request_kwargs({"metadata": {"tags": ["free", 7, None, "paid"]}}) == ["free", "paid"]
|
||||
|
||||
|
||||
# --- _split_tags unit tests ---
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7617,6 +7617,104 @@ class TestAutoRouterMaxInputCharsWiring:
|
|||
assert self._registered_auto_router(router).max_input_chars == DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS
|
||||
|
||||
|
||||
class TestTaggedAutoRouterOnSharedModelName:
|
||||
"""A tagged auto-router marker sharing its model_name with a plain deployment must not
|
||||
capture requests whose tags don't match it when tag filtering is enabled (#36620)."""
|
||||
|
||||
class _FixedRouteLayer:
|
||||
def __call__(self, text: str):
|
||||
from semantic_router.schema import RouteChoice
|
||||
|
||||
return RouteChoice(name="gemini-flash")
|
||||
|
||||
@classmethod
|
||||
def _router(cls, marker_tags, include_plain_sibling: bool, enable_tag_filtering: bool) -> "litellm.Router":
|
||||
pytest.importorskip("semantic_router", reason="auto-router needs the semantic-router extra")
|
||||
marker = {
|
||||
"model_name": "gpt4o",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/gpt4o-router",
|
||||
"auto_router_config": json.dumps(
|
||||
{"routes": [{"name": "gemini-flash", "utterances": ["capital city questions"]}]}
|
||||
),
|
||||
"auto_router_default_model": "gemini-flash",
|
||||
"auto_router_embedding_model": "text-embedding-3-small",
|
||||
**({"tags": marker_tags} if marker_tags else {}),
|
||||
},
|
||||
}
|
||||
plain = {"model_name": "gpt4o", "litellm_params": {"model": "openai/gpt-4o"}}
|
||||
tier = {"model_name": "gemini-flash", "litellm_params": {"model": "gemini/gemini-3.6-flash"}}
|
||||
router = litellm.Router(
|
||||
model_list=[plain, marker, tier] if include_plain_sibling else [marker, tier],
|
||||
enable_tag_filtering=enable_tag_filtering,
|
||||
)
|
||||
router.auto_routers["gpt4o"][0].strategy.routelayer = cls._FixedRouteLayer()
|
||||
return router
|
||||
|
||||
@staticmethod
|
||||
async def _hook_response(router: "litellm.Router", request_kwargs: dict):
|
||||
return await router.async_pre_routing_hook(
|
||||
model="gpt4o",
|
||||
request_kwargs=request_kwargs,
|
||||
messages=[{"role": "user", "content": "What is the capital of France?"}],
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_untagged_request_bypasses_the_tagged_marker_when_a_plain_deployment_shares_the_name(self):
|
||||
router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True)
|
||||
|
||||
assert await self._hook_response(router, {}) is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_tagged_for_the_marker_is_still_semantically_routed(self):
|
||||
router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True)
|
||||
|
||||
response = await self._hook_response(router, {"metadata": {"tags": ["route"]}})
|
||||
|
||||
assert response is not None
|
||||
assert response.model == "gemini-flash"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_marker_only_alias_still_captures_untagged_requests(self):
|
||||
router = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True)
|
||||
|
||||
response = await self._hook_response(router, {})
|
||||
|
||||
assert response is not None
|
||||
assert response.model == "gemini-flash"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_untagged_marker_sharing_the_name_still_captures_untagged_requests(self):
|
||||
router = self._router(marker_tags=None, include_plain_sibling=True, enable_tag_filtering=True)
|
||||
|
||||
response = await self._hook_response(router, {})
|
||||
|
||||
assert response is not None
|
||||
assert response.model == "gemini-flash"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_untagged_selection_never_lands_on_the_marker_deployment(self):
|
||||
router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True)
|
||||
|
||||
for _ in range(20):
|
||||
deployment = await router.async_get_available_deployment(
|
||||
model="gpt4o",
|
||||
request_kwargs={},
|
||||
messages=[{"role": "user", "content": "What is the capital of France?"}],
|
||||
)
|
||||
assert deployment["litellm_params"]["model"] == "openai/gpt-4o"
|
||||
|
||||
def test_deployment_without_litellm_params_mapping_is_not_a_marker(self):
|
||||
assert litellm.Router._is_strategy_marker_deployment({"model_name": "gpt4o"}) is False
|
||||
|
||||
def test_model_name_has_plain_deployments_reflects_the_pool(self):
|
||||
mixed = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True)
|
||||
marker_only = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True)
|
||||
|
||||
assert mixed._model_name_has_plain_deployments("gpt4o") is True
|
||||
assert marker_only._model_name_has_plain_deployments("gpt4o") is False
|
||||
|
||||
|
||||
class TestGetAllowedFailsFromPolicy:
|
||||
def _make_router(self, **policy_kwargs) -> litellm.Router:
|
||||
from litellm.types.router import AllowedFailsPolicy
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
import { render, screen } from "@testing-library/react";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import RouterSettingsSummary from "./RouterSettingsSummary";
|
||||
|
||||
describe("RouterSettingsSummary", () => {
|
||||
it("should list each configured fallback mapping", () => {
|
||||
render(
|
||||
<RouterSettingsSummary
|
||||
routerSettings={{
|
||||
fallbacks: [{ "gpt-4": ["gpt-4o", "claude-sonnet"] }, { "gpt-4o": ["gpt-4o-mini"] }],
|
||||
num_retries: 3,
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(screen.getByText("gpt-4")).toBeInTheDocument();
|
||||
expect(screen.getByText("gpt-4o, claude-sonnet")).toBeInTheDocument();
|
||||
expect(screen.getByText("gpt-4o")).toBeInTheDocument();
|
||||
expect(screen.getByText("gpt-4o-mini")).toBeInTheDocument();
|
||||
expect(screen.getByText("Number of Retries: 3")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should show the empty state when every setting is null", () => {
|
||||
render(<RouterSettingsSummary routerSettings={{ fallbacks: null, num_retries: null }} />);
|
||||
|
||||
expect(screen.getByText("No router settings configured")).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
import { Badge } from "@/components/ui/badge";
|
||||
import { hasRouterSettings } from "./routerSettingsPayload";
|
||||
|
||||
interface RouterSettingsSummaryProps {
|
||||
routerSettings: Record<string, unknown> | null | undefined;
|
||||
emptyText?: string;
|
||||
}
|
||||
|
||||
const fallbackEntries = (fallbacks: unknown): Array<[string, string[]]> => {
|
||||
if (!Array.isArray(fallbacks)) return [];
|
||||
return fallbacks.flatMap((entry) =>
|
||||
entry && typeof entry === "object" ? (Object.entries(entry) as Array<[string, string[]]>) : [],
|
||||
);
|
||||
};
|
||||
|
||||
export default function RouterSettingsSummary({
|
||||
routerSettings,
|
||||
emptyText = "No router settings configured",
|
||||
}: RouterSettingsSummaryProps) {
|
||||
if (!hasRouterSettings(routerSettings)) {
|
||||
return <div className="text-gray-400">{emptyText}</div>;
|
||||
}
|
||||
|
||||
const settings = routerSettings as Record<string, unknown>;
|
||||
const fallbacks = fallbackEntries(settings.fallbacks);
|
||||
|
||||
return (
|
||||
<div className="space-y-1 text-sm">
|
||||
{settings.routing_strategy != null && (
|
||||
<div>
|
||||
Routing Strategy: <Badge variant="secondary">{String(settings.routing_strategy)}</Badge>
|
||||
</div>
|
||||
)}
|
||||
{settings.num_retries != null && <div>Number of Retries: {String(settings.num_retries)}</div>}
|
||||
{settings.allowed_fails != null && <div>Allowed Failures: {String(settings.allowed_fails)}</div>}
|
||||
{settings.cooldown_time != null && <div>Cooldown Time: {String(settings.cooldown_time)}s</div>}
|
||||
{settings.timeout != null && <div>Timeout: {String(settings.timeout)}s</div>}
|
||||
{settings.retry_after != null && <div>Retry After: {String(settings.retry_after)}s</div>}
|
||||
{Boolean(settings.enable_tag_filtering) && <div>Tag Filtering: Enabled</div>}
|
||||
{fallbacks.length > 0 && (
|
||||
<div>
|
||||
<div>Fallbacks:</div>
|
||||
<div className="mt-1 space-y-1">
|
||||
{fallbacks.map(([model, targets]) => (
|
||||
<div key={model} className="text-xs text-gray-600">
|
||||
<span className="font-medium">{model}</span>
|
||||
<span className="mx-1 text-gray-400">-></span>
|
||||
{Array.isArray(targets) ? targets.join(", ") : String(targets)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { hasRouterSettings, routerSettingsEditorValue, routerSettingsUpdate } from "./routerSettingsPayload";
|
||||
|
||||
describe("hasRouterSettings", () => {
|
||||
it("should treat unset, empty and all-null settings as absent", () => {
|
||||
expect(hasRouterSettings(undefined)).toBe(false);
|
||||
expect(hasRouterSettings(null)).toBe(false);
|
||||
expect(hasRouterSettings({})).toBe(false);
|
||||
expect(
|
||||
hasRouterSettings({ num_retries: null, fallbacks: [], model_group_alias: {}, enable_tag_filtering: false }),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("should detect configured settings", () => {
|
||||
expect(hasRouterSettings({ num_retries: 3 })).toBe(true);
|
||||
expect(hasRouterSettings({ fallbacks: [{ "gpt-4": ["gpt-4o"] }] })).toBe(true);
|
||||
expect(hasRouterSettings({ enable_tag_filtering: true })).toBe(true);
|
||||
expect(hasRouterSettings({ num_retries: 0 })).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("routerSettingsEditorValue", () => {
|
||||
it("should hand the editor only the fields it renders", () => {
|
||||
expect(
|
||||
routerSettingsEditorValue({
|
||||
num_retries: 2,
|
||||
tag_routing_prefix: "team-",
|
||||
fallbacks: [{ "gpt-4": ["gpt-4o"] }],
|
||||
}),
|
||||
).toStrictEqual({ router_settings: { num_retries: 2, fallbacks: [{ "gpt-4": ["gpt-4o"] }] } });
|
||||
});
|
||||
|
||||
it("should keep an explicitly stored null so the editor renders it as empty", () => {
|
||||
expect(routerSettingsEditorValue({ num_retries: null })).toStrictEqual({ router_settings: { num_retries: null } });
|
||||
});
|
||||
|
||||
it("should leave the editor uninitialised when the key has no stored settings", () => {
|
||||
expect(routerSettingsEditorValue(null)).toBeUndefined();
|
||||
expect(routerSettingsEditorValue(undefined)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("routerSettingsUpdate", () => {
|
||||
const fallbacks = [{ "gpt-4": ["gpt-4o"] }];
|
||||
// Accepted by UpdateRouterConfig on /key/update but not rendered by the accordion.
|
||||
const unsupported = {
|
||||
tag_routing_prefix: "team-",
|
||||
model_group_retry_policy: { "gpt-4": { TimeoutErrorRetries: 2 } },
|
||||
};
|
||||
|
||||
// The editor is a fixed-field form, so the merge has to hold two opposing guarantees at once:
|
||||
// routing fields it cannot render survive, and a field it does render that the admin emptied
|
||||
// is still sent as null. Orderings vary because an object spread resolves collisions by position.
|
||||
const storedOrderings: Array<[string, Record<string, unknown>]> = [
|
||||
["unsupported fields first", { ...unsupported, num_retries: 2, fallbacks }],
|
||||
["unsupported fields last", { num_retries: 2, fallbacks, ...unsupported }],
|
||||
[
|
||||
"unsupported fields interleaved",
|
||||
{
|
||||
tag_routing_prefix: unsupported.tag_routing_prefix,
|
||||
num_retries: 2,
|
||||
model_group_retry_policy: unsupported.model_group_retry_policy,
|
||||
fallbacks,
|
||||
},
|
||||
],
|
||||
];
|
||||
|
||||
it.each(storedOrderings)(
|
||||
"should keep unsupported stored fields and still clear an emptied editor field (%s)",
|
||||
(_ordering, stored) => {
|
||||
const result = routerSettingsUpdate({ num_retries: 4, fallbacks: null }, stored);
|
||||
|
||||
expect(result).toMatchObject({ ...unsupported, num_retries: 4, fallbacks: null });
|
||||
},
|
||||
);
|
||||
|
||||
it("should send an empty object, not a null blob, when the last stored setting is cleared", () => {
|
||||
expect(routerSettingsUpdate({ fallbacks: null, num_retries: null }, { fallbacks, num_retries: 2 })).toEqual({});
|
||||
});
|
||||
|
||||
it("should keep clearing owned fields explicitly while an unsupported setting still stands", () => {
|
||||
expect(routerSettingsUpdate({ fallbacks: null, num_retries: null }, { fallbacks, ...unsupported })).toMatchObject({
|
||||
...unsupported,
|
||||
fallbacks: null,
|
||||
num_retries: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("should null every editor-owned field the editor left out", () => {
|
||||
expect(routerSettingsUpdate({ fallbacks: null }, { timeout: 30, ...unsupported })).toEqual({
|
||||
...unsupported,
|
||||
routing_strategy: null,
|
||||
allowed_fails: null,
|
||||
cooldown_time: null,
|
||||
num_retries: null,
|
||||
timeout: null,
|
||||
retry_after: null,
|
||||
fallbacks: null,
|
||||
context_window_fallbacks: null,
|
||||
retry_policy: null,
|
||||
model_group_alias: null,
|
||||
enable_tag_filtering: null,
|
||||
routing_strategy_args: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("should send the edited settings when the user configured something", () => {
|
||||
expect(routerSettingsUpdate({ fallbacks }, null)).toMatchObject({ fallbacks });
|
||||
});
|
||||
|
||||
it("should leave the field off when nothing is stored and nothing was configured", () => {
|
||||
expect(routerSettingsUpdate({ fallbacks: null, num_retries: null }, {})).toBeUndefined();
|
||||
expect(routerSettingsUpdate(undefined, { fallbacks })).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
import { RouterSettingsAccordionValue } from "./RouterSettingsAccordion";
|
||||
|
||||
export type RouterSettings = RouterSettingsAccordionValue["router_settings"];
|
||||
|
||||
const EDITOR_OWNED_FIELDS: Record<keyof RouterSettings, true> = {
|
||||
routing_strategy: true,
|
||||
allowed_fails: true,
|
||||
cooldown_time: true,
|
||||
num_retries: true,
|
||||
timeout: true,
|
||||
retry_after: true,
|
||||
fallbacks: true,
|
||||
context_window_fallbacks: true,
|
||||
retry_policy: true,
|
||||
model_group_alias: true,
|
||||
enable_tag_filtering: true,
|
||||
routing_strategy_args: true,
|
||||
};
|
||||
|
||||
const EDITOR_OWNED_KEYS = Object.keys(EDITOR_OWNED_FIELDS) as Array<keyof RouterSettings>;
|
||||
|
||||
const isMeaningfulRouterSetting = (value: unknown): boolean => {
|
||||
if (value === null || value === undefined || value === "" || value === false) return false;
|
||||
if (Array.isArray(value)) return value.length > 0;
|
||||
if (typeof value === "object") return Object.keys(value).length > 0;
|
||||
return true;
|
||||
};
|
||||
|
||||
export const hasRouterSettings = (settings: Record<string, unknown> | null | undefined): boolean =>
|
||||
settings != null && Object.values(settings).some(isMeaningfulRouterSetting);
|
||||
|
||||
/**
|
||||
* The stored settings narrowed to what the editor renders. The stored blob is untyped JSON
|
||||
* from /key/info, so this projection is the one place it is read as RouterSettings.
|
||||
*/
|
||||
export const routerSettingsEditorValue = (
|
||||
stored: Record<string, unknown> | null | undefined,
|
||||
): RouterSettingsAccordionValue | undefined =>
|
||||
stored
|
||||
? {
|
||||
router_settings: Object.fromEntries(
|
||||
EDITOR_OWNED_KEYS.filter((key) => key in stored).map((key) => [key, stored[key]]),
|
||||
) as RouterSettings,
|
||||
}
|
||||
: undefined;
|
||||
|
||||
/**
|
||||
* Router settings to put on a /key/update payload, or undefined to leave the field off.
|
||||
* The editor only renders EDITOR_OWNED_FIELDS, so its value is merged over the stored object
|
||||
* rather than replacing it, and routing fields the editor cannot show survive an unrelated edit.
|
||||
* Emptying every field sends {}, which the proxy reads as "no key-level override" so the key
|
||||
* falls back to its team and global settings, where an all-null blob would pin it to nulls.
|
||||
*/
|
||||
export const routerSettingsUpdate = (
|
||||
edited: RouterSettings | null | undefined,
|
||||
stored: Record<string, unknown> | null | undefined,
|
||||
): Record<string, unknown> | undefined => {
|
||||
if (!edited) return undefined;
|
||||
const editorOwned = Object.fromEntries(EDITOR_OWNED_KEYS.map((key) => [key, edited[key] ?? null]));
|
||||
const merged: Record<string, unknown> = { ...stored, ...editorOwned };
|
||||
if (hasRouterSettings(merged)) return merged;
|
||||
return hasRouterSettings(stored) ? {} : undefined;
|
||||
};
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { render, screen, waitFor } from "@testing-library/react";
|
||||
import type { ReactElement, ReactNode } from "react";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { FallbackGroup } from "../Settings/RouterSettings/Fallbacks/FallbackGroupConfig";
|
||||
import type { RouterSettingsFormValue } from "../router_settings/RouterSettingsForm";
|
||||
import RouterSettingsAccordion from "./RouterSettingsAccordion";
|
||||
import { routerSettingsEditorValue } from "./routerSettingsPayload";
|
||||
|
||||
vi.mock("../networking", () => ({
|
||||
getRouterSettingsCall: vi.fn().mockResolvedValue({}),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/llm_calls/fetch_models", () => ({
|
||||
fetchAvailableModels: vi.fn().mockResolvedValue([{ model_group: "gpt-5.5" }, { model_group: "gpt-4o-mini" }]),
|
||||
fetchAvailableModelsForTeam: vi.fn().mockResolvedValue([]),
|
||||
}));
|
||||
|
||||
vi.mock("@tremor/react", () => ({
|
||||
TabGroup: ({ children }: { children: ReactNode }) => <div>{children}</div>,
|
||||
TabList: ({ children }: { children: ReactNode }) => <div>{children}</div>,
|
||||
Tab: ({ children }: { children: ReactNode }) => <div>{children}</div>,
|
||||
TabPanels: ({ children }: { children: ReactNode }) => <div>{children}</div>,
|
||||
TabPanel: ({ children }: { children: ReactNode }) => <div>{children}</div>,
|
||||
}));
|
||||
|
||||
vi.mock("../router_settings/RouterSettingsForm", () => ({
|
||||
default: ({ value }: { value: RouterSettingsFormValue }) => (
|
||||
<div data-testid="loadbalancing">{JSON.stringify(value.routerSettings)}</div>
|
||||
),
|
||||
}));
|
||||
|
||||
vi.mock("../Settings/RouterSettings/Fallbacks/FallbackSelectionForm", () => ({
|
||||
FallbackSelectionForm: ({ groups }: { groups: FallbackGroup[] }) => (
|
||||
<div data-testid="fallbacks">
|
||||
{groups.map((g) => `${g.primaryModel ?? "none"}->${g.fallbackModels.join("|") || "none"}`).join(" ")}
|
||||
</div>
|
||||
),
|
||||
}));
|
||||
|
||||
// Captured verbatim from GET /key/info on a live proxy for a key created through the UI.
|
||||
// tag_routing_prefix is stored on the key and accepted by /key/update, but the accordion
|
||||
// has no control for it, so the projection must drop it without losing the rest.
|
||||
const KEY_INFO_ROUTER_SETTINGS: Record<string, unknown> = {
|
||||
fallbacks: [{ "gpt-5.5": ["gpt-4o-mini"] }],
|
||||
num_retries: 3,
|
||||
tag_routing_prefix: "team-",
|
||||
};
|
||||
|
||||
const renderAccordion = (stored: Record<string, unknown> | null): ReactElement => (
|
||||
<QueryClientProvider client={new QueryClient({ defaultOptions: { queries: { retry: false } } })}>
|
||||
<RouterSettingsAccordion accessToken="test-token" value={routerSettingsEditorValue(stored)} />
|
||||
</QueryClientProvider>
|
||||
);
|
||||
|
||||
describe("key router settings wiring, /key/info payload through to rendered output", () => {
|
||||
it("should prefill both tabs from the stored settings", async () => {
|
||||
render(renderAccordion(KEY_INFO_ROUTER_SETTINGS));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByTestId("fallbacks")).toHaveTextContent("gpt-5.5->gpt-4o-mini");
|
||||
});
|
||||
expect(JSON.parse(screen.getByTestId("loadbalancing").textContent ?? "{}")).toMatchObject({ num_retries: 3 });
|
||||
});
|
||||
|
||||
it("should not leak a field the accordion has no control for into the editor", async () => {
|
||||
render(renderAccordion(KEY_INFO_ROUTER_SETTINGS));
|
||||
|
||||
await waitFor(() => expect(screen.getByTestId("loadbalancing")).toBeInTheDocument());
|
||||
expect(screen.getByTestId("loadbalancing").textContent).not.toContain("tag_routing_prefix");
|
||||
});
|
||||
|
||||
it("should render an empty editor for a key holding only fields it cannot show", async () => {
|
||||
render(renderAccordion({ tag_routing_prefix: "team-" }));
|
||||
|
||||
await waitFor(() => expect(screen.getByTestId("fallbacks")).toHaveTextContent("none->none"));
|
||||
expect(JSON.parse(screen.getByTestId("loadbalancing").textContent ?? "null")).toEqual({});
|
||||
});
|
||||
});
|
||||
|
|
@ -95,6 +95,7 @@ export interface KeyResponse {
|
|||
object_permission?: ObjectPermission | null;
|
||||
access_group_ids?: string[];
|
||||
budget_fallbacks?: Record<string, string[]>;
|
||||
router_settings?: Record<string, unknown> | null;
|
||||
budget_limits?: Array<{ budget_duration: string; max_budget: number; reset_at?: string }>;
|
||||
auto_rotate?: boolean;
|
||||
rotation_interval?: string;
|
||||
|
|
|
|||
|
|
@ -59,6 +59,24 @@ vi.mock("../organisms/create_key_button", () => ({
|
|||
fetchTeamModels: vi.fn().mockResolvedValue(["team-model-1", "team-model-2"]),
|
||||
}));
|
||||
|
||||
const routerSettingsMocks = vi.hoisted(() => ({
|
||||
receivedValue: undefined as { router_settings: Record<string, unknown> } | undefined,
|
||||
editedValue: null as Record<string, unknown> | null,
|
||||
}));
|
||||
|
||||
vi.mock("../common_components/RouterSettingsAccordion", async () => {
|
||||
const { forwardRef, useImperativeHandle } = await import("react");
|
||||
return {
|
||||
default: forwardRef(({ value }: { value?: { router_settings: Record<string, unknown> } }, ref) => {
|
||||
routerSettingsMocks.receivedValue = value;
|
||||
useImperativeHandle(ref, () => ({
|
||||
getValue: () => ({ router_settings: routerSettingsMocks.editedValue ?? value?.router_settings ?? {} }),
|
||||
}));
|
||||
return <div data-testid="router-settings-accordion" />;
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@/app/(dashboard)/hooks/organizations/useOrganizations", () => ({
|
||||
useOrganizations: vi.fn().mockReturnValue({
|
||||
data: [
|
||||
|
|
@ -160,6 +178,83 @@ describe("KeyEditView", () => {
|
|||
last_rotation_at: undefined,
|
||||
key_rotation_at: undefined,
|
||||
};
|
||||
describe("router settings", () => {
|
||||
const UNSUPPORTED_STORED_FIELD = { tag_routing_prefix: "team-" };
|
||||
const STORED_ROUTER_SETTINGS = {
|
||||
num_retries: 2,
|
||||
fallbacks: [{ "gpt-4": ["gpt-4o"] }],
|
||||
...UNSUPPORTED_STORED_FIELD,
|
||||
};
|
||||
|
||||
const renderWithRouterSettings = (onSubmit: (values: Record<string, unknown>) => Promise<void>) =>
|
||||
renderWithProviders(
|
||||
<KeyEditView
|
||||
keyData={{ ...MOCK_KEY_DATA, router_settings: STORED_ROUTER_SETTINGS }}
|
||||
onCancel={() => {}}
|
||||
onSubmit={onSubmit}
|
||||
accessToken="test-token"
|
||||
userID="test-user"
|
||||
userRole="proxy_admin"
|
||||
premiumUser={true}
|
||||
/>,
|
||||
);
|
||||
|
||||
beforeEach(() => {
|
||||
routerSettingsMocks.receivedValue = undefined;
|
||||
routerSettingsMocks.editedValue = null;
|
||||
});
|
||||
|
||||
it("should load the fields it renders into the editor and withhold the ones it does not", async () => {
|
||||
renderWithRouterSettings(async () => {});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(routerSettingsMocks.receivedValue).toStrictEqual({
|
||||
router_settings: { num_retries: 2, fallbacks: [{ "gpt-4": ["gpt-4o"] }] },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("should submit edited fallbacks alongside routing fields the editor cannot show", async () => {
|
||||
const onSubmit = vi.fn().mockResolvedValue(undefined);
|
||||
renderWithRouterSettings(onSubmit);
|
||||
routerSettingsMocks.editedValue = { num_retries: 2, fallbacks: [{ "gpt-4": ["gpt-4o", "gpt-4o-mini"] }] };
|
||||
|
||||
fireEvent.click(screen.getByText("Save Changes"));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(onSubmit).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
router_settings: expect.objectContaining({
|
||||
...UNSUPPORTED_STORED_FIELD,
|
||||
num_retries: 2,
|
||||
fallbacks: [{ "gpt-4": ["gpt-4o", "gpt-4o-mini"] }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it("should submit cleared router settings so removing every fallback is persisted", async () => {
|
||||
const onSubmit = vi.fn().mockResolvedValue(undefined);
|
||||
renderWithRouterSettings(onSubmit);
|
||||
routerSettingsMocks.editedValue = { num_retries: null, fallbacks: null };
|
||||
|
||||
fireEvent.click(screen.getByText("Save Changes"));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(onSubmit).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
router_settings: expect.objectContaining({
|
||||
...UNSUPPORTED_STORED_FIELD,
|
||||
num_retries: null,
|
||||
fallbacks: null,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("should render", async () => {
|
||||
const { getByText } = renderWithProviders(
|
||||
<KeyEditView
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import PolicySelector from "@/components/policies/PolicySelector";
|
|||
import { InfoCircleOutlined } from "@ant-design/icons";
|
||||
import { TextInput, Button as TremorButton } from "@tremor/react";
|
||||
import { Form, Input, Select, Switch, Tooltip } from "antd";
|
||||
import { useEffect, useState } from "react";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { hasCapability } from "../../utils/capabilities";
|
||||
import { isProxyAdminRole, rolesWithWriteAccess } from "../../utils/roles";
|
||||
import AgentSelector from "../agent_management/AgentSelector";
|
||||
|
|
@ -17,6 +17,8 @@ import KeyLifecycleSettings from "../common_components/KeyLifecycleSettings";
|
|||
import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector";
|
||||
import RateLimitTypeFormItem from "../common_components/RateLimitTypeFormItem";
|
||||
import OrganizationDropdown from "../common_components/OrganizationDropdown";
|
||||
import RouterSettingsAccordion, { RouterSettingsAccordionRef } from "../common_components/RouterSettingsAccordion";
|
||||
import { routerSettingsEditorValue, routerSettingsUpdate } from "../common_components/routerSettingsPayload";
|
||||
import { extractLoggingSettings, formatMetadataForDisplay, stripTagsFromMetadata } from "../key_info_utils";
|
||||
import { estimateFields, estimateRules, estimateTooltips, withNormalizedEstimates } from "./estimatedOutputTokens";
|
||||
import { canonicalBudgetDuration, keyTypeFromRoutes } from "./keyEditFieldNormalizers";
|
||||
|
|
@ -91,6 +93,7 @@ export function KeyEditView({
|
|||
const [budgetFallbacks, setBudgetFallbacks] = useState<Record<string, string[]>>(
|
||||
keyData.budget_fallbacks && typeof keyData.budget_fallbacks === "object" ? keyData.budget_fallbacks : {},
|
||||
);
|
||||
const routerSettingsRef = useRef<RouterSettingsAccordionRef>(null);
|
||||
const { data: organizations, isLoading: isOrganizationsLoading } = useOrganizations();
|
||||
const { data: projects } = useProjects();
|
||||
const { data: uiSettingsData } = useUISettings();
|
||||
|
|
@ -286,6 +289,14 @@ export function KeyEditView({
|
|||
values.budget_fallbacks = {};
|
||||
}
|
||||
|
||||
const routerSettings = routerSettingsUpdate(
|
||||
routerSettingsRef.current?.getValue()?.router_settings,
|
||||
keyData.router_settings,
|
||||
);
|
||||
if (routerSettings) {
|
||||
values.router_settings = routerSettings;
|
||||
}
|
||||
|
||||
await onSubmit(withNormalizedEstimates(values));
|
||||
} finally {
|
||||
setIsKeySaving(false);
|
||||
|
|
@ -799,6 +810,15 @@ export function KeyEditView({
|
|||
<Input value={projectDisplay ?? ""} disabled />
|
||||
</Form.Item>
|
||||
)}
|
||||
<Form.Item label="Router Settings">
|
||||
<RouterSettingsAccordion
|
||||
ref={routerSettingsRef}
|
||||
accessToken={accessToken || ""}
|
||||
teamId={keyData.team_id}
|
||||
value={routerSettingsEditorValue(keyData.router_settings)}
|
||||
/>
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item label="Logging Settings" name="logging_settings">
|
||||
<EditLoggingSettings
|
||||
value={form.getFieldValue("logging_settings")}
|
||||
|
|
|
|||
|
|
@ -189,6 +189,25 @@ describe("KeyInfoView", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("should render the key's saved router fallbacks", async () => {
|
||||
vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock);
|
||||
|
||||
renderWithProviders(
|
||||
<KeyInfoView
|
||||
keyData={{ ...MOCK_KEY_DATA, router_settings: { num_retries: 2, fallbacks: [{ "gpt-4": ["gpt-4o"] }] } }}
|
||||
onClose={() => {}}
|
||||
keyId={"test-key-id"}
|
||||
onKeyDataUpdate={() => {}}
|
||||
teams={[]}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(await screen.findByText("Router Settings")).toBeInTheDocument();
|
||||
expect(screen.getByText("gpt-4")).toBeInTheDocument();
|
||||
expect(screen.getByText("gpt-4o")).toBeInTheDocument();
|
||||
expect(screen.getByText("Number of Retries: 2")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should render tags", async () => {
|
||||
vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock);
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ import { isProxyAdminRole, isUserTeamAdminForSingleTeam, rolesWithWriteAccess }
|
|||
import { mapDisplayToInternalNames, mapInternalToDisplayNames } from "../callback_info_helpers";
|
||||
import AutoRotationView from "../common_components/AutoRotationView";
|
||||
import DeleteResourceModal from "../common_components/DeleteResourceModal";
|
||||
import RouterSettingsSummary from "../common_components/RouterSettingsSummary";
|
||||
import { hasRouterSettings } from "../common_components/routerSettingsPayload";
|
||||
import { extractLoggingSettings, formatMetadataForDisplay, stripTagsFromMetadata } from "../key_info_utils";
|
||||
import { KeyResponse } from "../key_team_helpers/key_list";
|
||||
import LoggingSettingsView from "../logging_settings_view";
|
||||
|
|
@ -837,6 +839,15 @@ export default function KeyInfoView({
|
|||
</div>
|
||||
)}
|
||||
|
||||
{hasRouterSettings(currentKeyData.router_settings) && (
|
||||
<div>
|
||||
<Text className="font-medium">Router Settings</Text>
|
||||
<div className="mt-1">
|
||||
<RouterSettingsSummary routerSettings={currentKeyData.router_settings} />
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div>
|
||||
<Text className="font-medium">Tags</Text>
|
||||
<div className="flex flex-wrap gap-2 mt-1">
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue