scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-06 07:48:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 112

Merge branch 'litellm_internal_staging' into litellm_shadcn_shared_search_user_0812

Browse source
This commit is contained in:
yuneng-jiang 2026-08-12 23:35:24 -07:00 • committed by GitHub
commit 99b38c9fb9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
71 changed files with 4335 additions and 234 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
.github/workflows/test-terraform-modules.yml vendored Normal file
View file

@ -0,0 +1,54 @@
name: Terraform Modules
on:
push:
paths:
- "terraform/litellm/aws/**"
- ".github/workflows/test-terraform-modules.yml"
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
paths:
- "terraform/litellm/aws/**"
- ".github/workflows/test-terraform-modules.yml"
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
aws-module:
name: fmt, validate, test (aws)
runs-on: ubuntu-latest
timeout-minutes: 15
defaults:
run:
working-directory: terraform/litellm/aws
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
with:
terraform_version: 1.13.3
terraform_wrapper: false
- name: fmt
run: terraform fmt -recursive -check -diff
- name: init
run: terraform init -backend=false -input=false
- name: validate
run: terraform validate
# Plan-only, mock_provider-backed: no AWS credentials, no API calls.
- name: test
run: terraform test

2
CLAUDE.md
View file

@ -29,7 +29,7 @@ End-to-end tests belong in `tests/e2e/` and must follow the harness conventions
When creating PRs, don't set base to `main`. `litellm_internal_staging` is the default base branch and serves that purpose for both internal and external / OSS contributions
When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
When writing a PR body, treat the comments and imperative instructions inside .github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
Same applies for filing bug reports and feature requests, with .github/ISSUE_TEMPLATE/bug_report.yml and .github/ISSUE_TEMPLATE/feature_request.yml, respectively

10
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -90,7 +90,6 @@ class LangfuseOtelLogger(OpenTelemetry):
"generation_name": LangfuseSpanAttributes.GENERATION_NAME,
"generation_id": LangfuseSpanAttributes.GENERATION_ID,
"parent_observation_id": LangfuseSpanAttributes.PARENT_OBSERVATION_ID,
"version": LangfuseSpanAttributes.GENERATION_VERSION,
"mask_input": LangfuseSpanAttributes.MASK_INPUT,
"mask_output": LangfuseSpanAttributes.MASK_OUTPUT,
"trace_user_id": LangfuseSpanAttributes.TRACE_USER_ID,
@ -99,13 +98,18 @@ class LangfuseOtelLogger(OpenTelemetry):
"trace_name": LangfuseSpanAttributes.TRACE_NAME,
"trace_id": LangfuseSpanAttributes.TRACE_ID,
"trace_metadata": LangfuseSpanAttributes.TRACE_METADATA,
"trace_version": LangfuseSpanAttributes.TRACE_VERSION,
"trace_release": LangfuseSpanAttributes.TRACE_RELEASE,
"trace_release": LangfuseSpanAttributes.RELEASE,
"existing_trace_id": LangfuseSpanAttributes.EXISTING_TRACE_ID,
"update_trace_keys": LangfuseSpanAttributes.UPDATE_TRACE_KEYS,
"debug_langfuse": LangfuseSpanAttributes.DEBUG_LANGFUSE,
}
version: Final = (
metadata.get("trace_version") if metadata.get("trace_version") is not None else metadata.get("version")
)
if version is not None:
safe_set_attribute(span, LangfuseSpanAttributes.VERSION.value, version)
for key, enum_attr in mapping.items():
if key in metadata and metadata[key] is not None:
value = metadata[key]

8
litellm/integrations/otel/presets/langfuse.py
View file

@ -42,9 +42,7 @@ def langfuse_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str,
public_key: Final = params.get("langfuse_public_key")
secret_key: Final = params.get("langfuse_secret_key")
if public_key and secret_key:
return {
"Authorization": _V1Langfuse._get_langfuse_authorization_header(
public_key=public_key, secret_key=secret_key
)
}
return _V1Langfuse._build_langfuse_otel_headers(
_V1Langfuse._get_langfuse_authorization_header(public_key=public_key, secret_key=secret_key)
)
return {}

4
litellm/litellm_core_utils/core_helpers.py
View file

@ -1,7 +1,7 @@
# What is this?
## Helper utilities
import copy
from collections.abc import Iterable
from collections.abc import Iterable, Mapping
from typing import TYPE_CHECKING, Any, Final, Literal
import httpx
@ -181,7 +181,7 @@ def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metad
def get_metadata_variable_name_from_kwargs(
kwargs: dict,
kwargs: Mapping[str, object],
) -> Literal["metadata", "litellm_metadata"]:
"""
Helper to return what the "metadata" field should be called in the request data

117
litellm/model_prices_and_context_window_backup.json
View file

@ -15552,6 +15552,17 @@
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning": {
"max_input_tokens": 262144,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 2e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"source": "https://deepinfra.com/nvidia/NVIDIA-Nemotron-3.5-Lightning",
"supports_tool_choice": true,
"supports_function_calling": true,
"supports_reasoning": true
},
"deepinfra/nvidia/NVIDIA-Nemotron-Nano-9B-v2": {
"max_tokens": 131072,
"max_input_tokens": 131072,
@ -26109,11 +26120,12 @@
"supports_vision": true
},
"groq/llama-3.1-8b-instant": {
"deprecation_date": "2026-08-16",
"input_cost_per_token": 5e-08,
"litellm_provider": "groq",
"max_input_tokens": 128000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 8e-08,
"supports_function_calling": true,
@ -26121,9 +26133,10 @@
"supports_tool_choice": true
},
"groq/llama-3.3-70b-versatile": {
"deprecation_date": "2026-08-16",
"input_cost_per_token": 5.9e-07,
"litellm_provider": "groq",
"max_input_tokens": 128000,
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
@ -26144,7 +26157,28 @@
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/meta-llama/llama-prompt-guard-2-22m": {
"input_cost_per_token": 3e-08,
"litellm_provider": "groq",
"max_input_tokens": 512,
"max_output_tokens": 512,
"max_tokens": 512,
"mode": "chat",
"output_cost_per_token": 3e-08,
"source": "https://console.groq.com/docs/models"
},
"groq/meta-llama/llama-prompt-guard-2-86m": {
"input_cost_per_token": 4e-08,
"litellm_provider": "groq",
"max_input_tokens": 512,
"max_output_tokens": 512,
"max_tokens": 512,
"mode": "chat",
"output_cost_per_token": 4e-08,
"source": "https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m"
},
"groq/meta-llama/llama-guard-4-12b": {
"deprecation_date": "2026-03-05",
"input_cost_per_token": 2e-07,
"litellm_provider": "groq",
"max_input_tokens": 8192,
@ -26154,6 +26188,7 @@
"output_cost_per_token": 2e-07
},
"groq/meta-llama/llama-4-maverick-17b-128e-instruct": {
"deprecation_date": "2026-03-09",
"input_cost_per_token": 2e-07,
"litellm_provider": "groq",
"max_input_tokens": 131072,
@ -26167,6 +26202,7 @@
"supports_vision": true
},
"groq/meta-llama/llama-4-scout-17b-16e-instruct": {
"deprecation_date": "2026-07-17",
"input_cost_per_token": 1.1e-07,
"litellm_provider": "groq",
"max_input_tokens": 131072,
@ -26180,6 +26216,7 @@
"supports_vision": true
},
"groq/moonshotai/kimi-k2-instruct-0905": {
"deprecation_date": "2026-04-15",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 3e-06,
"cache_read_input_token_cost": 5e-07,
@ -26197,8 +26234,8 @@
"input_cost_per_token": 1.5e-07,
"litellm_provider": "groq",
"max_input_tokens": 131072,
"max_output_tokens": 32766,
"max_tokens": 32766,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 6e-07,
"search_context_cost_per_query": {
@ -26218,8 +26255,8 @@
"input_cost_per_token": 7.5e-08,
"litellm_provider": "groq",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 3e-07,
"search_context_cost_per_query": {
@ -26254,7 +26291,26 @@
"supports_tool_choice": true,
"supports_web_search": true
},
"groq/canopylabs/orpheus-v1-english": {
"input_cost_per_character": 2.2e-05,
"litellm_provider": "groq",
"max_input_tokens": 4000,
"max_output_tokens": 50000,
"max_tokens": 50000,
"mode": "audio_speech",
"source": "https://console.groq.com/docs/model/canopylabs/orpheus-v1-english"
},
"groq/canopylabs/orpheus-arabic-saudi": {
"input_cost_per_character": 4e-05,
"litellm_provider": "groq",
"max_input_tokens": 4000,
"max_output_tokens": 50000,
"max_tokens": 50000,
"mode": "audio_speech",
"source": "https://console.groq.com/docs/models"
},
"groq/playai-tts": {
"deprecation_date": "2025-12-31",
"input_cost_per_character": 5e-05,
"litellm_provider": "groq",
"max_input_tokens": 10000,
@ -26262,7 +26318,23 @@
"max_tokens": 10000,
"mode": "audio_speech"
},
"groq/qwen/qwen3.6-27b": {
"input_cost_per_token": 6e-07,
"litellm_provider": "groq",
"max_input_tokens": 131072,
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "chat",
"output_cost_per_token": 3e-06,
"source": "https://console.groq.com/docs/model/qwen/qwen3.6-27b",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": false,
"supports_tool_choice": true,
"supports_vision": true
},
"groq/qwen/qwen3-32b": {
"deprecation_date": "2026-07-17",
"input_cost_per_token": 2.9e-07,
"litellm_provider": "groq",
"max_input_tokens": 131000,
@ -31754,6 +31826,17 @@
"supports_video_input": true,
"supports_vision": true
},
"openrouter/nvidia/nemotron-3.5-lightning": {
"input_cost_per_token": 5e-08,
"litellm_provider": "openrouter",
"max_input_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 2e-07,
"source": "https://openrouter.ai/nvidia/nemotron-3.5-lightning",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-3.5-turbo": {
"input_cost_per_token": 1.5e-06,
"litellm_provider": "openrouter",
@ -45804,11 +45887,15 @@
},
"bedrock_mantle/openai.gpt-5.6-sol": {
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_272k_tokens": 1.1e-05,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1.375e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
"output_cost_per_token": 3.3e-05,
"output_cost_per_token_above_272k_tokens": 4.95e-05,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 272000,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
@ -45832,11 +45919,15 @@
},
"bedrock_mantle/openai.gpt-5.6-terra": {
"input_cost_per_token": 2.2e-06,
"input_cost_per_token_above_272k_tokens": 4.4e-06,
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5.5e-06,
"cache_read_input_token_cost": 2.2e-07,
"cache_read_input_token_cost_above_272k_tokens": 4.4e-07,
"output_cost_per_token": 1.32e-05,
"output_cost_per_token_above_272k_tokens": 1.98e-05,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 272000,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
@ -45860,11 +45951,15 @@
},
"bedrock_mantle/openai.gpt-5.6-luna": {
"input_cost_per_token": 2.2e-07,
"input_cost_per_token_above_272k_tokens": 4.4e-07,
"cache_creation_input_token_cost": 2.75e-07,
"cache_creation_input_token_cost_above_272k_tokens": 5.5e-07,
"cache_read_input_token_cost": 2.2e-08,
"cache_read_input_token_cost_above_272k_tokens": 4.4e-08,
"output_cost_per_token": 1.32e-06,
"output_cost_per_token_above_272k_tokens": 1.98e-06,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 272000,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",

29
litellm/proxy/health_endpoints/_health_endpoints.py
View file

@ -50,6 +50,7 @@ from litellm.router_utils.clientside_credential_handler import (
_ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path
clientside_credential_keys,
)
from litellm.secret_managers.main import get_secret_bool
#### Health ENDPOINTS ####
@ -1447,6 +1448,31 @@ def callback_name(callback):
return str(callback)
DISABLE_NO_REDIS_WARNING_ENV_VAR: Final = "LITELLM_DISABLE_NO_REDIS_WARNING"
def _show_no_redis_warning() -> bool:
"""
Whether the UI should warn that no Redis is configured.
Redis is what makes rate limits, budgets, router state, and cache
invalidation consistent across workers, so a proxy running without it is
only safe as a single worker. Both places a Redis can land count: the
coordination cache (from a Redis response cache, general_settings.
coordination_redis, or the REDIS_* env fallback) and the router's own
Redis (router_settings.redis_host), which backs cooldowns and usage-based
routing on its own. Operators who know they run one worker can silence the
warning with LITELLM_DISABLE_NO_REDIS_WARNING=true.
"""
from litellm.proxy.proxy_server import llm_router, redis_usage_cache
if redis_usage_cache is not None:
return False
if llm_router is not None and llm_router.cache.redis_cache is not None:
return False
return get_secret_bool(DISABLE_NO_REDIS_WARNING_ENV_VAR, False) is not True
async def _get_health_readiness_details(
response: Response | None = None,
) -> dict[str, Any]:
@ -1487,6 +1513,7 @@ async def _get_health_readiness_details(
# check log level
log_level_name: Final = logging.getLevelName(verbose_logger.getEffectiveLevel())
is_detailed_debug: Final = verbose_logger.isEnabledFor(logging.DEBUG)
show_no_redis_warning: Final = _show_no_redis_warning()
# check DB
if prisma_client is not None: # if db passed in, check if it's connected
@ -1506,6 +1533,7 @@ async def _get_health_readiness_details(
"use_aiohttp_transport": AsyncHTTPHandler._should_use_aiohttp_transport(),
"log_level": log_level_name,
"is_detailed_debug": is_detailed_debug,
"show_no_redis_warning": show_no_redis_warning,
}
else:
return {
@ -1517,6 +1545,7 @@ async def _get_health_readiness_details(
"use_aiohttp_transport": AsyncHTTPHandler._should_use_aiohttp_transport(),
"log_level": log_level_name,
"is_detailed_debug": is_detailed_debug,
"show_no_redis_warning": show_no_redis_warning,
}
except Exception as e:
raise HTTPException(status_code=503, detail=f"Service Unhealthy ({e})")

22
litellm/proxy/proxy_server.py
View file

@ -845,6 +845,22 @@ def cleanup_router_config_variables():
prisma_client = None
async def _flush_spend_logs_queue_on_shutdown() -> None:
if prisma_client is None:
return
try:
from litellm.proxy.utils import drain_spend_logs_queue
await drain_spend_logs_queue(
prisma_client=prisma_client,
db_writer_client=db_writer_client,
proxy_logging_obj=proxy_logging_obj,
)
except Exception as e: # noqa: BLE001 # shutdown must continue even if the drain fails
verbose_proxy_logger.exception("Error flushing spend logs queue on shutdown: %s", e)
async def proxy_shutdown_event():
global prisma_client, master_key, user_custom_auth, user_custom_key_generate, user_custom_key_update
verbose_proxy_logger.info("Shutting down LiteLLM Proxy Server")
@ -1255,6 +1271,8 @@ async def proxy_startup_event(app: FastAPI):
except Exception as e:
verbose_proxy_logger.error("Error stopping DB health watchdog task: %s", e)
await _flush_spend_logs_queue_on_shutdown()
await proxy_config.stop_config_sync_subscriber()
await proxy_config.stop_auth_cache_invalidation_subscriber()
@ -8731,14 +8749,14 @@ class ProxyStartupEvent:
if general_settings.get("disable_spend_logs", False) is False:
from litellm.proxy.utils import _monitor_spend_logs_queue
# Start background task to monitor spend logs queue size
asyncio.create_task(
monitor_task: Final = asyncio.create_task(
_monitor_spend_logs_queue(
prisma_client=prisma_client,
db_writer_client=db_writer_client,
proxy_logging_obj=proxy_logging_obj,
)
)
prisma_client.spend_logs_queue_monitor_task = monitor_task # rebind-ok: the client owns its monitor handle
### ADD NEW MODELS ###
store_model_in_db = get_secret_bool("STORE_MODEL_IN_DB", store_model_in_db) or store_model_in_db

58
litellm/proxy/utils.py
View file

@ -1,4 +1,5 @@
import asyncio
import contextlib
import copy
import hashlib
import inspect
@ -3006,6 +3007,7 @@ async def prefetch_config_params(prisma_client: "PrismaClient | None", param_nam
class PrismaClient:
spend_log_transactions: list = []
_spend_log_transactions_lock = asyncio.Lock()
spend_logs_queue_monitor_task: "asyncio.Task[None] | None" = None
tool_usage_transactions: list["ToolUsageTransaction"] = []
_tool_usage_transactions_lock = asyncio.Lock()
autorouter_turn_transactions: ClassVar[
@ -5722,13 +5724,22 @@ async def update_spend_logs_job(
logs_to_process: Final = prisma_client.spend_log_transactions[:MAX_LOGS_PER_INTERVAL]
prisma_client.spend_log_transactions = prisma_client.spend_log_transactions[len(logs_to_process) :]
await ProxyUpdateSpend.update_spend_logs(
n_retry_times=n_retry_times,
prisma_client=prisma_client,
proxy_logging_obj=proxy_logging_obj,
db_writer_client=db_writer_client,
logs_to_process=logs_to_process,
)
try:
await ProxyUpdateSpend.update_spend_logs(
n_retry_times=n_retry_times,
prisma_client=prisma_client,
proxy_logging_obj=proxy_logging_obj,
db_writer_client=db_writer_client,
logs_to_process=logs_to_process,
)
except asyncio.CancelledError:
async with prisma_client._spend_log_transactions_lock:
prisma_client.spend_log_transactions[:0] = logs_to_process
verbose_proxy_logger.warning(
"Spend tracking - spend log write cancelled, requeued %d rows for the next flush",
len(logs_to_process),
)
raise
# Guardrail/policy usage tracking (same batch, outside spend-logs update)
try:
@ -5787,6 +5798,39 @@ async def update_spend_logs_job(
)
MAX_SPEND_LOG_DRAIN_ITERATIONS: Final = 20
async def drain_spend_logs_queue(
prisma_client: PrismaClient,
db_writer_client: "AsyncHTTPHandler | None",
proxy_logging_obj: ProxyLogging,
) -> None:
monitor_task: Final = prisma_client.spend_logs_queue_monitor_task
if monitor_task is not None:
monitor_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await monitor_task
prisma_client.spend_logs_queue_monitor_task = None # rebind-ok: the client owns its monitor handle
for _ in range(MAX_SPEND_LOG_DRAIN_ITERATIONS):
if await _total_queued_spend_transactions(prisma_client) == 0:
return
await update_spend_logs_job(
prisma_client=prisma_client,
db_writer_client=db_writer_client,
proxy_logging_obj=proxy_logging_obj,
)
remaining: Final = await _total_queued_spend_transactions(prisma_client)
if remaining > 0:
spend_log_error(
"Spend tracking - %d spend log rows still queued after %d drain passes",
remaining,
MAX_SPEND_LOG_DRAIN_ITERATIONS,
)
async def _monitor_spend_logs_queue(
prisma_client: PrismaClient,
db_writer_client: AsyncHTTPHandler | None,

99
litellm/router.py
View file

@ -96,6 +96,7 @@ from litellm.router_utils.add_retry_fallback_headers import (
response_in_flight_token_count,
)
from litellm.router_utils.auto_router_model_naming import (
AUTO_ROUTER_MODEL_PREFIX,
classify_strategy_router_model,
)
from litellm.router_utils.batch_utils import (
@ -318,6 +319,8 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None
_PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT")
_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"})
def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) -> bool:
for chunk in chunks:
@ -8612,7 +8615,18 @@ class Router:
Nothing is recorded for replay: a refresh walks the live routers instead,
so a deleted, repointed or never-added deployment, and a discarded router,
drop out of the rebuild on their own.
A strategy-router alias is never the deployment actually called or
billed, so custom pricing configured on it must not become a cost-map
price: an explicit zero would let ``_is_cost_explicitly_configured``
treat the alias as a genuinely free model and waive budget checks for
requests that route to (and bill as) a real deployment.
"""
if classify_strategy_router_model(model) is not None:
model_info = { # mutable-ok: filtered copy of the caller's entry, handed straight to register_model
k: v for k, v in model_info.items() if k not in CustomPricingLiteLLMParams.model_fields
}
if model_id is not None:
litellm.register_model(model_cost={model_id: model_info}, persist_across_reloads=False)
@ -10701,6 +10715,14 @@ class Router:
return None
@staticmethod
def _is_strategy_marker_deployment(deployment: Mapping[str, object]) -> bool:
litellm_params: Final = deployment.get("litellm_params")
if not isinstance(litellm_params, Mapping):
return False
deployment_model: Final = litellm_params.get("model")
return isinstance(deployment_model, str) and classify_strategy_router_model(deployment_model) is not None
def _common_checks_available_deployment(
self,
model: str,
@ -10828,7 +10850,12 @@ class Router:
model
] # update the model to the actual value if an alias has been passed in
return model, healthy_deployments
marker_flags: Final = tuple(self._is_strategy_marker_deployment(d) for d in healthy_deployments)
if all(marker_flags) or not any(marker_flags):
return model, healthy_deployments
return model, [ # mutable-ok: matches this function's list contract expected by downstream filters
d for d, is_marker in zip(healthy_deployments, marker_flags, strict=True) if not is_marker
]
def _filter_deployments_by_model_access_groups(
self,
@ -11341,6 +11368,10 @@ class Router:
return filtered
def _model_name_has_plain_deployments(self, model: str) -> bool:
indices: Final = self.model_name_to_deployment_indices.get(model) or ()
return any(not self._is_strategy_marker_deployment(self.model_list[idx]) for idx in indices)
def _select_pre_routing_strategy(
self, model: str, request_kwargs: dict
) -> "TaggedPreRoutingStrategy[PreRoutingStrategy] | None":
@ -11349,7 +11380,14 @@ class Router:
that share a `model_name` by matching the request's tags against each
registered strategy's tags before falling back to the first registered.
Returns the tagged registry entry so the caller can tell whether the
request's tags were what selected it.
request's tags were what selected it, and can locate the marker
deployment the strategy was registered from via its (model_name, tags)
pair.
With tag filtering enabled, strategies that all carry real tags matching
none of the request's do not capture it when the name also has plain
deployments: returning None hands the request to ordinary tag-aware
deployment selection.
"""
candidates: Final[list[TaggedPreRoutingStrategy[PreRoutingStrategy]]] = [
*self.auto_routers.get(model, []),
@ -11359,8 +11397,6 @@ class Router:
]
if not candidates:
return None
if len(candidates) == 1:
return candidates[0]
request_tags: Final = _get_tags_from_request_kwargs(request_kwargs)
if request_tags:
@ -11372,6 +11408,12 @@ class Router:
for tagged in candidates:
if "default" in tagged.tags:
return tagged
if (
self.enable_tag_filtering
and all(tagged.tags for tagged in candidates)
and self._model_name_has_plain_deployments(model)
):
return None
return candidates[0]
async def async_pre_routing_hook(
@ -11434,22 +11476,47 @@ class Router:
)
# `model` (the alias, e.g. "smart-router") is never the deployment actually
# called - apply the alias's own litellm_params (besides `model` itself,
# which is just the alias marker) to the request, since the tier/route
# deployment the hook selected won't have them. Router-only fields
# (tpm, rpm, weight, complexity_router_config, ...) are excluded from the
# actual outbound LLM call downstream by litellm.types.utils.all_litellm_params,
# not here.
# called - apply the router marker's own litellm_params to the request,
# since the tier/route deployment the hook selected won't have them. The
# marker entry is looked up by its `auto_router/` model prefix and the
# selected strategy's tags, never by list position: plain deployments may
# share the alias `model_name` and must not leak their params (`api_base`,
# `api_key`, ...) onto the routed call. Router-only fields (tpm, rpm,
# weight, complexity_router_config, ...) are excluded from the actual
# outbound LLM call downstream by litellm.types.utils.all_litellm_params,
# not here. Custom pricing fields ARE call params, so they must be
# excluded here: they price the alias, not the deployment the hook
# selected, and forwarding them re-registers the routed deployment at
# the alias's price (an explicit 0 makes every alias request bill $0).
if pre_routing_hook_response is not None:
alias_index: Final = self.model_name_to_deployment_indices.get(model, [])
if alias_index:
alias_litellm_params: Final = self.model_list[alias_index[0]].get("litellm_params", {})
for key, value in alias_litellm_params.items():
if key != "model" and value is not None:
request_kwargs.setdefault(key, value)
for key, value in self._forwardable_alias_marker_params(model=model, strategy_tags=selected_strategy.tags):
request_kwargs.setdefault(key, value)
return pre_routing_hook_response
def _forwardable_alias_marker_params(
self, model: str, strategy_tags: tuple[str, ...]
) -> tuple[tuple[str, object], ...]:
marker_params: Final = tuple(
litellm_params
for idx in self.model_name_to_deployment_indices.get(model, ())
if isinstance(litellm_params := self.model_list[idx].get("litellm_params", {}), dict)
and str(litellm_params.get("model", "")).startswith(AUTO_ROUTER_MODEL_PREFIX)
)
tag_matched: Final = tuple(
params for params in marker_params if tuple(params.get("tags") or ()) == strategy_tags
)
selected: Final = tag_matched[0] if tag_matched else (marker_params[0] if marker_params else None)
if selected is None:
return ()
return tuple(
(key, value)
for key, value in selected.items()
if key not in _ALIAS_PARAMS_NEVER_FORWARDED
and key not in CustomPricingLiteLLMParams.model_fields
and value is not None
)
def _consumed_request_tags_stamp(
self,
selected_strategy: "TaggedPreRoutingStrategy[PreRoutingStrategy]",

35
litellm/router_strategy/tag_based_routing.py
View file

@ -584,8 +584,26 @@ async def get_deployments_for_tag(
return healthy_deployments
def _tags_in_metadata(metadata: object) -> list[str]:
"""
Tags out of a metadata bucket the caller controls the shape of.
A request can send its metadata (and its ``tags``) as anything the JSON body
allowed, an unparsed string or null included, so any shape that is not a list
of string tags carries no tags rather than raising.
"""
if not isinstance(metadata, Mapping):
return []
typed_metadata: Final[Mapping[str, object]] = metadata
tags: Final = typed_metadata.get("tags")
if isinstance(tags, str) or not isinstance(tags, Sequence):
return []
typed_tags: Final[Sequence[object]] = tags
return [tag for tag in typed_tags if isinstance(tag, str)]
def _get_tags_from_request_kwargs(
request_kwargs: dict[Any, Any] | None = None,
request_kwargs: Mapping[Any, Any] | None = None,
metadata_variable_name: Literal["metadata", "litellm_metadata"] | None = None,
) -> list[str]:
"""
@ -604,12 +622,11 @@ def _get_tags_from_request_kwargs(
return []
resolved_variable_name: Final = metadata_variable_name or get_metadata_variable_name_from_kwargs(request_kwargs)
if resolved_variable_name in request_kwargs:
metadata: Final = request_kwargs[resolved_variable_name] or {}
tags = metadata.get("tags", [])
return tags if tags is not None else []
elif "litellm_params" in request_kwargs:
litellm_params: Final = request_kwargs["litellm_params"] or {}
_metadata: Final = litellm_params.get(resolved_variable_name, {}) or {}
tags = _metadata.get("tags", [])
return tags if tags is not None else []
return _tags_in_metadata(request_kwargs[resolved_variable_name])
if "litellm_params" in request_kwargs:
litellm_params: Final = request_kwargs["litellm_params"]
if not isinstance(litellm_params, Mapping):
return []
typed_litellm_params: Final[Mapping[str, object]] = litellm_params
return _tags_in_metadata(typed_litellm_params.get(resolved_variable_name))
return []

5
litellm/types/integrations/langfuse_otel.py
View file

@ -16,12 +16,13 @@ class LangfuseOtelConfig(BaseModel):
class LangfuseSpanAttributes(str, Enum):
LANGFUSE_ENVIRONMENT = "langfuse.environment"
VERSION = "langfuse.version"
RELEASE = "langfuse.release"
# ---- Generation-level metadata ----
GENERATION_NAME = "langfuse.generation.name"
GENERATION_ID = "langfuse.generation.id"
PARENT_OBSERVATION_ID = "langfuse.generation.parent_observation_id"
GENERATION_VERSION = "langfuse.generation.version"
MASK_INPUT = "langfuse.generation.mask_input"
MASK_OUTPUT = "langfuse.generation.mask_output"
@ -36,8 +37,6 @@ class LangfuseSpanAttributes(str, Enum):
TRACE_NAME = "langfuse.trace.name"
TRACE_ID = "langfuse.trace.id"
TRACE_METADATA = "langfuse.trace.metadata"
TRACE_VERSION = "langfuse.trace.version"
TRACE_RELEASE = "langfuse.trace.release"
EXISTING_TRACE_ID = "langfuse.trace.existing_id"
UPDATE_TRACE_KEYS = "langfuse.trace.update_keys"

117
model_prices_and_context_window.json
View file

@ -15552,6 +15552,17 @@
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning": {
"max_input_tokens": 262144,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 2e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"source": "https://deepinfra.com/nvidia/NVIDIA-Nemotron-3.5-Lightning",
"supports_tool_choice": true,
"supports_function_calling": true,
"supports_reasoning": true
},
"deepinfra/nvidia/NVIDIA-Nemotron-Nano-9B-v2": {
"max_tokens": 131072,
"max_input_tokens": 131072,
@ -26109,11 +26120,12 @@
"supports_vision": true
},
"groq/llama-3.1-8b-instant": {
"deprecation_date": "2026-08-16",
"input_cost_per_token": 5e-08,
"litellm_provider": "groq",
"max_input_tokens": 128000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 8e-08,
"supports_function_calling": true,
@ -26121,9 +26133,10 @@
"supports_tool_choice": true
},
"groq/llama-3.3-70b-versatile": {
"deprecation_date": "2026-08-16",
"input_cost_per_token": 5.9e-07,
"litellm_provider": "groq",
"max_input_tokens": 128000,
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
@ -26144,7 +26157,28 @@
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/meta-llama/llama-prompt-guard-2-22m": {
"input_cost_per_token": 3e-08,
"litellm_provider": "groq",
"max_input_tokens": 512,
"max_output_tokens": 512,
"max_tokens": 512,
"mode": "chat",
"output_cost_per_token": 3e-08,
"source": "https://console.groq.com/docs/models"
},
"groq/meta-llama/llama-prompt-guard-2-86m": {
"input_cost_per_token": 4e-08,
"litellm_provider": "groq",
"max_input_tokens": 512,
"max_output_tokens": 512,
"max_tokens": 512,
"mode": "chat",
"output_cost_per_token": 4e-08,
"source": "https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m"
},
"groq/meta-llama/llama-guard-4-12b": {
"deprecation_date": "2026-03-05",
"input_cost_per_token": 2e-07,
"litellm_provider": "groq",
"max_input_tokens": 8192,
@ -26154,6 +26188,7 @@
"output_cost_per_token": 2e-07
},
"groq/meta-llama/llama-4-maverick-17b-128e-instruct": {
"deprecation_date": "2026-03-09",
"input_cost_per_token": 2e-07,
"litellm_provider": "groq",
"max_input_tokens": 131072,
@ -26167,6 +26202,7 @@
"supports_vision": true
},
"groq/meta-llama/llama-4-scout-17b-16e-instruct": {
"deprecation_date": "2026-07-17",
"input_cost_per_token": 1.1e-07,
"litellm_provider": "groq",
"max_input_tokens": 131072,
@ -26180,6 +26216,7 @@
"supports_vision": true
},
"groq/moonshotai/kimi-k2-instruct-0905": {
"deprecation_date": "2026-04-15",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 3e-06,
"cache_read_input_token_cost": 5e-07,
@ -26197,8 +26234,8 @@
"input_cost_per_token": 1.5e-07,
"litellm_provider": "groq",
"max_input_tokens": 131072,
"max_output_tokens": 32766,
"max_tokens": 32766,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 6e-07,
"search_context_cost_per_query": {
@ -26218,8 +26255,8 @@
"input_cost_per_token": 7.5e-08,
"litellm_provider": "groq",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 3e-07,
"search_context_cost_per_query": {
@ -26254,7 +26291,26 @@
"supports_tool_choice": true,
"supports_web_search": true
},
"groq/canopylabs/orpheus-v1-english": {
"input_cost_per_character": 2.2e-05,
"litellm_provider": "groq",
"max_input_tokens": 4000,
"max_output_tokens": 50000,
"max_tokens": 50000,
"mode": "audio_speech",
"source": "https://console.groq.com/docs/model/canopylabs/orpheus-v1-english"
},
"groq/canopylabs/orpheus-arabic-saudi": {
"input_cost_per_character": 4e-05,
"litellm_provider": "groq",
"max_input_tokens": 4000,
"max_output_tokens": 50000,
"max_tokens": 50000,
"mode": "audio_speech",
"source": "https://console.groq.com/docs/models"
},
"groq/playai-tts": {
"deprecation_date": "2025-12-31",
"input_cost_per_character": 5e-05,
"litellm_provider": "groq",
"max_input_tokens": 10000,
@ -26262,7 +26318,23 @@
"max_tokens": 10000,
"mode": "audio_speech"
},
"groq/qwen/qwen3.6-27b": {
"input_cost_per_token": 6e-07,
"litellm_provider": "groq",
"max_input_tokens": 131072,
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "chat",
"output_cost_per_token": 3e-06,
"source": "https://console.groq.com/docs/model/qwen/qwen3.6-27b",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": false,
"supports_tool_choice": true,
"supports_vision": true
},
"groq/qwen/qwen3-32b": {
"deprecation_date": "2026-07-17",
"input_cost_per_token": 2.9e-07,
"litellm_provider": "groq",
"max_input_tokens": 131000,
@ -31754,6 +31826,17 @@
"supports_video_input": true,
"supports_vision": true
},
"openrouter/nvidia/nemotron-3.5-lightning": {
"input_cost_per_token": 5e-08,
"litellm_provider": "openrouter",
"max_input_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 2e-07,
"source": "https://openrouter.ai/nvidia/nemotron-3.5-lightning",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-3.5-turbo": {
"input_cost_per_token": 1.5e-06,
"litellm_provider": "openrouter",
@ -45804,11 +45887,15 @@
},
"bedrock_mantle/openai.gpt-5.6-sol": {
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_272k_tokens": 1.1e-05,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1.375e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
"output_cost_per_token": 3.3e-05,
"output_cost_per_token_above_272k_tokens": 4.95e-05,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 272000,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
@ -45832,11 +45919,15 @@
},
"bedrock_mantle/openai.gpt-5.6-terra": {
"input_cost_per_token": 2.2e-06,
"input_cost_per_token_above_272k_tokens": 4.4e-06,
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5.5e-06,
"cache_read_input_token_cost": 2.2e-07,
"cache_read_input_token_cost_above_272k_tokens": 4.4e-07,
"output_cost_per_token": 1.32e-05,
"output_cost_per_token_above_272k_tokens": 1.98e-05,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 272000,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
@ -45860,11 +45951,15 @@
},
"bedrock_mantle/openai.gpt-5.6-luna": {
"input_cost_per_token": 2.2e-07,
"input_cost_per_token_above_272k_tokens": 4.4e-07,
"cache_creation_input_token_cost": 2.75e-07,
"cache_creation_input_token_cost_above_272k_tokens": 5.5e-07,
"cache_read_input_token_cost": 2.2e-08,
"cache_read_input_token_cost_above_272k_tokens": 4.4e-08,
"output_cost_per_token": 1.32e-06,
"output_cost_per_token_above_272k_tokens": 1.98e-06,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 272000,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",

65
terraform/litellm/aws/README.md
View file

@ -2,9 +2,9 @@
Deploys the componentized LiteLLM proxy on AWS:
- **VPC** with public + private subnets across the AZs you pass in, one NAT gateway
- **Aurora Postgres** cluster — one writer instance + one reader instance, **IAM database authentication enabled**
- **ElastiCache Redis** (private, replication group with multi-AZ failover and at-rest + in-transit encryption) for caching + rate limiting
- **VPC** with public + private subnets across the AZs you pass in, one NAT gateway (skipped when you pass an existing `vpc_id`)
- **Aurora Postgres** cluster — one writer instance + one reader instance, **IAM database authentication enabled** (skipped when `create_database = false`)
- **ElastiCache Redis** (private, replication group with multi-AZ failover and at-rest + in-transit encryption) for caching + rate limiting (skipped when `create_redis = false`)
- **S3 bucket** (private, versioned, SSE-S3) — exposed to gateway + backend as `S3_BUCKET_NAME` / `S3_REGION_NAME` for cache backend, request log archival, and `/v1/files` storage
- **Secrets Manager** entries for `LITELLM_MASTER_KEY` (auto-generated, `sk-…`) and the Aurora master password (bootstrap-only)
- **ECS Fargate cluster** running three services — `gateway`, `backend`, `ui`
@ -14,6 +14,58 @@ Deploys the componentized LiteLLM proxy on AWS:
- Everything else (management API: `/key/*`, `/user/*`, …) → `backend`
- **One-off migration task** (`litellm-migrations`) that runs `prisma migrate deploy` from the dedicated `ghcr.io/berriai/litellm-migrations` image
## Bring your own networking, database, and Redis
The three infrastructure pieces the stack would otherwise own are each
optional, so it can slot into an account where networking and data stores are
already provisioned (often by another team, in another Terraform state).
**Networking.** Set `vpc_id` plus `public_subnet_ids` and `private_subnet_ids`
and no VPC, subnet, route table, internet gateway, or NAT gateway is created.
The ALB goes in the public subnets, the ECS tasks and any subnet group the
stack still needs go in the private ones, and `vpc_cidr` / `azs` go unused.
The private subnets need their own egress (NAT gateway, or VPC endpoints
covering ECR, S3, CloudWatch Logs, and Secrets Manager) since tasks pull
images, resolve secrets, and call LLM providers.
Security groups stay module-owned in either mode: the ALB group, the tasks
group, and the database/cache groups when it creates those. To let the tasks
reach infrastructure the module doesn't manage, either allow inbound from the
group named by the `task_security_group_id` output, or attach a group of your
own with `additional_task_security_group_ids`.
```hcl
vpc_id = "vpc-0123456789abcdef0"
public_subnet_ids = ["subnet-aaa", "subnet-bbb"]
private_subnet_ids = ["subnet-ccc", "subnet-ddd"]
```
**Database and Redis.** `create_database` and `create_redis` default to `true`
(today's behavior). Set one to `false` and pass a connection string to use
something you already run: the value lands in a Secrets Manager entry and
reaches gateway, backend, and the migration task as `DATABASE_URL` /
`REDIS_URL`, both of which outrank the discrete `DATABASE_*` / `REDIS_*` vars
in the proxy, so nothing appears in plain text in a task definition.
```hcl
create_database = false
database_url = "postgresql://litellm:...@db.internal:5432/litellm"
create_redis = false
redis_url = "rediss://:...@cache.internal:6379"
```
The schema migration still runs on every apply against an existing database;
only the Aurora-specific IAM-user bootstrap drops out, since those credentials
are already in the URL.
Leaving the URL empty runs without the component entirely:
- No database: no virtual keys, teams, spend tracking, or UI persistence, and
`STORE_MODEL_IN_DB` is not set, so models come from `proxy_config`. Requests
authenticate with `LITELLM_MASTER_KEY` only.
- No Redis: rate limits, budgets, and router cooldowns are per-task rather
than cluster-wide, which is only sane at one task per service.
## Aurora + IAM auth
The cluster runs with `iam_database_authentication_enabled = true`. Enabling
@ -345,7 +397,7 @@ trial / dev stacks only.
## Storage and database retention
Three opt-in tripwires guard against accidental data loss on
Two opt-in tripwires guard against accidental data loss on
`terraform destroy`:
- **`skip_final_snapshot`** (Aurora; default `false`) — destroying the
@ -354,6 +406,9 @@ Three opt-in tripwires guard against accidental data loss on
`/v1/files` content, and the S3 cache backend; default `false`) —
`terraform destroy` against a non-empty bucket fails.
Neither applies to a database you brought yourself: its lifecycle stays with
whoever provisioned it, and `terraform destroy` leaves it alone.
Flip either to `true` only for ephemeral / CI stacks where you accept
losing the contents.
@ -365,7 +420,7 @@ losing the contents.
| `examples/default/` | Thin root: `aws` provider (with an optional `default_tags` slot for org-wide tags) + a call to the module. The one-command deploy path. |
| `variables.tf` | All input variables |
| `locals.tf` | Path-prefix lists for ALB routing (mirror of `helm/.../ingress.yaml`) |
| `network.tf` | VPC, subnets, IGW, NAT, route tables, security groups |
| `network.tf` | VPC, subnets, IGW, NAT, route tables (all optional), security groups |
| `secrets.tf` | Secrets Manager entries + random passwords |
| `rds.tf` | Aurora Postgres cluster + writer / reader instances |
| `redis.tf` | ElastiCache Redis |

15
terraform/litellm/aws/alb.tf
View file

@ -3,10 +3,17 @@ resource "aws_lb" "this" {
load_balancer_type = "application"
internal = false
security_groups = [aws_security_group.alb.id]
subnets = aws_subnet.public[*].id
subnets = local.public_subnet_ids
idle_timeout = 120
lifecycle {
precondition {
condition = length(local.public_subnet_ids) >= 2
error_message = "The ALB needs at least 2 public subnets in different AZs. Set `public_subnet_ids` when using `vpc_id`, or list at least 2 `azs` when the module creates the VPC."
}
}
tags = local.tags
}
@ -25,7 +32,7 @@ resource "aws_lb_target_group" "gateway" {
port = 4000
protocol = "HTTP"
target_type = "ip"
vpc_id = aws_vpc.this.id
vpc_id = local.vpc_id
health_check {
path = "/health/readiness"
@ -46,7 +53,7 @@ resource "aws_lb_target_group" "backend" {
port = 4001
protocol = "HTTP"
target_type = "ip"
vpc_id = aws_vpc.this.id
vpc_id = local.vpc_id
health_check {
path = "/health/readiness"
@ -67,7 +74,7 @@ resource "aws_lb_target_group" "ui" {
port = 3000
protocol = "HTTP"
target_type = "ip"
vpc_id = aws_vpc.this.id
vpc_id = local.vpc_id
health_check {
path = "/healthz"

68
terraform/litellm/aws/bootstrap.tf
View file

@ -1,9 +1,12 @@
# Auto-runs the two manual steps that used to follow `terraform apply`:
#
# 1. Create the IAM-authed Postgres user (litellm_app) — uses the postgres:16
# image with the master password from Secrets Manager.
# image with the master password from Secrets Manager. Only relevant to
# the Aurora cluster this module creates, so it is skipped when
# create_database = false.
# 2. Run prisma migrate deploy — reuses the existing aws_ecs_task_definition
# .migrations task def from migrations.tf.
# .migrations task def from migrations.tf. Runs against an existing
# database too, and only disappears when there is no database at all.
#
# Both are invoked via `terraform_data` provisioners. Gateway/backend services
# in ecs.tf depend on `terraform_data.migration`, so on a fresh apply they
@ -23,13 +26,14 @@
# extras — see iam.tf). The DB master password lives in a separate secret used
# only here, so we grant access in an additive policy.
resource "aws_iam_policy" "bootstrap_secrets" {
name = "${local.name}-bootstrap-secrets-access"
count = var.create_database ? 1 : 0
name = "${local.name}-bootstrap-secrets-access"
policy = jsonencode({
Version = "2012-10-17"
Statement = [{
Effect = "Allow"
Action = ["secretsmanager:GetSecretValue"]
Resource = [aws_secretsmanager_secret.db_master_password.arn]
Resource = [aws_secretsmanager_secret.db_master_password[0].arn]
}]
})
@ -37,12 +41,14 @@ resource "aws_iam_policy" "bootstrap_secrets" {
}
resource "aws_iam_role_policy_attachment" "task_execution_bootstrap_secrets" {
count = var.create_database ? 1 : 0
role = aws_iam_role.task_execution.name
policy_arn = aws_iam_policy.bootstrap_secrets.arn
policy_arn = aws_iam_policy.bootstrap_secrets[0].arn
}
# ---------- Bootstrap task def ----------
resource "aws_cloudwatch_log_group" "bootstrap_db" {
count = var.create_database ? 1 : 0
name = "/ecs/${local.name}/bootstrap-db"
retention_in_days = var.log_retention_days
@ -68,6 +74,7 @@ locals {
}
resource "aws_ecs_task_definition" "bootstrap_db" {
count = var.create_database ? 1 : 0
family = "${local.name}-bootstrap-db"
network_mode = "awsvpc"
requires_compatibilities = ["FARGATE"]
@ -82,15 +89,15 @@ resource "aws_ecs_task_definition" "bootstrap_db" {
essential = true
environment = [
{ name = "PGHOST", value = aws_rds_cluster.this.endpoint },
{ name = "PGPORT", value = tostring(aws_rds_cluster.this.port) },
{ name = "PGHOST", value = aws_rds_cluster.this[0].endpoint },
{ name = "PGPORT", value = tostring(aws_rds_cluster.this[0].port) },
{ name = "PGUSER", value = var.db_master_username },
{ name = "PGDATABASE", value = var.db_name },
{ name = "BOOTSTRAP_SQL", value = local.bootstrap_sql },
]
secrets = [
# `:password::` extracts the password field out of the JSON secret.
{ name = "PGPASSWORD", valueFrom = "${aws_secretsmanager_secret.db_master_password.arn}:password::" },
{ name = "PGPASSWORD", valueFrom = "${aws_secretsmanager_secret.db_master_password[0].arn}:password::" },
]
entryPoint = ["sh", "-c"]
@ -99,7 +106,7 @@ resource "aws_ecs_task_definition" "bootstrap_db" {
logConfiguration = {
logDriver = "awslogs"
options = {
awslogs-group = aws_cloudwatch_log_group.bootstrap_db.name
awslogs-group = aws_cloudwatch_log_group.bootstrap_db[0].name
awslogs-region = var.region
awslogs-stream-prefix = "bootstrap"
}
@ -111,20 +118,22 @@ resource "aws_ecs_task_definition" "bootstrap_db" {
# ---------- Bootstrap trigger ----------
resource "terraform_data" "bootstrap_db" {
count = var.create_database ? 1 : 0
triggers_replace = {
cluster_resource_id = aws_rds_cluster.this.cluster_resource_id
task_def_revision = aws_ecs_task_definition.bootstrap_db.revision
cluster_resource_id = aws_rds_cluster.this[0].cluster_resource_id
task_def_revision = aws_ecs_task_definition.bootstrap_db[0].revision
}
provisioner "local-exec" {
interpreter = ["bash", "-c"]
environment = {
CLUSTER = aws_ecs_cluster.this.name
TASK_DEF = aws_ecs_task_definition.bootstrap_db.arn
SUBNETS = join(",", aws_subnet.private[*].id)
SG = aws_security_group.tasks.id
TASK_DEF = aws_ecs_task_definition.bootstrap_db[0].arn
SUBNETS = join(",", local.private_subnet_ids)
SG = join(",", local.task_security_group_ids)
REGION = var.region
LOG_GRP = aws_cloudwatch_log_group.bootstrap_db.name
LOG_GRP = aws_cloudwatch_log_group.bootstrap_db[0].name
}
command = <<-EOT
set -euo pipefail
@ -144,9 +153,13 @@ resource "terraform_data" "bootstrap_db" {
EOT
}
# Same secret-by-ARN gap as the migration below. The margin here is wide,
# since the writer instance takes minutes while the version write does not,
# but both hang off the cluster in parallel and nothing orders them.
depends_on = [
aws_rds_cluster_instance.writer,
aws_iam_role_policy_attachment.task_execution_bootstrap_secrets,
aws_secretsmanager_secret_version.db_master_password,
]
}
@ -154,20 +167,22 @@ resource "terraform_data" "bootstrap_db" {
# Reuses the task definition from migrations.tf — this resource just invokes
# it and waits.
resource "terraform_data" "migration" {
count = local.database_enabled ? 1 : 0
triggers_replace = {
task_def_revision = aws_ecs_task_definition.migrations.revision
bootstrap_id = terraform_data.bootstrap_db.id
task_def_revision = aws_ecs_task_definition.migrations[0].revision
bootstrap_id = join(",", terraform_data.bootstrap_db[*].id)
}
provisioner "local-exec" {
interpreter = ["bash", "-c"]
environment = {
CLUSTER = aws_ecs_cluster.this.name
TASK_DEF = aws_ecs_task_definition.migrations.arn
SUBNETS = join(",", aws_subnet.private[*].id)
SG = aws_security_group.tasks.id
TASK_DEF = aws_ecs_task_definition.migrations[0].arn
SUBNETS = join(",", local.private_subnet_ids)
SG = join(",", local.task_security_group_ids)
REGION = var.region
LOG_GRP = aws_cloudwatch_log_group.migrations.name
LOG_GRP = aws_cloudwatch_log_group.migrations[0].name
}
command = <<-EOT
set -euo pipefail
@ -187,5 +202,14 @@ resource "terraform_data" "migration" {
EOT
}
depends_on = [terraform_data.bootstrap_db]
# A container reads a secret by ARN, so Terraform sees no edge from the
# ARN to the _version that gives it a value. The managed-Aurora path hides
# that: the cluster create takes long enough that the version always lands
# first. A bring-your-own database has nothing slow in between, so without
# this the run-task below can fire against a valueless secret and fail the
# apply with ResourceInitializationError.
depends_on = [
terraform_data.bootstrap_db,
aws_secretsmanager_secret_version.database_url,
]
}

102
terraform/litellm/aws/ecs.tf
View file

@ -31,6 +31,7 @@ resource "aws_cloudwatch_log_group" "ui" {
}
resource "aws_cloudwatch_log_group" "migrations" {
count = local.database_enabled ? 1 : 0
name = "/ecs/${local.name}/migrations"
retention_in_days = var.log_retention_days
@ -38,11 +39,13 @@ resource "aws_cloudwatch_log_group" "migrations" {
}
# Shared env block fed to gateway, backend, and the migration task. Mirrors
# the helm chart's `litellm.serverEnv` helper on the IAM-auth branch:
# DATABASE_URL is assembled at runtime by
# the helm chart's `litellm.serverEnv` helper on the IAM-auth branch: for the
# module-created Aurora, DATABASE_URL is assembled at runtime by
# litellm/proxy/auth/rds_iam_token.py::init_iam_db_url_from_env from
# HOST/PORT/USER/NAME plus an IAM-signed token, so no DB password is needed
# in the task definition.
# in the task definition. An existing database instead arrives as a
# DATABASE_URL secret (var.database_url), which run.py and the proxy both
# take as-is.
locals {
# OTel v2 is opt-in and gated on otel_endpoint, matching the GCP stack.
# When set, LITELLM_OTEL_V2 flips on alongside the OTEL_* block, with
@ -103,29 +106,50 @@ locals {
] : [],
)
shared_env = [
managed_db_env = var.create_database ? [
{ name = "IAM_TOKEN_DB_AUTH", value = "true" },
{ name = "DATABASE_HOST", value = aws_rds_cluster.this.endpoint },
{ name = "DATABASE_PORT", value = tostring(aws_rds_cluster.this.port) },
{ name = "DATABASE_HOST", value = aws_rds_cluster.this[0].endpoint },
{ name = "DATABASE_PORT", value = tostring(aws_rds_cluster.this[0].port) },
{ name = "DATABASE_USER", value = var.db_username },
{ name = "DATABASE_NAME", value = var.db_name },
{ name = "DATABASE_HOST_READ_REPLICA", value = aws_rds_cluster.this.reader_endpoint },
{ name = "DATABASE_PORT_READ_REPLICA", value = tostring(aws_rds_cluster.this.port) },
{ name = "REDIS_HOST", value = aws_elasticache_replication_group.this.primary_endpoint_address },
{ name = "REDIS_PORT", value = tostring(aws_elasticache_replication_group.this.port) },
{ name = "DATABASE_HOST_READ_REPLICA", value = aws_rds_cluster.this[0].reader_endpoint },
{ name = "DATABASE_PORT_READ_REPLICA", value = tostring(aws_rds_cluster.this[0].port) },
] : []
managed_redis_env = var.create_redis ? [
{ name = "REDIS_HOST", value = aws_elasticache_replication_group.this[0].primary_endpoint_address },
{ name = "REDIS_PORT", value = tostring(aws_elasticache_replication_group.this[0].port) },
# transit_encryption_enabled = true on the replication group means the
# proxy must connect via rediss://. _redis.get_redis_url_from_environment
# honors REDIS_SSL to flip the scheme.
{ name = "REDIS_SSL", value = "true" },
# S3 bucket — referenced from proxy_config via os.environ/S3_BUCKET_NAME
# (e.g. cache backend, request log archival, /files passthrough).
{ name = "S3_BUCKET_NAME", value = aws_s3_bucket.this.bucket },
{ name = "S3_REGION_NAME", value = var.region },
# boto3 inside generate_iam_auth_token reads AWS_REGION_NAME first, then
# AWS_REGION. Set both for compatibility.
{ name = "AWS_REGION", value = var.region },
{ name = "AWS_REGION_NAME", value = var.region },
]
] : []
shared_env = concat(
local.managed_db_env,
local.managed_redis_env,
[
# S3 bucket — referenced from proxy_config via os.environ/S3_BUCKET_NAME
# (e.g. cache backend, request log archival, /files passthrough).
{ name = "S3_BUCKET_NAME", value = aws_s3_bucket.this.bucket },
{ name = "S3_REGION_NAME", value = var.region },
# boto3 inside generate_iam_auth_token reads AWS_REGION_NAME first, then
# AWS_REGION. Set both for compatibility.
{ name = "AWS_REGION", value = var.region },
{ name = "AWS_REGION_NAME", value = var.region },
],
)
# DATABASE_URL / REDIS_URL both outrank the discrete host/port vars in the
# proxy, so the BYO branch needs nothing removed from shared_env: the
# managed_*_env blocks are already empty whenever these are set.
byo_database_secrets = local.byo_database ? [
{ name = "DATABASE_URL", valueFrom = aws_secretsmanager_secret.database_url[0].arn },
] : []
byo_redis_secrets = local.byo_redis ? [
{ name = "REDIS_URL", valueFrom = aws_secretsmanager_secret.redis_url[0].arn },
] : []
shared_secrets = concat(
[
@ -134,6 +158,8 @@ locals {
var.litellm_license == "" ? [] : [
{ name = "LITELLM_LICENSE", valueFrom = aws_secretsmanager_secret.license[0].arn },
],
local.byo_database_secrets,
local.byo_redis_secrets,
local.otel_secrets,
local.billing_metrics_secrets,
)
@ -151,9 +177,11 @@ locals {
for k, v in var.backend_extra_env : { name = k, value = v }
]
backend_default_env = [
# Storing models in the DB needs a DB. Without one the backend reads its
# model list from proxy_config only.
backend_default_env = local.database_enabled ? [
{ name = "STORE_MODEL_IN_DB", value = "true" },
]
] : []
gateway_extra_secrets_list = [
for k, v in var.gateway_extra_secrets : { name = k, valueFrom = v }
]
@ -286,8 +314,8 @@ resource "aws_ecs_service" "gateway" {
launch_type = "FARGATE"
network_configuration {
subnets = aws_subnet.private[*].id
security_groups = [aws_security_group.tasks.id]
subnets = local.private_subnet_ids
security_groups = local.task_security_group_ids
assign_public_ip = false
}
@ -308,10 +336,20 @@ resource "aws_ecs_service" "gateway" {
# Don't start until the schema migration has run. Otherwise the proxy
# boots, Prisma fails on the missing tables, and ECS thrashes the task.
# The _version entries are listed because a task reads its secrets by ARN,
# which gives Terraform no edge to the resource that writes the value; the
# migration covers that ordering only while a database exists.
depends_on = [
aws_lb_listener.http,
aws_lb_listener.https,
terraform_data.migration,
aws_secretsmanager_secret_version.master_key,
aws_secretsmanager_secret_version.license,
aws_secretsmanager_secret_version.database_url,
aws_secretsmanager_secret_version.redis_url,
aws_secretsmanager_secret_version.billing_metrics_client_cert,
aws_secretsmanager_secret_version.billing_metrics_client_key,
aws_secretsmanager_secret_version.billing_metrics_ca_cert,
]
tags = local.tags
@ -381,8 +419,8 @@ resource "aws_ecs_service" "backend" {
launch_type = "FARGATE"
network_configuration {
subnets = aws_subnet.private[*].id
security_groups = [aws_security_group.tasks.id]
subnets = local.private_subnet_ids
security_groups = local.task_security_group_ids
assign_public_ip = false
}
@ -399,10 +437,20 @@ resource "aws_ecs_service" "backend" {
ignore_changes = [desired_count]
}
# Same secret-version ordering as the gateway, plus UI_PASSWORD, which only
# the backend consumes.
depends_on = [
aws_lb_listener.http,
aws_lb_listener.https,
terraform_data.migration,
aws_secretsmanager_secret_version.master_key,
aws_secretsmanager_secret_version.license,
aws_secretsmanager_secret_version.ui_password,
aws_secretsmanager_secret_version.database_url,
aws_secretsmanager_secret_version.redis_url,
aws_secretsmanager_secret_version.billing_metrics_client_cert,
aws_secretsmanager_secret_version.billing_metrics_client_key,
aws_secretsmanager_secret_version.billing_metrics_ca_cert,
]
tags = local.tags
@ -451,8 +499,8 @@ resource "aws_ecs_service" "ui" {
launch_type = "FARGATE"
network_configuration {
subnets = aws_subnet.private[*].id
security_groups = [aws_security_group.tasks.id]
subnets = local.private_subnet_ids
security_groups = local.task_security_group_ids
assign_public_ip = false
}

10
terraform/litellm/aws/examples/default/main.tf
View file

@ -24,6 +24,16 @@ module "litellm" {
env = var.env
azs = var.azs
vpc_id = var.vpc_id
public_subnet_ids = var.public_subnet_ids
private_subnet_ids = var.private_subnet_ids
additional_task_security_group_ids = var.additional_task_security_group_ids
create_database = var.create_database
database_url = var.database_url
create_redis = var.create_redis
redis_url = var.redis_url
litellm_master_key = var.litellm_master_key
litellm_license = var.litellm_license
ui_password = var.ui_password

10
terraform/litellm/aws/examples/default/outputs.tf
View file

@ -13,6 +13,16 @@ output "ecs_cluster" {
value = module.litellm.ecs_cluster
}
output "vpc_id" {
description = "VPC the stack runs in, whether module-created or supplied."
value = module.litellm.vpc_id
}
output "task_security_group_id" {
description = "Tasks security group. Allow this inbound on an existing database or Redis."
value = module.litellm.task_security_group_id
}
output "aurora_writer_endpoint" {
description = "Aurora writer endpoint."
value = module.litellm.aurora_writer_endpoint

32
terraform/litellm/aws/examples/default/terraform.tfvars.example
View file

@ -1,5 +1,35 @@
region = "us-west-2"
azs = ["us-west-2a", "us-west-2b"]
# Networking: by default the module creates a VPC, public/private subnets in
# each AZ listed here, an internet gateway, a NAT gateway, and route tables.
azs = ["us-west-2a", "us-west-2b"]
# To deploy into networking you already own, drop `azs` and set these
# instead. Nothing network-related is created then, so the private subnets
# need their own egress for LLM providers, image pulls, and Secrets Manager.
# vpc_id = "vpc-0123456789abcdef0"
# public_subnet_ids = ["subnet-aaa", "subnet-bbb"]
# private_subnet_ids = ["subnet-ccc", "subnet-ddd"]
#
# The tasks get their own security group either way. To reach a store that
# only allows a group you already have, attach it here as well; the
# `task_security_group_id` output names the module's own group.
# additional_task_security_group_ids = ["sg-0123456789abcdef0"]
# Data stores: Aurora Postgres and ElastiCache Redis are created by default.
# Set create_* = false to point at your own, passing a connection string
# (stored in Secrets Manager, injected as DATABASE_URL / REDIS_URL). Make
# sure they allow inbound from the stack's tasks security group, which the
# `task_security_group_id` output names.
# create_database = false
# database_url = "postgresql://litellm:...@db.internal:5432/litellm"
# create_redis = false
# redis_url = "rediss://:...@cache.internal:6379"
#
# Leaving the URL empty runs without that component: no database means no
# virtual keys, spend tracking, or UI persistence (master-key auth only), and
# no Redis means rate limits, budgets, and router cooldowns go per-task
# instead of cluster-wide.
# Resource naming: every AWS resource the stack creates is named
# `${tenant}-litellm-${env}` (or that plus a per-resource suffix). E.g.

58
terraform/litellm/aws/examples/default/variables.tf
View file

@ -21,8 +21,64 @@ variable "env" {
}
variable "azs" {
description = "Availability zones for subnets. At least 2 (RDS + ALB)."
description = "Availability zones for the subnets the module creates. At least 2 (RDS + ALB). Unused when vpc_id is set."
type = list(string)
default = []
}
# Bring-your-own networking. Leave vpc_id empty to have the module create the
# VPC, subnets, NAT gateway, and route tables.
variable "vpc_id" {
description = "Existing VPC to deploy into. Empty → module creates its own networking."
type = string
default = ""
}
variable "public_subnet_ids" {
description = "Existing public subnets for the ALB (≥ 2 AZs). Required with vpc_id."
type = list(string)
default = []
}
variable "private_subnet_ids" {
description = "Existing private subnets for tasks, Aurora, and Redis. Required with vpc_id."
type = list(string)
default = []
}
variable "additional_task_security_group_ids" {
description = "Extra security groups for the tasks, e.g. one an existing database already allows."
type = list(string)
default = []
}
# Bring-your-own data stores. create_* false with an empty URL runs without
# that component: no DB means no key management or spend tracking, no Redis
# means per-task rate limits instead of cluster-wide.
variable "create_database" {
description = "Create the Aurora Postgres cluster. False → use database_url, or run DB-less."
type = bool
default = true
}
variable "database_url" {
description = "Postgres connection string for an existing database. Read only when create_database = false."
type = string
default = ""
sensitive = true
}
variable "create_redis" {
description = "Create the ElastiCache Redis group. False → use redis_url, or run without Redis."
type = bool
default = true
}
variable "redis_url" {
description = "Connection string for an existing Redis. Read only when create_redis = false."
type = string
default = ""
sensitive = true
}
# Sensitive — prefer TF_VAR_litellm_master_key / TF_VAR_litellm_license /

15
terraform/litellm/aws/iam.tf
View file

@ -56,6 +56,8 @@ data "aws_iam_policy_document" "secrets_access" {
aws_secretsmanager_secret.billing_metrics_client_cert[*].arn,
aws_secretsmanager_secret.billing_metrics_client_key[*].arn,
aws_secretsmanager_secret.billing_metrics_ca_cert[*].arn,
aws_secretsmanager_secret.database_url[*].arn,
aws_secretsmanager_secret.redis_url[*].arn,
local.extra_secret_arns,
var.otel_headers_secret_arn == "" ? [] : [var.otel_headers_secret_arn],
)
@ -79,6 +81,9 @@ resource "aws_iam_role_policy_attachment" "task_execution_secrets" {
# Assumed by the running container. Gets `rds-db:connect` so the proxy can
# mint IAM-signed Postgres tokens for the app user. Layer additional
# policies here (e.g. Bedrock invoke, S3 read) when the proxy needs them.
# IAM auth only applies to the Aurora cluster this module creates: an
# existing database is reached with the credentials embedded in
# var.database_url, so the policy is skipped there.
resource "aws_iam_role" "task" {
name = "${local.name}-task"
@ -90,24 +95,28 @@ resource "aws_iam_role" "task" {
data "aws_caller_identity" "current" {}
data "aws_iam_policy_document" "rds_iam_connect" {
count = var.create_database ? 1 : 0
statement {
actions = ["rds-db:connect"]
resources = [
"arn:aws:rds-db:${var.region}:${data.aws_caller_identity.current.account_id}:dbuser:${aws_rds_cluster.this.cluster_resource_id}/${var.db_username}",
"arn:aws:rds-db:${var.region}:${data.aws_caller_identity.current.account_id}:dbuser:${aws_rds_cluster.this[0].cluster_resource_id}/${var.db_username}",
]
}
}
resource "aws_iam_policy" "rds_iam_connect" {
count = var.create_database ? 1 : 0
name = "${local.name}-rds-iam-connect"
policy = data.aws_iam_policy_document.rds_iam_connect.json
policy = data.aws_iam_policy_document.rds_iam_connect[0].json
tags = local.tags
}
resource "aws_iam_role_policy_attachment" "task_rds_iam_connect" {
count = var.create_database ? 1 : 0
role = aws_iam_role.task.name
policy_arn = aws_iam_policy.rds_iam_connect.arn
policy_arn = aws_iam_policy.rds_iam_connect[0].arn
}
# ---------- UI task role ----------

30
terraform/litellm/aws/locals.tf
View file

@ -25,6 +25,36 @@ locals {
var.tags,
)
# Networking, database, and cache are each either module-owned or
# bring-your-own. Everything downstream reads these locals rather than the
# resources, so a resource going to zero instances doesn't ripple.
create_vpc = var.vpc_id == ""
vpc_id = local.create_vpc ? aws_vpc.this[0].id : var.vpc_id
public_subnet_ids = local.create_vpc ? aws_subnet.public[*].id : var.public_subnet_ids
private_subnet_ids = local.create_vpc ? aws_subnet.private[*].id : var.private_subnet_ids
task_security_group_ids = concat([aws_security_group.tasks.id], var.additional_task_security_group_ids)
# `byo_*` is the existing-store branch, `database_enabled` is either branch.
# Neither branch means the component is absent: no DB (no key management,
# spend tracking, or UI persistence) or no Redis (per-task rate limits and
# cooldowns instead of cluster-wide).
# nonsensitive() on the emptiness check only: without it the sensitivity of
# the URLs propagates into every value derived from these flags, redacting
# unrelated task-definition and output diffs in the plan.
byo_database = !var.create_database && nonsensitive(var.database_url != "")
byo_redis = !var.create_redis && nonsensitive(var.redis_url != "")
database_enabled = var.create_database || local.byo_database
redis_enabled = var.create_redis || local.byo_redis
# Aurora and ElastiCache subnet groups both demand two AZs, so supplied
# private subnets have to cover two whenever either store is module-created.
managed_stores_need_two_azs = var.create_database || var.create_redis
# Every uvicorn worker in every gateway task counts its own rate limits when
# there is no Redis to share them through, so the ceiling is tasks x workers.
max_gateway_processes = (var.gateway_autoscaling_enabled ? var.gateway_max_capacity : var.gateway_desired_count) * var.gateway_num_workers
gateway_path_prefixes = [
"/v1/chat/*", "/chat/*",
"/v1/completions*", "/completions*",

4
terraform/litellm/aws/migrations.tf
View file

@ -13,6 +13,7 @@
# every apply (after the IAM-authed user has been created). The
# `migration_run_command` output is preserved for break-glass manual re-runs.
resource "aws_ecs_task_definition" "migrations" {
count = local.database_enabled ? 1 : 0
family = "${local.name}-migrations"
network_mode = "awsvpc"
requires_compatibilities = ["FARGATE"]
@ -32,11 +33,12 @@ resource "aws_ecs_task_definition" "migrations" {
# No entryPoint/command override — the image's ENTRYPOINT runs run.py.
environment = local.shared_env
secrets = local.byo_database_secrets
logConfiguration = {
logDriver = "awslogs"
options = {
awslogs-group = aws_cloudwatch_log_group.migrations.name
awslogs-group = aws_cloudwatch_log_group.migrations[0].name
awslogs-region = var.region
awslogs-stream-prefix = "migrations"
}

73
terraform/litellm/aws/network.tf
View file

@ -1,24 +1,34 @@
data "aws_availability_zones" "available" {
state = "available"
}
# Networking is created only when the caller didn't supply a VPC. With
# var.vpc_id set, every resource in this file except the security groups has
# zero instances and the stack consumes the caller's subnets through
# local.public_subnet_ids / local.private_subnet_ids (see locals.tf).
resource "aws_vpc" "this" {
count = local.create_vpc ? 1 : 0
cidr_block = var.vpc_cidr
enable_dns_hostnames = true
enable_dns_support = true
lifecycle {
precondition {
condition = length(var.azs) >= 2
error_message = "Provide at least 2 availability zones in `azs`, or set `vpc_id` + `public_subnet_ids` + `private_subnet_ids` to deploy into an existing VPC."
}
}
tags = merge(local.tags, { Name = local.name })
}
resource "aws_internet_gateway" "this" {
vpc_id = aws_vpc.this.id
count = local.create_vpc ? 1 : 0
vpc_id = aws_vpc.this[0].id
tags = merge(local.tags, { Name = local.name })
}
# Public subnets (ALB + NAT). One per AZ.
resource "aws_subnet" "public" {
count = length(var.azs)
vpc_id = aws_vpc.this.id
count = local.create_vpc ? length(var.azs) : 0
vpc_id = aws_vpc.this[0].id
cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index)
availability_zone = var.azs[count.index]
map_public_ip_on_launch = true
@ -29,8 +39,8 @@ resource "aws_subnet" "public" {
# Private subnets (ECS tasks, RDS, ElastiCache). One per AZ, separate from
# public range.
resource "aws_subnet" "private" {
count = length(var.azs)
vpc_id = aws_vpc.this.id
count = local.create_vpc ? length(var.azs) : 0
vpc_id = aws_vpc.this[0].id
cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index + 10)
availability_zone = var.azs[count.index]
@ -38,6 +48,7 @@ resource "aws_subnet" "private" {
}
resource "aws_eip" "nat" {
count = local.create_vpc ? 1 : 0
domain = "vpc"
tags = merge(local.tags, { Name = "${local.name}-nat" })
@ -47,7 +58,8 @@ resource "aws_eip" "nat" {
# Single NAT gateway in the first public subnet. For HA, replicate per AZ —
# adds ~$30/mo per gateway, so off by default for a baseline deployment.
resource "aws_nat_gateway" "this" {
allocation_id = aws_eip.nat.id
count = local.create_vpc ? 1 : 0
allocation_id = aws_eip.nat[0].id
subnet_id = aws_subnet.public[0].id
tags = merge(local.tags, { Name = local.name })
@ -56,45 +68,53 @@ resource "aws_nat_gateway" "this" {
}
resource "aws_route_table" "public" {
vpc_id = aws_vpc.this.id
count = local.create_vpc ? 1 : 0
vpc_id = aws_vpc.this[0].id
route {
cidr_block = "0.0.0.0/0"
gateway_id = aws_internet_gateway.this.id
gateway_id = aws_internet_gateway.this[0].id
}
tags = merge(local.tags, { Name = "${local.name}-public" })
}
resource "aws_route_table_association" "public" {
count = length(var.azs)
count = local.create_vpc ? length(var.azs) : 0
subnet_id = aws_subnet.public[count.index].id
route_table_id = aws_route_table.public.id
route_table_id = aws_route_table.public[0].id
}
resource "aws_route_table" "private" {
vpc_id = aws_vpc.this.id
count = local.create_vpc ? 1 : 0
vpc_id = aws_vpc.this[0].id
route {
cidr_block = "0.0.0.0/0"
nat_gateway_id = aws_nat_gateway.this.id
nat_gateway_id = aws_nat_gateway.this[0].id
}
tags = merge(local.tags, { Name = "${local.name}-private" })
}
resource "aws_route_table_association" "private" {
count = length(var.azs)
count = local.create_vpc ? length(var.azs) : 0
subnet_id = aws_subnet.private[count.index].id
route_table_id = aws_route_table.private.id
route_table_id = aws_route_table.private[0].id
}
# ---------- Security groups ----------
#
# Always module-owned, in local.vpc_id, so the stack keeps a least-privilege
# path between its own components even when it borrows someone else's VPC.
# Existing databases and caches reached over var.database_url / var.redis_url
# need to allow inbound from the tasks group (or from a group passed via
# var.additional_task_security_group_ids).
resource "aws_security_group" "alb" {
name = "${local.name}-alb"
description = "Inbound HTTP/HTTPS to the LiteLLM ALB."
vpc_id = aws_vpc.this.id
vpc_id = local.vpc_id
ingress {
description = "HTTP from anywhere"
@ -126,7 +146,7 @@ resource "aws_security_group" "alb" {
resource "aws_security_group" "tasks" {
name = "${local.name}-tasks"
description = "ECS tasks (gateway/backend/ui)."
vpc_id = aws_vpc.this.id
vpc_id = local.vpc_id
ingress {
description = "ALB to tasks"
@ -144,13 +164,23 @@ resource "aws_security_group" "tasks" {
cidr_blocks = ["0.0.0.0/0"]
}
# The tasks group is created in every mode, so this is where the
# bring-your-own-VPC inputs get checked.
lifecycle {
precondition {
condition = local.create_vpc || length(var.private_subnet_ids) >= (local.managed_stores_need_two_azs ? 2 : 1)
error_message = "`private_subnet_ids` is required when `vpc_id` is set: the tasks, Aurora, and ElastiCache all live in private subnets. Aurora and ElastiCache subnet groups need subnets in at least 2 AZs, so pass 2 unless both `create_database` and `create_redis` are false."
}
}
tags = local.tags
}
resource "aws_security_group" "rds" {
count = var.create_database ? 1 : 0
name = "${local.name}-rds"
description = "RDS Postgres - tasks only."
vpc_id = aws_vpc.this.id
vpc_id = local.vpc_id
ingress {
description = "Postgres from ECS tasks"
@ -164,9 +194,10 @@ resource "aws_security_group" "rds" {
}
resource "aws_security_group" "redis" {
count = var.create_redis ? 1 : 0
name = "${local.name}-redis"
description = "ElastiCache Redis - tasks only."
vpc_id = aws_vpc.this.id
vpc_id = local.vpc_id
ingress {
description = "Redis from ECS tasks"

42
terraform/litellm/aws/outputs.tf
View file

@ -13,19 +13,29 @@ output "ecs_cluster" {
value = aws_ecs_cluster.this.name
}
output "vpc_id" {
description = "VPC the stack runs in, whether module-created or passed in via `vpc_id`."
value = local.vpc_id
}
output "task_security_group_id" {
description = "Security group attached to the ECS tasks. Allow inbound from this group on an existing database or Redis reached over `database_url` / `redis_url`."
value = aws_security_group.tasks.id
}
output "aurora_writer_endpoint" {
description = "Aurora writer endpoint (cluster endpoint). Used by gateway/backend as DATABASE_HOST."
value = aws_rds_cluster.this.endpoint
description = "Aurora writer endpoint (cluster endpoint). Used by gateway/backend as DATABASE_HOST. Null when `create_database = false`."
value = one(aws_rds_cluster.this[*].endpoint)
}
output "aurora_reader_endpoint" {
description = "Aurora reader endpoint. Used by gateway/backend as DATABASE_HOST_READ_REPLICA."
value = aws_rds_cluster.this.reader_endpoint
description = "Aurora reader endpoint. Used by gateway/backend as DATABASE_HOST_READ_REPLICA. Null when `create_database = false`."
value = one(aws_rds_cluster.this[*].reader_endpoint)
}
output "redis_endpoint" {
description = "ElastiCache Redis primary endpoint (TLS, transit_encryption_enabled = true)."
value = "${aws_elasticache_replication_group.this.primary_endpoint_address}:${aws_elasticache_replication_group.this.port}"
description = "ElastiCache Redis primary endpoint (TLS, transit_encryption_enabled = true). Null when `create_redis = false`."
value = one([for r in aws_elasticache_replication_group.this : "${r.primary_endpoint_address}:${r.port}"])
}
output "s3_bucket" {
@ -39,15 +49,17 @@ output "master_key_secret_arn" {
}
output "db_master_password_secret_arn" {
description = "Secrets Manager ARN holding the Aurora master credentials (bootstrap-only). Used to create the IAM-authed application user."
value = aws_secretsmanager_secret.db_master_password.arn
description = "Secrets Manager ARN holding the Aurora master credentials (bootstrap-only). Used to create the IAM-authed application user. Null when `create_database = false`."
value = one(aws_secretsmanager_secret.db_master_password[*].arn)
}
# Pre-baked SQL to run once as the master user, creating the IAM-authed
# application user that gateway/backend/migration tasks will authenticate as.
# Irrelevant to an existing database reached over `database_url`, whose
# credentials are already in the URL.
output "db_bootstrap_sql" {
description = "Run this once as the master DB user (after the first apply) to create the IAM-authed app user."
value = <<-SQL
description = "Run this once as the master DB user (after the first apply) to create the IAM-authed app user. Empty when `create_database = false`."
value = !var.create_database ? "" : <<-SQL
CREATE USER ${var.db_username};
GRANT rds_iam TO ${var.db_username};
GRANT ALL PRIVILEGES ON DATABASE ${var.db_name} TO ${var.db_username};
@ -60,13 +72,13 @@ output "db_bootstrap_sql" {
# Pre-baked command for running the one-off migration task. ECS run-task
# needs the subnet + SG IDs at call time, so we render the full command.
output "migration_run_command" {
description = "Shell command that runs the one-off prisma migration task against Aurora. Run this once, after the bootstrap SQL above, before sending traffic."
value = format(
description = "Shell command that runs the one-off prisma migration task against the database. Run this once, after the bootstrap SQL above, before sending traffic. Empty when the stack has no database."
value = !local.database_enabled ? "" : format(
"aws ecs run-task --cluster %s --launch-type FARGATE --task-definition %s --network-configuration 'awsvpcConfiguration={subnets=[%s],securityGroups=[%s],assignPublicIp=DISABLED}' --region %s",
aws_ecs_cluster.this.name,
aws_ecs_task_definition.migrations.arn,
join(",", aws_subnet.private[*].id),
aws_security_group.tasks.id,
aws_ecs_task_definition.migrations[0].arn,
join(",", local.private_subnet_ids),
join(",", local.task_security_group_ids),
var.region,
)
}

31
terraform/litellm/aws/rds.tf
View file

@ -1,5 +1,7 @@
# Aurora Postgres cluster with one writer + one reader instance, IAM
# database authentication enabled.
# database authentication enabled. Skipped entirely when
# create_database = false, in which case the stack either talks to the
# database named by var.database_url or runs without one.
#
# Important: enabling IAM auth on the cluster does not by itself grant any
# Postgres user the ability to log in with an IAM token. After the first
@ -17,13 +19,15 @@
# superusers — keep it for break-glass only.
resource "aws_db_subnet_group" "this" {
count = var.create_database ? 1 : 0
name = "${local.name}-db"
subnet_ids = aws_subnet.private[*].id
subnet_ids = local.private_subnet_ids
tags = local.tags
}
resource "aws_rds_cluster_parameter_group" "this" {
count = var.create_database ? 1 : 0
name = "${local.name}-cluster-pg"
family = "aurora-postgresql${split(".", var.db_engine_version)[0]}"
description = "LiteLLM Aurora Postgres cluster parameters."
@ -32,16 +36,17 @@ resource "aws_rds_cluster_parameter_group" "this" {
}
resource "aws_rds_cluster" "this" {
count = var.create_database ? 1 : 0
cluster_identifier = local.name
engine = "aurora-postgresql"
engine_mode = "provisioned"
engine_version = var.db_engine_version
database_name = var.db_name
master_username = var.db_master_username
master_password = random_password.db_master_password.result
db_subnet_group_name = aws_db_subnet_group.this.name
vpc_security_group_ids = [aws_security_group.rds.id]
db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.this.name
master_password = random_password.db_master_password[0].result
db_subnet_group_name = aws_db_subnet_group.this[0].name
vpc_security_group_ids = [aws_security_group.rds[0].id]
db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.this[0].name
iam_database_authentication_enabled = true
storage_encrypted = true
@ -61,11 +66,12 @@ resource "aws_rds_cluster" "this" {
}
resource "aws_rds_cluster_instance" "writer" {
count = var.create_database ? 1 : 0
identifier = "${local.name}-writer"
cluster_identifier = aws_rds_cluster.this.id
cluster_identifier = aws_rds_cluster.this[0].id
instance_class = var.db_instance_class
engine = aws_rds_cluster.this.engine
engine_version = aws_rds_cluster.this.engine_version
engine = aws_rds_cluster.this[0].engine
engine_version = aws_rds_cluster.this[0].engine_version
publicly_accessible = false
performance_insights_enabled = true
@ -78,11 +84,12 @@ resource "aws_rds_cluster_instance" "writer" {
}
resource "aws_rds_cluster_instance" "reader" {
count = var.create_database ? 1 : 0
identifier = "${local.name}-reader"
cluster_identifier = aws_rds_cluster.this.id
cluster_identifier = aws_rds_cluster.this[0].id
instance_class = var.db_instance_class
engine = aws_rds_cluster.this.engine
engine_version = aws_rds_cluster.this.engine_version
engine = aws_rds_cluster.this[0].engine
engine_version = aws_rds_cluster.this[0].engine_version
publicly_accessible = false
performance_insights_enabled = true

20
terraform/litellm/aws/redis.tf
View file

@ -1,6 +1,7 @@
resource "aws_elasticache_subnet_group" "this" {
count = var.create_redis ? 1 : 0
name = "${local.name}-redis"
subnet_ids = aws_subnet.private[*].id
subnet_ids = local.private_subnet_ids
tags = local.tags
}
@ -13,6 +14,7 @@ resource "aws_elasticache_subnet_group" "this" {
# TLS-protected — the proxy connects via the rediss:// scheme thanks to
# REDIS_SSL=true in the shared task env (see ecs.tf).
resource "aws_elasticache_replication_group" "this" {
count = var.create_redis ? 1 : 0
replication_group_id = "${local.name}-redis"
description = "LiteLLM ElastiCache Redis"
@ -23,8 +25,8 @@ resource "aws_elasticache_replication_group" "this" {
parameter_group_name = "default.redis7"
port = 6379
subnet_group_name = aws_elasticache_subnet_group.this.name
security_group_ids = [aws_security_group.redis.id]
subnet_group_name = aws_elasticache_subnet_group.this[0].name
security_group_ids = [aws_security_group.redis[0].id]
automatic_failover_enabled = var.redis_num_replicas >= 1
multi_az_enabled = var.redis_num_replicas >= 1
@ -35,3 +37,15 @@ resource "aws_elasticache_replication_group" "this" {
tags = local.tags
}
# Rate limits, budgets, and router cooldowns are shared through Redis. Without
# it each gateway process counts on its own, so a caller spread across tasks
# collects the full per-key allowance from every one of them. A `check` rather
# than a precondition: running without Redis is a legitimate choice when you do
# not rely on per-key limits, so this warns instead of blocking the plan.
check "redis_less_rate_limits_are_per_process" {
assert {
condition = local.redis_enabled || local.max_gateway_processes <= 1
error_message = "No Redis is configured while the gateway can run up to ${local.max_gateway_processes} processes, so per-key RPM/TPM limits, budgets, and cooldowns apply per process and a caller can multiply them across tasks. Set `create_redis = true`, pass `redis_url`, or hold the gateway to one process (`gateway_autoscaling_enabled = false`, `gateway_desired_count = 1`, `gateway_num_workers = 1`)."
}
}

48
terraform/litellm/aws/secrets.tf
View file

@ -10,6 +10,7 @@ resource "random_password" "master_key" {
# user (see rds.tf header). Runtime services authenticate via IAM tokens
# and never read this secret.
resource "random_password" "db_master_password" {
count = var.create_database ? 1 : 0
length = 32
special = false
min_lower = 4
@ -130,6 +131,7 @@ resource "aws_secretsmanager_secret_version" "billing_metrics_ca_cert" {
}
resource "aws_secretsmanager_secret" "db_master_password" {
count = var.create_database ? 1 : 0
name = "${local.name}-db-master-password"
description = "Aurora master-user password - bootstrap only. Runtime auth is IAM-token."
recovery_window_in_days = 0
@ -138,12 +140,50 @@ resource "aws_secretsmanager_secret" "db_master_password" {
}
resource "aws_secretsmanager_secret_version" "db_master_password" {
secret_id = aws_secretsmanager_secret.db_master_password.id
count = var.create_database ? 1 : 0
secret_id = aws_secretsmanager_secret.db_master_password[0].id
secret_string = jsonencode({
username = var.db_master_username
password = random_password.db_master_password.result
host = aws_rds_cluster.this.endpoint
port = aws_rds_cluster.this.port
password = random_password.db_master_password[0].result
host = aws_rds_cluster.this[0].endpoint
port = aws_rds_cluster.this[0].port
dbname = var.db_name
})
}
# Bring-your-own connection strings. Both hold credentials, so they go to
# Secrets Manager and reach the containers as ECS `secrets` rather than as
# plain-text env in the task definition.
resource "aws_secretsmanager_secret" "database_url" {
count = local.byo_database ? 1 : 0
name = "${local.name}-database-url"
description = "DATABASE_URL for an existing Postgres, used when create_database = false."
recovery_window_in_days = 0
tags = local.tags
}
resource "aws_secretsmanager_secret_version" "database_url" {
count = local.byo_database ? 1 : 0
secret_id = aws_secretsmanager_secret.database_url[0].id
secret_string = var.database_url
}
resource "aws_secretsmanager_secret" "redis_url" {
count = local.byo_redis ? 1 : 0
name = "${local.name}-redis-url"
description = "REDIS_URL for an existing Redis, used when create_redis = false."
recovery_window_in_days = 0
tags = local.tags
}
resource "aws_secretsmanager_secret_version" "redis_url" {
count = local.byo_redis ? 1 : 0
secret_id = aws_secretsmanager_secret.redis_url[0].id
secret_string = var.redis_url
}

272
terraform/litellm/aws/tests/byo_infrastructure.tftest.hcl Normal file
View file

@ -0,0 +1,272 @@
# Plan-only coverage for the four networking/database/cache permutations.
# `mock_provider` keeps this offline: no AWS credentials, no API calls, no
# resources. Run from terraform/litellm/aws with `terraform test`.
mock_provider "aws" {
# IAM policy documents are validated as JSON by the provider, so the
# generated placeholder string has to be replaced with a parsable one.
mock_data "aws_iam_policy_document" {
defaults = {
json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"
}
}
}
mock_provider "random" {}
variables {
region = "us-east-1"
tenant = "acme"
env = "test"
allow_plaintext_alb = true
}
run "module_owns_everything_by_default" {
command = plan
variables {
azs = ["us-east-1a", "us-east-1b"]
}
assert {
condition = length(aws_vpc.this) == 1 && length(aws_nat_gateway.this) == 1 && length(aws_subnet.private) == 2
error_message = "The default path must still create its own VPC, NAT gateway, and one private subnet per AZ."
}
assert {
condition = length(aws_rds_cluster.this) == 1 && length(aws_elasticache_replication_group.this) == 1
error_message = "The default path must still create Aurora and ElastiCache."
}
assert {
condition = length(aws_secretsmanager_secret.database_url) == 0 && length(aws_secretsmanager_secret.redis_url) == 0
error_message = "Connection-string secrets belong to the bring-your-own path only."
}
assert {
condition = length(local.managed_db_env) == 7 && length(local.managed_redis_env) == 3
error_message = "Gateway, backend, and migration tasks must keep the discrete DATABASE_*/REDIS_* env for the module-created stores."
}
assert {
condition = length(terraform_data.bootstrap_db) == 1 && length(aws_ecs_task_definition.migrations) == 1
error_message = "The IAM-user bootstrap and the schema migration must both run against the module-created Aurora."
}
}
run "existing_vpc_creates_no_networking" {
command = plan
variables {
vpc_id = "vpc-00000000000000001"
public_subnet_ids = ["subnet-pub-a", "subnet-pub-b"]
private_subnet_ids = ["subnet-priv-a", "subnet-priv-b"]
additional_task_security_group_ids = ["sg-caller-owned"]
}
assert {
condition = alltrue([
length(aws_vpc.this) == 0,
length(aws_subnet.public) == 0,
length(aws_subnet.private) == 0,
length(aws_internet_gateway.this) == 0,
length(aws_nat_gateway.this) == 0,
length(aws_eip.nat) == 0,
length(aws_route_table.public) == 0,
length(aws_route_table.private) == 0,
])
error_message = "An existing vpc_id must suppress every network resource, including the route tables and NAT gateway."
}
assert {
condition = aws_lb.this.subnets == toset(var.public_subnet_ids)
error_message = "The ALB must land in the caller's public subnets."
}
assert {
condition = alltrue([
aws_db_subnet_group.this[0].subnet_ids == toset(var.private_subnet_ids),
aws_elasticache_subnet_group.this[0].subnet_ids == toset(var.private_subnet_ids),
aws_ecs_service.gateway.network_configuration[0].subnets == toset(var.private_subnet_ids),
])
error_message = "Tasks, Aurora, and ElastiCache must land in the caller's private subnets."
}
assert {
condition = length(local.task_security_group_ids) == 2
error_message = "additional_task_security_group_ids must be attached alongside the module's own tasks group."
}
}
run "existing_database_and_redis_replace_the_managed_ones" {
command = plan
variables {
azs = ["us-east-1a", "us-east-1b"]
create_database = false
database_url = "postgresql://litellm:pw@db.internal:5432/litellm"
create_redis = false
redis_url = "rediss://:pw@cache.internal:6379"
}
assert {
condition = alltrue([
length(aws_rds_cluster.this) == 0,
length(aws_rds_cluster_instance.writer) == 0,
length(aws_db_subnet_group.this) == 0,
length(aws_security_group.rds) == 0,
length(aws_elasticache_replication_group.this) == 0,
length(aws_elasticache_subnet_group.this) == 0,
length(aws_security_group.redis) == 0,
])
error_message = "Pointing at an existing database and cache must create neither Aurora nor ElastiCache."
}
assert {
condition = length(local.managed_db_env) == 0 && length(local.managed_redis_env) == 0
error_message = "The discrete DATABASE_*/REDIS_* env vars must be dropped so DATABASE_URL/REDIS_URL are the only connection targets."
}
assert {
condition = alltrue([
length([for s in local.shared_secrets : s if s.name == "DATABASE_URL"]) == 1,
length([for s in local.shared_secrets : s if s.name == "REDIS_URL"]) == 1,
])
error_message = "Both connection strings must reach the containers as Secrets Manager references, not plain-text env."
}
assert {
condition = length(terraform_data.bootstrap_db) == 0 && length(aws_ecs_task_definition.migrations) == 1
error_message = "An existing database still needs the schema migration, but not the Aurora IAM-user bootstrap."
}
assert {
condition = length([for e in local.backend_default_env : e if e.name == "STORE_MODEL_IN_DB"]) == 1
error_message = "STORE_MODEL_IN_DB must stay set when a database is reachable."
}
}
run "vpc_without_subnets_fails_at_plan" {
command = plan
variables {
vpc_id = "vpc-00000000000000001"
}
expect_failures = [
aws_lb.this,
aws_security_group.tasks,
]
}
run "neither_vpc_nor_azs_fails_at_plan" {
command = plan
expect_failures = [
aws_vpc.this,
]
}
# Aurora and ElastiCache subnet groups need two AZs, so one private subnet is
# only enough when neither store is module-created.
run "one_private_subnet_fails_while_a_managed_store_needs_two_azs" {
command = plan
variables {
vpc_id = "vpc-00000000000000001"
public_subnet_ids = ["subnet-pub-a", "subnet-pub-b"]
private_subnet_ids = ["subnet-priv-a"]
}
expect_failures = [
aws_security_group.tasks,
]
}
run "one_private_subnet_is_enough_without_managed_stores" {
command = plan
variables {
vpc_id = "vpc-00000000000000001"
public_subnet_ids = ["subnet-pub-a", "subnet-pub-b"]
private_subnet_ids = ["subnet-priv-a"]
create_database = false
create_redis = false
# Single process, so the Redis-less rate-limit check stays quiet and this
# run is only exercising the subnet rule.
gateway_autoscaling_enabled = false
gateway_desired_count = 1
gateway_num_workers = 1
}
assert {
condition = length(aws_security_group.tasks.vpc_id) > 0
error_message = "With no module-created database or cache, a single private subnet must plan cleanly."
}
}
# The default sizing is 10 tasks under autoscaling, so a Redis-less stack must
# warn that per-key limits are counted per process.
run "redis_less_multi_process_gateway_is_flagged" {
command = plan
variables {
azs = ["us-east-1a", "us-east-1b"]
create_redis = false
}
expect_failures = [
check.redis_less_rate_limits_are_per_process,
]
}
run "redis_less_single_process_gateway_is_not_flagged" {
command = plan
variables {
azs = ["us-east-1a", "us-east-1b"]
create_redis = false
gateway_autoscaling_enabled = false
gateway_desired_count = 1
gateway_num_workers = 1
}
assert {
condition = local.max_gateway_processes == 1
error_message = "One task with one worker is a single process, which is the supported way to run without Redis."
}
}
run "no_database_and_no_redis_drops_the_schema_migration" {
command = plan
variables {
azs = ["us-east-1a", "us-east-1b"]
create_database = false
create_redis = false
# Single process, so the Redis-less rate-limit check stays quiet here; it
# has its own run above.
gateway_autoscaling_enabled = false
gateway_desired_count = 1
gateway_num_workers = 1
}
assert {
condition = alltrue([
length(aws_ecs_task_definition.migrations) == 0,
length(terraform_data.migration) == 0,
length(aws_iam_policy.rds_iam_connect) == 0,
length(aws_secretsmanager_secret.db_master_password) == 0,
])
error_message = "With no database at all there is nothing to migrate, bootstrap, or grant rds-db:connect on."
}
assert {
condition = length(local.backend_default_env) == 0
error_message = "STORE_MODEL_IN_DB must not be set without a database to store models in."
}
assert {
condition = length(local.shared_env) == 4
error_message = "The shared env must narrow to the S3 bucket and region pair when both data stores are gone."
}
}

108
terraform/litellm/aws/variables.tf
View file

@ -74,20 +74,63 @@ variable "ui_password" {
}
# ---------- Networking ----------
#
# Two modes:
#
# 1. Module-owned (default, `vpc_id = ""`): the stack creates a VPC, public
# and private subnets per AZ, an internet gateway, a NAT gateway, and
# the route tables wiring them together. `vpc_cidr` + `azs` drive it.
# 2. Bring-your-own (`vpc_id` set): the stack creates no networking and
# places the ALB in `public_subnet_ids` and every task, plus the Aurora
# and ElastiCache subnet groups, in `private_subnet_ids`. `vpc_cidr` and
# `azs` are then unused.
variable "vpc_id" {
description = <<-EOT
Existing VPC to deploy into. Leave empty ("") to have the module create
its own VPC, subnets, NAT gateway, and route tables. When set,
`public_subnet_ids` and `private_subnet_ids` are required and no
networking is created: the private subnets must already have egress
(NAT gateway or equivalent) so tasks can reach LLM providers, ECR/GHCR,
and Secrets Manager.
EOT
type = string
default = ""
}
variable "public_subnet_ids" {
description = "Existing public subnets for the ALB, in at least 2 AZs. Required when `vpc_id` is set, ignored otherwise."
type = list(string)
default = []
}
variable "private_subnet_ids" {
description = "Existing private subnets for the ECS tasks, Aurora, and ElastiCache. Required when `vpc_id` is set, ignored otherwise."
type = list(string)
default = []
}
variable "additional_task_security_group_ids" {
description = <<-EOT
Extra security groups to attach to the ECS tasks, on top of the one the
module creates. Useful with `vpc_id`: attach a group your existing
database or cache already allows inbound from, instead of editing their
ingress rules.
EOT
type = list(string)
default = []
}
variable "vpc_cidr" {
description = "CIDR block for the VPC."
description = "CIDR block for the VPC the module creates. Unused when `vpc_id` is set."
type = string
default = "10.40.0.0/16"
}
variable "azs" {
description = "Availability zones to spread subnets across. At least 2 required for RDS and ALB."
description = "Availability zones to spread the module-created subnets across. At least 2 required for Aurora and the ALB. Unused when `vpc_id` is set."
type = list(string)
validation {
condition = length(var.azs) >= 2
error_message = "Provide at least 2 availability zones."
}
default = []
}
# ---------- Component images ----------
@ -279,6 +322,34 @@ variable "ui_cpu_target" {
# ---------- RDS ----------
variable "create_database" {
description = <<-EOT
Create the Aurora Postgres cluster (default). Set false to skip it and
either point the stack at an existing database via `database_url`, or
run without a database at all when `database_url` is also empty. The
DB-less mode drops key management, spend tracking, and the admin UI's
persistence: the proxy then serves traffic authenticated by
LITELLM_MASTER_KEY only.
EOT
type = bool
default = true
}
variable "database_url" {
description = <<-EOT
Postgres connection string for an existing database, e.g.
`postgresql://user:pass@host:5432/litellm`. Only read when
`create_database = false`. Stored in a
`<tenant>-litellm-<env>-database-url` Secrets Manager entry and injected
into gateway, backend, and the migration task as DATABASE_URL, so the
value never lands in a task definition. The schema migration still runs
against it on every apply.
EOT
type = string
default = ""
sensitive = true
}
variable "db_instance_class" {
description = "Aurora instance class for both writer and reader."
type = string
@ -311,6 +382,31 @@ variable "db_username" {
# ---------- Redis ----------
variable "create_redis" {
description = <<-EOT
Create the ElastiCache Redis replication group (default). Set false to
skip it and either point the stack at an existing cache via `redis_url`,
or run with no Redis at all when `redis_url` is also empty. Without
Redis the proxy loses cross-task state: rate limits, budgets, and the
router's cooldowns become per-task instead of cluster-wide.
EOT
type = bool
default = true
}
variable "redis_url" {
description = <<-EOT
Connection string for an existing Redis, e.g.
`rediss://:password@host:6379`. Only read when `create_redis = false`.
Stored in a `<tenant>-litellm-<env>-redis-url` Secrets Manager entry and
injected as REDIS_URL, which takes precedence over REDIS_HOST/REDIS_PORT
in the proxy.
EOT
type = string
default = ""
sensitive = true
}
variable "redis_node_type" {
description = "ElastiCache node type."
type = string

0
tests/e2e/claude_code/_builder_unit_tests/__init__.py Normal file
View file

148
tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py Normal file
View file

@ -0,0 +1,148 @@
"""Unit tests for `find_regressions`, the green→red detector that gates
auto-merge on the daily compat-matrix docs PR (see `cron_vm/`).
Markerless harness tests: they exercise publisher plumbing, not a product
feature, so they run without a proxy and carry no `e2e` marker.
"""
from __future__ import annotations
from typing import Mapping, Union
from claude_code.matrix_builder import find_regressions
_CellSpec = Union[str, Mapping[str, str]]
def _matrix(
cells: Mapping[tuple[str, str], _CellSpec],
*,
names: Mapping[str, str] | None = None,
) -> dict[str, object]:
"""Build a minimal matrix dict from a {(feature_id, provider): status}
or {(feature_id, provider): cell_dict} mapping."""
names = names or {}
features: dict[str, dict[str, dict[str, str]]] = {}
for (feature_id, provider), value in cells.items():
cell = {"status": value} if isinstance(value, str) else dict(value)
features.setdefault(feature_id, {})[provider] = cell
return {
"features": [
{
"id": feature_id,
"name": names.get(feature_id, feature_id.upper()),
"providers": providers,
}
for feature_id, providers in features.items()
]
}
def test_find_regressions_flags_pass_to_fail() -> None:
old = _matrix({("vision", "anthropic"): "pass"})
new = _matrix(
{("vision", "anthropic"): {"status": "fail", "error": "credit balance too low"}}
)
regressions = find_regressions(old, new)
assert len(regressions) == 1
r = regressions[0]
assert r["feature_id"] == "vision"
assert r["provider"] == "anthropic"
assert r["old_status"] == "pass"
assert r["new_status"] == "fail"
assert r["error"] == "credit balance too low"
def test_find_regressions_ignores_red_to_red() -> None:
"""An already-failing cell that stays failing is NOT a regression — a
provider that's independently broken (e.g. out of credits) must not
block the daily auto-merge forever."""
old = _matrix({("vision", "anthropic"): "fail"})
new = _matrix({("vision", "anthropic"): "fail"})
assert find_regressions(old, new) == []
def test_find_regressions_ignores_improvements_and_steady_green() -> None:
old = _matrix(
{
("vision", "anthropic"): "fail", # red -> green
("tool_use", "azure"): "pass", # green -> green
}
)
new = _matrix(
{
("vision", "anthropic"): "pass",
("tool_use", "azure"): "pass",
}
)
assert find_regressions(old, new) == []
def test_find_regressions_ignores_green_to_grey() -> None:
"""green→not_tested / green→not_applicable are degradations but not
*red* regressions; we deliberately don't block on them."""
old = _matrix(
{
("vision", "azure"): "pass",
("tool_use", "azure"): "pass",
}
)
new = _matrix(
{
("vision", "azure"): "not_tested",
("tool_use", "azure"): {"status": "not_applicable", "reason": "skip"},
}
)
assert find_regressions(old, new) == []
def test_find_regressions_ignores_new_cells_without_baseline() -> None:
"""A cell only present in the new matrix (new feature/provider) has no
baseline, so a fail there can't be a regression."""
old = _matrix({("vision", "anthropic"): "pass"})
new = _matrix(
{
("vision", "anthropic"): "pass",
("brand_new_feature", "anthropic"): "fail",
}
)
assert find_regressions(old, new) == []
def test_find_regressions_matches_by_id_not_name() -> None:
"""Renaming a feature's display name must not hide a regression: cells
are matched on the stable id."""
old = _matrix({("thinking", "anthropic"): "pass"}, names={"thinking": "Old Name"})
new = _matrix(
{("thinking", "anthropic"): "fail"}, names={"thinking": "Totally New Name"}
)
regressions = find_regressions(old, new)
assert len(regressions) == 1
assert regressions[0]["feature_id"] == "thinking"
assert regressions[0]["feature_name"] == "Totally New Name"
def test_find_regressions_reports_multiple_sorted() -> None:
old = _matrix(
{
("vision", "anthropic"): "pass",
("tool_use", "anthropic"): "pass",
("vision", "azure"): "pass",
}
)
new = _matrix(
{
("vision", "anthropic"): "fail",
("tool_use", "anthropic"): "fail",
("vision", "azure"): "pass", # stays green
}
)
regressions = find_regressions(old, new)
keys = [(r["feature_id"], r["provider"]) for r in regressions]
assert keys == [("tool_use", "anthropic"), ("vision", "anthropic")]
def test_find_regressions_empty_old_matrix_is_safe() -> None:
"""No baseline at all (first publish) yields no regressions."""
new = _matrix({("vision", "anthropic"): "fail"})
assert find_regressions({}, new) == []

195
tests/e2e/claude_code/cron_vm/README.md Normal file
View file

@ -0,0 +1,195 @@
# Cron VM setup for the Claude Code compatibility-matrix populator
The populator runs daily on a dedicated GCP VM
(`litellm-compatibility-matrix-populator`) rather than as a GitHub
Action. Trade-offs:
- ✅ Real VM means we can `gh auth login` against an account that's
already a collaborator on `BerriAI/litellm-docs`, instead of
provisioning a GitHub App with `pull-requests: write`.
- ✅ Persistent state (a single `~/litellm-cron-worktree/` and its `.venv`)
is reused across runs, so each daily run does a fast `git checkout` +
incremental `uv sync` rather than a fresh clone + cold sync.
- ✅ No Docker dependency — the proxy runs directly via `uv run litellm`.
- ⚠️ The VM has to actually be on. systemd's `Persistent=true` recovers
from short outages, but a multi-day outage means the matrix goes
stale until the VM is back.
- ⚠️ Provider credentials live on the VM filesystem
(`/etc/litellm-compat-matrix.env`) instead of GitHub secrets. Treat
the VM as an environment with comparable blast radius to a CI runner.
This directory used to live at `tests/claude_code/cron_vm/` (paired with
the standalone `tests/claude_code/` suite); it now runs the maintained
`tests/e2e/claude_code/` suite instead. The pytest env interface changed
accordingly: the runner exports `LITELLM_PROXY_URL` / `LITELLM_MASTER_KEY`
(previously `LITELLM_PROXY_BASE_URL` / `LITELLM_PROXY_API_KEY`), the azure
column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously
`AZURE_FOUNDRY_*`), and the GPT columns need `OPENAI_API_KEY` and
`AZURE_API_BASE` / `AZURE_API_KEY` — see `litellm-compat-matrix.env.example`.
## Layout
| File | Purpose |
| --- | --- |
| `run_daily.sh` | The actual cron job. Resolves versions, updates the worktree, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR, sweeps stale compat-matrix PRs. |
| `build_matrix.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.build_from_paths`. Exists only because the bash script needs *some* way to render the per-cell aggregation, and the builder is already Python. |
| `check_regressions.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.find_regressions`. Diffs the freshly built matrix against the currently-published one and exits `3` if any cell flipped green→red, which gates auto-merge. |
| `litellm-compat-matrix.service` | systemd oneshot that invokes `run_daily.sh`. |
| `litellm-compat-matrix.timer` | `OnCalendar=*-*-* 06:00:00 UTC`, `Persistent=true`. |
| `litellm-compat-matrix.env.example` | Template for `/etc/litellm-compat-matrix.env`. |
## What `run_daily.sh` does
1. **Resolves the latest LiteLLM final release tag** (newest bare
`vX.Y.Z`, skipping `-rc.N`/`-dev.N` pre-releases) by paging the
GitHub Releases API (`curl | jq`).
2. **Reads the local Claude Code CLI version** via `claude --version`.
The cron does not auto-upgrade the CLI — operators do that
out-of-band by running `npm install -g @anthropic-ai/claude-code@latest`.
3. **Updates the persistent worktree** at `~/litellm-cron-worktree/`:
`git fetch --tags --force`, `git reset --hard`,
`git clean -fdx -e .venv -e .uv-bin`, `git checkout --force <tag>`.
The `.venv` is preserved across runs so `uv sync --frozen` is
incremental. Then **shims the test suite**: `tests/e2e/` in the
worktree is rebuilt from the dev checkout — the `claude_code/` suite
plus the five shared transport helpers it imports (`proxy_client.py`,
`e2e_http.py`, `models.py`, `e2e_config.py`, `transport.py`) — so the
cron always runs *today's* tests against the latest stable proxy. The
tag's own `tests/e2e/` tree (including the EKS-harness `conftest.py`,
whose imports the stable venv doesn't install) is deliberately not
used.
4. **Boots the proxy** as a `setsid` background process on port `4100`
(so it can't collide with a developer's `:4000`), then polls
`/health/liveliness` until it's up.
5. **Runs pytest** on `tests/e2e/claude_code/` with `LITELLM_PROXY_URL`
pointed at the proxy and `COMPAT_RESULTS_PATH` set so the conftest
hook writes the per-test results artifact. Test failures become
`fail` cells in the JSON, not script errors.
6. **Builds `compatibility-matrix.json`** by handing the artifact +
manifest to `build_matrix.py`.
7. **Opens or updates a docs PR**: `gh repo clone` of `litellm-docs`
into a tempdir, deterministic head branch
(`compat-matrix/<litellm-version>-<claude-code-version>-<UTC-date>`),
`--force` push **directly to `BerriAI/litellm-docs`** (the
`mateo-berri` token has write access, so this is a same-repo branch,
not a fork), `gh pr create`. A re-run on the same day fast-forwards
the existing branch and `gh pr create` no-ops ("a pull request for
branch ... already exists" is treated as success). These PRs are no
longer gated on a second human review.
8. **Gates auto-merge on a regression check**: before enabling
auto-merge, `check_regressions.py` diffs the new matrix against the
one currently on `main`. Auto-merge (`gh pr merge --auto --squash`)
is only enabled when **no cell flipped green→red** — i.e. every
transition is red→green, green→green, or red→red. A pre-existing red
cell (e.g. a provider that's out of API credits) is `red→red` and
does **not** block; only a `pass`→`fail` flip does. When a regression
is detected the PR is still opened/updated (with a warning banner
naming the offending cells) but auto-merge is left **off** — and any
auto-merge a prior same-day run enabled is explicitly disabled — so a
human reviews before it lands on the public table. The check fails
*closed*: if it errors, auto-merge is withheld.
9. **Sweeps stale compat-matrix PRs**: once today's PR exists, every
other open `compat-matrix/*` PR on the docs repo is closed (and its
bot-owned branch deleted), so at most one compat-matrix PR is ever
open — the newest.
## One-time VM setup
Run as `mateo` on the cron VM:
```bash
# 1. Toolchain
sudo apt-get update
sudo apt-get install -y git nodejs npm jq curl
curl -LsSf https://astral.sh/uv/install.sh | sh
sudo apt-get install -y gh # or follow https://cli.github.com/
# 2. Claude Code CLI (the cron does NOT auto-upgrade this; rerun this
# line out-of-band when you want a fresh CLI to be tested)
sudo npm install -g @anthropic-ai/claude-code@latest
# 3. Litellm checkout. Used by systemd's WorkingDirectory and as the
# source of the .service / .timer files. The cron itself runs out
# of the separate worktree at ~/litellm-cron-worktree/.
mkdir -p ~/litellm
git clone https://github.com/BerriAI/litellm.git ~/litellm/litellm
git -C ~/litellm/litellm checkout litellm_internal_staging
# 4. gh auth — must be a collaborator on BerriAI/litellm-docs.
gh auth login # follow prompts; pick HTTPS + token paste flow
# 5. Provider credentials + the publish token.
sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example \
/etc/litellm-compat-matrix.env
sudoedit /etc/litellm-compat-matrix.env # fill in real values
sudo chmod 0600 /etc/litellm-compat-matrix.env
# The mateo-berri PAT lives in its own file, mapped into the service via
# systemd LoadCredential so it stays out of the test processes' env
# (see the env.example comment for why).
sudo install -m 0600 /dev/null /etc/litellm-compat-matrix-github-token
sudoedit /etc/litellm-compat-matrix-github-token # single line: the PAT
# 6. systemd units.
sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/
sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now litellm-compat-matrix.timer
```
## Operating it
```bash
# When does it run next?
systemctl list-timers litellm-compat-matrix.timer
# Trigger a real run right now (PRs to litellm-docs).
sudo systemctl start litellm-compat-matrix.service
# Trigger a run that does NOT open a PR (good for first-time validation).
SKIP_PUBLISH=1 ~/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh
# Narrow to one cell while debugging.
SKIP_PUBLISH=1 PYTEST_K='basic_messaging_non_streaming and anthropic' \
~/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh
# Watch the most recent run.
journalctl -u litellm-compat-matrix.service -f
# Read older runs.
journalctl -u litellm-compat-matrix.service --since '2 days ago'
# Disable until further notice (e.g. while debugging).
sudo systemctl disable --now litellm-compat-matrix.timer
```
## Gotchas
- **The venv is pinned to Python 3.12 (`CRON_PYTHON_VERSION`).** The
e2e suite uses PEP 695 `type` aliases, which the VM's system Python
(3.11) can't parse; `run_daily.sh` has uv fetch a managed CPython
into `~/litellm-cron-worktree/.uv-python/` and syncs the venv against
it. The first run after a version bump is a cold venv rebuild.
- **The proxy port is `4100`, not `4000`.** This is so a developer SSH'd
into the same VM with their own `:4000` proxy doesn't collide with a
cron run. Override with `PROXY_PORT=...` in `/etc/litellm-compat-matrix.env`
if you need to.
- **`uv sync --frozen` requires the resolved tag to be tagged on
GitHub.** If the latest stable release was made but not pushed as a
git tag, the `git checkout` step fails. Push the tag, then rerun.
- **Publish-token rotation is your problem.** The cron does not
refresh the token; if `mateo-berri`'s PAT in
`/etc/litellm-compat-matrix-github-token` expires, the run fails at
the `git push`/`gh pr create` step with a 401 ("Bad credentials" /
"Authentication failed"). Mint a fresh PAT and update that file.
The token needs write access to `BerriAI/litellm-docs` (classic
`repo` scope, or fine-grained Contents:RW + Pull requests:RW). It is
delivered via systemd `LoadCredential`, not the env file, so pytest,
the proxy, and the claude CLI never inherit it; manual runs export
`GITHUB_TOKEN` instead.
- **First run after upgrading the Claude Code CLI is the riskiest one.**
If the new CLI changes its wire format the matrix run can produce
systematic failures. Always run with `SKIP_PUBLISH=1` after a CLI
upgrade before letting the next scheduled fire happen.
- **Disk:** the worktree's `.venv` is ~1.3 GB and the `.git` directory
is ~1 GB. Plan for at least 5 GB free on the VM, otherwise
`uv sync` will fail mid-run and leave you with a half-installed venv.

52
tests/e2e/claude_code/cron_vm/build_matrix.py Normal file
View file

@ -0,0 +1,52 @@
"""Tiny CLI wrapper around `claude_code.matrix_builder.build_from_paths`.
Exists only so `run_daily.sh` can hand the version metadata + paths into
the matrix builder without re-implementing it in bash. All real logic
lives in `matrix_builder.py`.
The suite imports its own modules with `tests/e2e/` on sys.path (that is
how pytest resolves them: `tests/e2e/` has no `__init__.py`, while
`claude_code/` does), so this script bootstraps the same root — two
levels up from this file — before importing.
"""
from __future__ import annotations
import argparse
import datetime
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from claude_code.matrix_builder import (
build_from_paths,
) # noqa: E402 # needs the sys.path bootstrap above
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--results", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--litellm-version", required=True)
parser.add_argument("--claude-code-version", required=True)
args = parser.parse_args()
generated_at = datetime.datetime.now(datetime.timezone.utc).strftime(
"%Y-%m-%dT%H:%M:%SZ"
)
build_from_paths(
manifest_path=args.manifest,
results_path=args.results,
litellm_version=args.litellm_version,
claude_code_version=args.claude_code_version,
generated_at=generated_at,
output_path=args.output,
)
print(f"wrote {args.output}") # noqa: T201 # CLI output read by run_daily.sh
return 0
if __name__ == "__main__":
sys.exit(main())

80
tests/e2e/claude_code/cron_vm/check_regressions.py Normal file
View file

@ -0,0 +1,80 @@
"""CLI: detect green→red regressions between the published matrix and a
freshly built one, so `run_daily.sh` can decide whether to enable
auto-merge on the daily docs PR.
All real logic lives in `claude_code.matrix_builder.find_regressions`;
this file only does the I/O and maps the result onto an exit code the
bash caller can branch on.
Exit codes (the bash gate depends on these exact values):
0 no green→red regressions -> safe to auto-merge
3 one or more green→red regressions -> do NOT auto-merge (human review)
2 argparse/usage error (argparse default)
The `--old` file is allowed to be missing: on the first-ever publish there
is no baseline to regress against, so we exit 0.
Imports resolve with `tests/e2e/` on sys.path, mirroring build_matrix.py.
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from claude_code.matrix_builder import (
find_regressions,
) # noqa: E402 # needs the sys.path bootstrap above
REGRESSION_EXIT = 3
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--old",
type=Path,
required=True,
help="currently published matrix JSON (may be absent on first publish)",
)
parser.add_argument(
"--new",
type=Path,
required=True,
help="freshly built matrix JSON",
)
args = parser.parse_args()
if not args.old.exists():
print( # noqa: T201 # CLI output read by run_daily.sh
"no published matrix to compare against "
"(first publish); treating as no regressions"
)
return 0
old_matrix = json.loads(args.old.read_text())
new_matrix = json.loads(args.new.read_text())
regressions = find_regressions(old_matrix, new_matrix)
if not regressions:
print("no green->red regressions detected") # noqa: T201 # CLI output
return 0
print( # noqa: T201 # CLI output read by run_daily.sh
f"detected {len(regressions)} green->red regression(s):"
)
for r in regressions:
line = f" - {r['feature_name']} [{r['provider']}]: pass -> fail"
if r["error"]:
line += f" ({r['error'][:160]})"
print(line) # noqa: T201 # CLI output read by run_daily.sh
return REGRESSION_EXIT
if __name__ == "__main__":
sys.exit(main())

68
tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example Normal file
View file

@ -0,0 +1,68 @@
# Environment file consumed by `litellm-compat-matrix.service`.
#
# Install at `/etc/litellm-compat-matrix.env` and chmod 0600.
# `EnvironmentFile=-` in the unit means the service is allowed to start
# even if this file is missing, but the populator will fail at the
# first provider request without these credentials.
# Anthropic
ANTHROPIC_API_KEY=
# Bedrock (invoke + converse columns; also bedrock_mantle when enabled).
# Use Anthropic's Bedrock API-key passthrough (long-lived bearer token).
# No AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY required for the matrix --
# both the LiteLLM invoke and converse routes pick up
# AWS_BEARER_TOKEN_BEDROCK when present.
AWS_BEARER_TOKEN_BEDROCK=
AWS_REGION_NAME=us-east-1
# Vertex AI (vertex_ai + vertex_ai_gpt columns).
# On the GCP VM, the default service-account ADC from the metadata server
# is used -- no JSON key file is needed. If you ever need to run outside
# GCP, also export GOOGLE_APPLICATION_CREDENTIALS=/path/to/sa.json.
VERTEXAI_PROJECT=
VERTEXAI_LOCATION=global
# Azure AI Foundry (azure column — Claude models on Foundry)
AZURE_AI_API_KEY=
AZURE_AI_API_BASE=
# OpenAI (openai GPT column)
OPENAI_API_KEY=
# Azure OpenAI (azure_openai GPT column)
AZURE_API_BASE=
AZURE_API_KEY=
# The publish PAT (mateo-berri, write access on BerriAI/litellm-docs)
# deliberately does NOT live in this file. Everything here lands in the
# process environment of pytest, the proxy, and the model-driven claude
# CLI, where any same-UID reader can lift it from /proc/<pid>/environ.
# Instead, install the token at /etc/litellm-compat-matrix-github-token
# (chmod 0600, single line); the service maps it in via systemd
# LoadCredential and run_daily.sh keeps it out of every child process
# env. Used to (a) resolve the latest stable release, (b) push the
# daily compat-matrix branch directly to BerriAI/litellm-docs, (c) open
# the same-repo PR, and (d) enable squash auto-merge on it. Scopes:
# classic `repo` + `workflow`, or fine-grained on BerriAI/litellm-docs
# with Contents:RW + Pull requests:RW + Workflows:RW.
# Manual runs export GITHUB_TOKEN instead, or skip publishing entirely
# with SKIP_PUBLISH=1 (only writes the matrix JSON locally).
# Optional: the bedrock_mantle column is opt-in because the AWS account
# needs the Mantle (OpenAI-on-Bedrock) models enabled. Without this the
# mantle cells are skipped and recorded as not_tested rather than fail.
# COMPAT_MANTLE_CELLS=1
# Optional: the openai column is likewise opt-in; its cells hit CLI
# timeouts under the concurrent stage suite, but the serial cron can
# usually run them. Skipped cells are recorded as not_tested.
# COMPAT_OPENAI_GPT_CELLS=1
# Optional overrides; defaults are sensible for the cron VM.
# PROXY_PORT=4100
# LITELLM_WORKTREE=/home/mateo/litellm-cron-worktree
# DOCS_REPO=BerriAI/litellm-docs
# DOCS_BRANCH=main
# DOCS_TARGET_PATH=src/data/compatibility-matrix.json
# AUTO_MERGE_METHOD=squash

113
tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service Normal file
View file

@ -0,0 +1,113 @@
# systemd service for the Claude Code compatibility-matrix populator.
#
# Triggered by `litellm-compat-matrix.timer`; not started directly. The
# unit is a `Type=oneshot` so the timer's `OnCalendar=` semantics
# describe "run once per day" cleanly — there's no long-lived daemon to
# supervise; each invocation runs the populator end-to-end and exits.
#
# Install
# -------
#
# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/
# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/
# sudo systemctl daemon-reload
# sudo systemctl enable --now litellm-compat-matrix.timer
#
# Paths are hard-coded to /home/mateo rather than using systemd's %h
# specifier. Why: in *system* units (this one), %h is expanded at
# parse time against the *manager's* home -- which is /root for PID 1
# -- and *not* against the User= directive. That mismatch makes
# ReadWritePaths point at /root/.cache (which doesn't exist), causing
# the namespace setup to fail with status=226/NAMESPACE before the
# script ever runs. The runtime user (`User=mateo`) must:
#
# * have a checkout of `BerriAI/litellm` at `~/litellm/litellm` so the
# publisher module is importable;
# * have a uv venv at `~/litellm/litellm/.venv` (created by
# `uv sync --frozen` inside that checkout once);
# * have `gh` already authenticated against an account with
# `pull-requests: write` on `BerriAI/litellm-docs`;
# * have provider credentials exported in `/etc/litellm-compat-matrix.env`
# (see `litellm-compat-matrix.env.example` in this directory);
# * have the mateo-berri publish PAT at
# `/etc/litellm-compat-matrix-github-token` (chmod 0600, single
# line), delivered via `LoadCredential=` below.
[Unit]
Description=Claude Code compatibility-matrix populator (oneshot)
Documentation=file:///home/mateo/litellm/litellm/tests/e2e/claude_code/cron_vm/README.md
Wants=network-online.target
After=network-online.target
[Service]
Type=oneshot
User=mateo
Group=mateo
# Provider credentials + any gh/PROXY_PORT overrides live here. Format
# is the standard `KEY=value` one line per env var.
EnvironmentFile=-/etc/litellm-compat-matrix.env
# The mateo-berri publish PAT is mapped in via the credential store, NOT
# the EnvironmentFile, so it never lands in the process environment that
# pytest, the proxy, and the model-driven claude CLI inherit (any
# same-UID process can read /proc/<pid>/environ). run_daily.sh reads
# ${CREDENTIALS_DIRECTORY}/github-token and hands it to gh per call.
# Unlike EnvironmentFile= above, this is deliberately NOT optional: a
# missing token file fails the unit at start instead of 30 minutes in.
LoadCredential=github-token:/etc/litellm-compat-matrix-github-token
# systemd starts with a minimal PATH (~/usr/local/bin:/usr/bin:/bin).
# `uv` and `claude` are installed under the runtime user's `~/.local/bin`
# so we have to prepend it explicitly; otherwise run_daily.sh fails at
# the up-front command-presence check.
Environment=PATH=/home/mateo/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
# `HOME` is auto-set to /home/mateo when User=mateo is honored, but be
# explicit so anything that reads $HOME (e.g. uv's cache lookup, the
# claude CLI's per-session dir) sees the right value even if a future
# refactor flips DynamicUser= or PrivateUsers= on.
Environment=HOME=/home/mateo
WorkingDirectory=/home/mateo/litellm/litellm
ExecStart=/home/mateo/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh
# 90 minutes is generous: cold runs do `git clone` + `uv sync` of a new
# tag's lockfile, which can take a couple of minutes on a 2-vCPU VM,
# plus the full feature x provider grid of pytest cells hitting several
# cloud providers.
TimeoutStartSec=90min
# A failed run shouldn't restart automatically — the next timer fire is
# the right retry. Reruns of the same day's matrix are idempotent.
Restart=no
# Security hardening: the populator only reads the litellm checkout and
# the env-file; everything else it writes lives in either the worktree
# (managed) or `/tmp` (cleaned up by tempfile).
#
# ReadWritePaths whitelist:
# * litellm-cron-worktree - the long-lived stable-tag checkout +
# its `.venv` (`uv sync` rewrites every
# run) + `.uv-bin` (pinned `uv` binary
# cache).
# * .cache - uv's wheel cache (~/.cache/uv) so we
# don't redownload pinned deps each run.
# * .claude - `claude` CLI's per-session state under
# `~/.claude/projects/<sha>/`; created
# on every `claude --print` invocation.
# * .config/gh - `gh` CLI host config; technically not
# needed when we pass GH_TOKEN inline,
# but cheap to whitelist and prevents
# future regressions if a code path
# ever falls back to the host config.
# * /tmp - mktemp -d workdir + proxy logs.
NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=read-only
ReadWritePaths=/home/mateo/litellm-cron-worktree /home/mateo/.cache /home/mateo/.claude /home/mateo/.config/gh /tmp
PrivateTmp=true
[Install]
WantedBy=multi-user.target

25
tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer Normal file
View file

@ -0,0 +1,25 @@
# Daily timer for the compatibility-matrix populator.
#
# 06:00 UTC matches the original GitHub Actions cron schedule; chosen so
# operators in US/EU timezones see fresh PRs at the start of their work
# day.
#
# `Persistent=true` causes a missed run (VM was off / suspended) to
# fire the next time the timer is started, which is the property we
# want for a once-a-day job: the matrix should refresh as soon as the
# VM is reachable again, not wait another 24h.
#
# `RandomizedDelaySec=10min` smears load if multiple matrix-style
# pipelines are ever colocated on the same VM in the future.
[Unit]
Description=Run the Claude Code compatibility-matrix populator daily
[Timer]
OnCalendar=*-*-* 06:00:00 UTC
Persistent=true
RandomizedDelaySec=10min
Unit=litellm-compat-matrix.service
[Install]
WantedBy=timers.target

672
tests/e2e/claude_code/cron_vm/run_daily.sh Executable file
View file

@ -0,0 +1,672 @@
#!/usr/bin/env bash
# Daily Claude Code compatibility-matrix populator.
#
# Runs from the GCP VM `litellm-compatibility-matrix-populator` via the
# systemd timer in this directory. The flow is:
#
# 1. Resolve the latest LiteLLM final release tag from the GitHub
# Releases API.
# 2. Update a long-lived worktree at $WORKTREE to that tag and `uv sync` it.
# 3. Boot the proxy as a background subprocess on $PROXY_PORT (default
# 4100; a separate port from the human-tended :4000 proxy).
# 4. Run `pytest tests/e2e/claude_code/` against the proxy. Test
# failures become `fail` cells in the JSON, not script errors.
# 5. Hand the per-test results artifact + manifest to a small Python
# CLI (`build_matrix.py`) that wraps the existing
# `matrix_builder.build_from_paths` to produce the published
# compatibility-matrix.json.
# 6. `gh repo clone` litellm-docs, write the JSON to a deterministic
# branch (`compat-matrix/<litellm>-<claude>-<UTC-date>`), commit,
# push the branch straight to BerriAI/litellm-docs (mateo-berri has
# write access), `gh pr create`, then — *only if no cell regressed
# green→red versus the currently-published matrix* — enable squash
# auto-merge so the PR merges itself once required checks pass. A
# green→red regression leaves auto-merge off for human review; an
# already-red cell (red→red) does not block.
# 7. Sweep stale compat-matrix PRs: once today's PR exists, close any
# other open `compat-matrix/*` PR (and delete its bot-owned branch)
# so at most ONE compat-matrix PR is ever open — the newest. A
# gate-withheld PR that nobody triages is superseded by the next
# day's run rather than accumulating in the queue.
#
# Same-day reruns land on the same branch so they update the existing PR
# rather than spawning a new one. If the JSON is byte-identical to the
# docs branch, we skip the push entirely.
#
# Required commands on $PATH: git, uv, gh, jq, curl, claude, npm.
# Required state: a litellm checkout at $LITELLM_REPO (this file lives in
# it), $WORKTREE is created on first run, gh is already authenticated.
#
# Override any default by setting the matching env var; see the systemd
# unit for the production wiring.
set -Eeuo pipefail
LITELLM_REPO="${LITELLM_REPO:-${HOME}/litellm/litellm}"
WORKTREE="${LITELLM_WORKTREE:-${HOME}/litellm-cron-worktree}"
PROXY_PORT="${PROXY_PORT:-4100}"
PROXY_API_KEY="${PROXY_API_KEY:-sk-cron-matrix}"
DOCS_REPO="${DOCS_REPO:-BerriAI/litellm-docs}"
DOCS_BRANCH="${DOCS_BRANCH:-main}"
DOCS_TARGET_PATH="${DOCS_TARGET_PATH:-src/data/compatibility-matrix.json}"
SKIP_PUBLISH="${SKIP_PUBLISH:-0}"
PYTEST_K="${PYTEST_K:-}"
# The e2e suite uses PEP 695 `type` aliases, so the venv needs Python
# >= 3.12 (also what repo CI runs) even when the VM's system python is
# older. uv fetches a managed CPython of this version on first use --
# checksum-verified against the manifest baked into the pinned uv
# binary -- and installs it under ${WORKTREE}/.uv-python (see
# UV_PYTHON_INSTALL_DIR below) so it lives inside the one tree the
# systemd sandbox lets us write to.
CRON_PYTHON_VERSION="${CRON_PYTHON_VERSION:-3.12}"
# Merge method for auto-merge. BerriAI/litellm-docs only allows squash
# merges (merge-commit and rebase are disabled at the repo level), so
# `squash` is the only valid value here unless that changes upstream.
AUTO_MERGE_METHOD="${AUTO_MERGE_METHOD:-squash}"
POPULATOR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
WORKDIR="$(mktemp -d -t litellm-compat-matrix.XXXXXX)"
PROXY_PID_FILE="${WORKDIR}/proxy.pid"
# Cleanup is intentionally aggressive: it can run on normal exit, on a
# signal received by the script, or after a partial failure where the
# proxy is up but ${PROXY_PID_FILE} is stale. We try four things in
# order and stop as soon as the proxy port is free:
#
# 1. SIGTERM the pid recorded in proxy.pid.
# 2. SIGKILL anything from `pgrep -f "litellm.*--port ${PROXY_PORT}"`
# that survived. This catches the common case where the recorded
# pid was the sh wrapper, not the long-lived python child.
# 3. ss -K on the port (kernel kills sockets but not processes;
# mostly useful for catching lingering CLOSE_WAITs).
# 4. wipe ${WORKDIR}.
cleanup() {
local rc=$?
set +e
local proxy_pid
if [[ -f "${PROXY_PID_FILE}" ]]; then
proxy_pid="$(cat "${PROXY_PID_FILE}")"
if [[ -n "${proxy_pid}" ]]; then
kill -TERM "-${proxy_pid}" 2>/dev/null || kill -TERM "${proxy_pid}" 2>/dev/null || true
for _ in 1 2 3 4 5; do
kill -0 "${proxy_pid}" 2>/dev/null || break
sleep 1
done
fi
fi
# Belt-and-braces: any python or uv talking to ${PROXY_PORT} that
# survived the SIGTERM gets SIGKILL'd by name.
pgrep -f "litellm.*--port[ =]?${PROXY_PORT}([^0-9]|$)" 2>/dev/null \
| xargs -r kill -KILL 2>/dev/null || true
pgrep -f "${WORKTREE}/.uv-bin/uv.*run litellm" 2>/dev/null \
| xargs -r kill -KILL 2>/dev/null || true
rm -rf "${WORKDIR}"
exit "${rc}"
}
trap cleanup EXIT INT TERM
log() { printf '==> %s\n' "$*" >&2; }
die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
for cmd in git uv gh jq curl claude; do
command -v "${cmd}" >/dev/null 2>&1 || die "missing required command: ${cmd}"
done
# Publishing pushes the branch straight to BerriAI/litellm-docs and opens
# the PR as mateo-berri, who has write access on the docs repo. Under
# systemd the PAT arrives as a file via LoadCredential=, NOT via the
# EnvironmentFile: several suite cells let the model-driven claude CLI
# read arbitrary files as this user, and /proc/<pid>/environ of the
# script, pytest, and the proxy would hand an env-borne token to any
# same-UID reader. Kept as an unexported shell variable and passed per
# invocation (GH_TOKEN=... / curl header / push URL), it never enters a
# child's environment. Manual runs may export GITHUB_TOKEN instead.
# Require it up front -- failing 30 minutes into a run is a waste of CI
# quota.
if [[ -z "${GITHUB_TOKEN:-}" && -n "${CREDENTIALS_DIRECTORY:-}" && -f "${CREDENTIALS_DIRECTORY}/github-token" ]]; then
GITHUB_TOKEN="$(<"${CREDENTIALS_DIRECTORY}/github-token")"
log "publish token source: systemd credential store"
elif [[ -n "${GITHUB_TOKEN:-}" ]]; then
log "publish token source: process environment"
fi
if [[ "${SKIP_PUBLISH}" != "1" ]]; then
[[ -n "${GITHUB_TOKEN:-}" ]] \
|| die "publish token required: /etc/litellm-compat-matrix-github-token via LoadCredential under systemd, or an exported GITHUB_TOKEN for manual runs (or set SKIP_PUBLISH=1)"
fi
# ---------------------------------------------------------------------------
# 1. Resolve versions
# ---------------------------------------------------------------------------
# Newest PEP 440 *final* release on BerriAI/litellm. LiteLLM moved off
# the legacy `vX.Y.Z-stable` tag convention to PEP 440: a final/stable
# release is now a bare `vX.Y.Z` tag, while pre-releases carry a
# `-rc.N` / `-dev.N` segment (and the old `…-stable` / `…-stable.patch.N`
# tags are legacy and frozen at v1.83.x). We therefore select the newest
# tag with no pre-release segment -- matching `^v[0-9]+\.[0-9]+\.[0-9]+$`
# -- and skip drafts. The numeric version_key sort handles 1.10 > 1.9.
#
# Paginate through the releases endpoint instead of grabbing only page 1
# (default page_size=30). LiteLLM ships multiple pre-releases per day, so
# it's common to need to walk past 30+ entries before hitting the most
# recent final release. We cap at 5 pages (500 releases) which is
# conservatively beyond the worst observed gap.
GH_AUTH_HEADER=()
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
GH_AUTH_HEADER=(-H "Authorization: Bearer ${GITHUB_TOKEN}")
fi
RELEASES_JSON="${WORKDIR}/releases.json"
echo "[]" >"${RELEASES_JSON}"
for page in 1 2 3 4 5; do
PAGE_JSON="${WORKDIR}/releases.page${page}.json"
curl -fsS \
-H 'Accept: application/vnd.github+json' \
-H 'User-Agent: litellm-compat-matrix' \
"${GH_AUTH_HEADER[@]}" \
"https://api.github.com/repos/BerriAI/litellm/releases?per_page=100&page=${page}" \
>"${PAGE_JSON}"
jq -s '.[0] + .[1]' "${RELEASES_JSON}" "${PAGE_JSON}" >"${RELEASES_JSON}.merged"
mv "${RELEASES_JSON}.merged" "${RELEASES_JSON}"
# Stop early once we've seen at least one final release tag — no point
# paging further for a daily script that only needs the newest.
if jq -e '[.[] | select((.draft // false) == false) | .tag_name // "" | select(test("^v[0-9]+\\.[0-9]+\\.[0-9]+$"))] | length > 0' "${PAGE_JSON}" >/dev/null; then
break
fi
# No more pages? GitHub returns an empty array past the last page.
if [[ "$(jq 'length' "${PAGE_JSON}")" == "0" ]]; then
break
fi
done
LITELLM_VERSION="$(
jq -r '
[ .[]
| select((.draft // false) == false)
| .tag_name // empty
| select(test("^v[0-9]+\\.[0-9]+\\.[0-9]+$"))
]
| sort_by(
capture("^v(?<a>[0-9]+)\\.(?<b>[0-9]+)\\.(?<c>[0-9]+)$")
| [(.a|tonumber), (.b|tonumber), (.c|tonumber)]
)
| last // empty
' "${RELEASES_JSON}"
)"
[[ -n "${LITELLM_VERSION}" ]] || die "could not resolve latest PEP 440 final release (vX.Y.Z) in 5 pages of releases"
log "resolved litellm: ${LITELLM_VERSION}"
CLAUDE_CODE_VERSION="$(claude --version 2>/dev/null | awk '{print $1}')"
[[ -n "${CLAUDE_CODE_VERSION}" ]] || die "could not read 'claude --version'"
log "local claude code: ${CLAUDE_CODE_VERSION}"
# ---------------------------------------------------------------------------
# 2. Update the worktree to that tag
# ---------------------------------------------------------------------------
if [[ ! -d "${WORKTREE}/.git" ]]; then
log "first run: cloning litellm into ${WORKTREE}"
mkdir -p "$(dirname "${WORKTREE}")"
git clone https://github.com/BerriAI/litellm.git "${WORKTREE}"
fi
log "updating worktree to ${LITELLM_VERSION}"
git -C "${WORKTREE}" fetch --tags --force
git -C "${WORKTREE}" reset --hard
# Keep the venv, the .uv-bin cache, and the .uv-python managed
# interpreter around — uv sync will reconcile the venv on every run,
# and we don't want to re-download the pinned uv binary or the managed
# CPython each time. Drop everything else (including any prior
# tests/e2e/ shim) so each run starts clean before the shim below
# rewrites it from the dev checkout.
git -C "${WORKTREE}" clean -fdx -e .venv -e .uv-bin -e .uv-python
git -C "${WORKTREE}" checkout --force "${LITELLM_VERSION}"
# Always rebuild tests/e2e/ in the worktree from the dev checkout,
# regardless of what the resolved ${LITELLM_VERSION} tag ships. Two
# reasons:
#
# * The matrix populator's job is to exercise *today's* tests against
# the latest stable proxy. The dev checkout carries the most recent
# test fixes that haven't yet rolled into a stable release, and we
# want every cron run to pick those up the moment they land on
# ${LITELLM_REPO}, not whenever the next stable release happens.
# * The tag's own tests/e2e/ ships the full EKS e2e harness, whose
# top-level conftest.py imports modules (e2e_db, lifecycle,
# otel_client, ...) that the stable venv does not install. Copying
# the whole tree would make pytest collection blow up on those
# imports.
#
# So the shim is a fresh `rm -rf` of tests/e2e/ followed by copying ONLY
# the claude_code suite plus the shared transport helpers it imports.
# pytest puts tests/e2e/ itself on sys.path (it has no __init__.py, while
# claude_code/ does), which is what resolves both the `claude_code.*`
# and the bare `proxy_client` / `e2e_http` imports inside the suite.
E2E_HELPER_FILES=(proxy_client.py e2e_http.py models.py e2e_config.py transport.py)
if [[ ! -d "${LITELLM_REPO}/tests/e2e/claude_code" ]]; then
die "no shim source at ${LITELLM_REPO}/tests/e2e/claude_code"
fi
for helper in "${E2E_HELPER_FILES[@]}"; do
[[ -f "${LITELLM_REPO}/tests/e2e/${helper}" ]] \
|| die "missing shim helper: ${LITELLM_REPO}/tests/e2e/${helper}"
done
log "shimming tests/e2e/claude_code/ + helpers from ${LITELLM_REPO} (always-overwrite)"
rm -rf "${WORKTREE}/tests/e2e"
mkdir -p "${WORKTREE}/tests/e2e"
cp -r "${LITELLM_REPO}/tests/e2e/claude_code" "${WORKTREE}/tests/e2e/"
for helper in "${E2E_HELPER_FILES[@]}"; do
cp "${LITELLM_REPO}/tests/e2e/${helper}" "${WORKTREE}/tests/e2e/"
done
# litellm pins an exact uv version in pyproject.toml's [tool.uv]
# `required-version` field, so a system uv that's newer or older
# refuses to sync. We pin our own local copy at the version the
# checked-out tag asks for, cached under .uv-bin/ inside the worktree
# so subsequent runs skip the download.
PINNED_UV_VERSION="$(
awk -F'"' '
/^required-version[[:space:]]*=/ {
# Field 2 is the value between the quotes, e.g. ">=0.10.9" or
# "0.10.9". Strip any leading specifier prefix so we end up with
# the bare version string, which is what /releases/download/<v>/
# expects.
v = $2
sub(/^[[:space:]=<>!~]+/, "", v)
if (v != "") { print v; exit }
}
' "${WORKTREE}/pyproject.toml"
)"
if [[ -z "${PINNED_UV_VERSION}" ]]; then
log "no uv version pin in pyproject.toml; using system uv"
WORKTREE_UV="$(command -v uv)"
else
WORKTREE_UV="${WORKTREE}/.uv-bin/uv-${PINNED_UV_VERSION}"
if [[ ! -x "${WORKTREE_UV}" ]]; then
log "downloading uv ${PINNED_UV_VERSION} for the worktree"
mkdir -p "${WORKTREE}/.uv-bin"
UV_TARBALL_NAME="uv-x86_64-unknown-linux-gnu.tar.gz"
UV_DOWNLOAD_URL="https://github.com/astral-sh/uv/releases/download/${PINNED_UV_VERSION}/${UV_TARBALL_NAME}"
UV_TMPDIR="$(mktemp -d -t uv-download.XXXXXX)"
# Download the tarball and Astral's official .sha256 sidecar to disk
# and verify the digest before extracting/executing anything. This
# closes the supply-chain trust gap of piping a remote binary
# straight into `tar -xzO ... > file ; chmod +x` (see CLAUDE.md
# "CI Supply-Chain Safety").
curl -fsSL --output "${UV_TMPDIR}/${UV_TARBALL_NAME}" "${UV_DOWNLOAD_URL}"
curl -fsSL --output "${UV_TMPDIR}/${UV_TARBALL_NAME}.sha256" "${UV_DOWNLOAD_URL}.sha256"
(cd "${UV_TMPDIR}" && sha256sum -c "${UV_TARBALL_NAME}.sha256") \
|| { rm -rf "${UV_TMPDIR}"; die "uv ${PINNED_UV_VERSION} sha256 mismatch — refusing to install"; }
tar -xzf "${UV_TMPDIR}/${UV_TARBALL_NAME}" -C "${UV_TMPDIR}" "uv-x86_64-unknown-linux-gnu/uv"
mv "${UV_TMPDIR}/uv-x86_64-unknown-linux-gnu/uv" "${WORKTREE_UV}.tmp"
chmod +x "${WORKTREE_UV}.tmp"
mv "${WORKTREE_UV}.tmp" "${WORKTREE_UV}"
rm -rf "${UV_TMPDIR}"
fi
fi
# `--extra proxy` pulls fastapi/uvicorn/etc. so `uv run litellm` can
# actually serve. `--group proxy-dev` brings in pytest and the rest of
# what tests/e2e/claude_code/ needs. `--python` pins the venv to
# ${CRON_PYTHON_VERSION}; the first run after a version bump recreates
# the venv from scratch (a one-time cold sync).
export UV_PYTHON_INSTALL_DIR="${WORKTREE}/.uv-python"
log "uv sync --frozen --group proxy-dev --extra proxy --python ${CRON_PYTHON_VERSION} (uv ${PINNED_UV_VERSION:-system})"
(cd "${WORKTREE}" && "${WORKTREE_UV}" sync --frozen --group proxy-dev --extra proxy --python "${CRON_PYTHON_VERSION}")
PROXY_CONFIG="${WORKTREE}/tests/e2e/claude_code/test_config.yaml"
[[ -f "${PROXY_CONFIG}" ]] || die "proxy config not found at ${PROXY_CONFIG} (shim incomplete?)"
# ---------------------------------------------------------------------------
# 3. Boot the proxy
# ---------------------------------------------------------------------------
log "starting proxy on 127.0.0.1:${PROXY_PORT}"
# Bind the proxy to loopback only. The populator proxy is talked to
# exclusively by the pytest run on the same host (the health check and
# the test env set `LITELLM_PROXY_URL=http://127.0.0.1:...`),
# so there's no reason to expose it on the VM's external interfaces.
# Without `--host`, `litellm` defaults to 0.0.0.0, which combined with
# the predictable default `LITELLM_MASTER_KEY=sk-cron-matrix` would
# allow anything that can reach :${PROXY_PORT} on the VM to authenticate
# and burn upstream provider credentials.
#
# `setsid` puts the proxy in its own session+pgroup so cleanup() can
# SIGTERM the whole tree by passing the pgid as a negative pid. We
# write that pid to a file so cleanup() doesn't need to remember a
# variable that might be stale by the time the trap fires.
setsid env LITELLM_MASTER_KEY="${PROXY_API_KEY}" bash -c '
echo "$$" > "$0"
cd "$1"
exec "$2" run litellm --config "$3" --host 127.0.0.1 --port "$4"
' "${PROXY_PID_FILE}" "${WORKTREE}" "${WORKTREE_UV}" "${PROXY_CONFIG}" "${PROXY_PORT}" \
>"${WORKDIR}/proxy.log" 2>&1 &
disown
HEALTH_URL="http://127.0.0.1:${PROXY_PORT}/health/liveliness"
for _ in $(seq 1 45); do
if curl -fsS "${HEALTH_URL}" >/dev/null 2>&1; then
break
fi
sleep 2
done
curl -fsS "${HEALTH_URL}" >/dev/null \
|| { tail -50 "${WORKDIR}/proxy.log" >&2; die "proxy did not become healthy"; }
# ---------------------------------------------------------------------------
# 4. Run pytest
# ---------------------------------------------------------------------------
RESULTS_JSON="${WORKDIR}/compat-results.json"
# The `_*_unit_tests` ignore is defensive: those harness-only trees are
# markerless (they run without a proxy) and don't feed matrix cells, so
# the cron skips them if/when they land in the suite.
PYTEST_ARGS=(
tests/e2e/claude_code/
"--ignore-glob=*_unit_tests*"
)
if [[ -n "${PYTEST_K}" ]]; then
log "PYTEST_K set; narrowing to: ${PYTEST_K}"
PYTEST_ARGS+=(-k "${PYTEST_K}")
fi
log "running pytest"
set +e
(
cd "${WORKTREE}" \
&& LITELLM_PROXY_URL="http://127.0.0.1:${PROXY_PORT}" \
LITELLM_MASTER_KEY="${PROXY_API_KEY}" \
COMPAT_RESULTS_PATH="${RESULTS_JSON}" \
"${WORKTREE_UV}" run pytest "${PYTEST_ARGS[@]}"
)
PYTEST_EXIT=$?
set -e
log "pytest exit code: ${PYTEST_EXIT} (failures become 'fail' cells, not script errors)"
# 0=green, 1=test failures (fail cells); >=2 = interrupted/internal/usage/no
# tests, i.e. a partial run whose missing cells would publish as not_tested.
[[ ${PYTEST_EXIT} -le 1 ]] \
|| die "pytest exited abnormally (${PYTEST_EXIT}); refusing to publish a partial matrix"
[[ -f "${RESULTS_JSON}" ]] || die "pytest did not produce ${RESULTS_JSON}"
# ---------------------------------------------------------------------------
# 5. Build the matrix JSON
# ---------------------------------------------------------------------------
MATRIX_JSON="${WORKDIR}/compatibility-matrix.json"
log "building ${MATRIX_JSON}"
(
cd "${WORKTREE}" \
&& "${WORKTREE_UV}" run python "${POPULATOR_DIR}/build_matrix.py" \
--manifest "${WORKTREE}/tests/e2e/claude_code/manifest.yaml" \
--results "${RESULTS_JSON}" \
--output "${MATRIX_JSON}" \
--litellm-version "${LITELLM_VERSION}" \
--claude-code-version "${CLAUDE_CODE_VERSION}"
)
# ---------------------------------------------------------------------------
# 6. Open a docs-repo PR
# ---------------------------------------------------------------------------
if [[ "${SKIP_PUBLISH}" == "1" ]]; then
cp "${MATRIX_JSON}" "${LITELLM_REPO}/compatibility-matrix.json"
log "SKIP_PUBLISH=1; matrix written to ${LITELLM_REPO}/compatibility-matrix.json"
exit 0
fi
DATE_UTC="$(date -u +%Y-%m-%d)"
BRANCH_NAME="compat-matrix/${LITELLM_VERSION}-${CLAUDE_CODE_VERSION}-${DATE_UTC}"
DOCS_CLONE="${WORKDIR}/litellm-docs"
log "cloning ${DOCS_REPO}@${DOCS_BRANCH}"
gh repo clone "${DOCS_REPO}" "${DOCS_CLONE}" -- --depth 1 --branch "${DOCS_BRANCH}"
cd "${DOCS_CLONE}"
git config user.email "litellm-bot@berri.ai"
git config user.name "litellm-compat-matrix-bot"
git checkout -b "${BRANCH_NAME}"
# Snapshot the currently-published matrix *before* we overwrite it, so the
# auto-merge gate below can diff old→new cell statuses. On the first-ever
# publish the file won't exist yet; we leave ${PUBLISHED_MATRIX} pointing
# at a path that doesn't exist and let check_regressions.py treat that as
# "no baseline → no regressions".
PUBLISHED_MATRIX="${WORKDIR}/published-matrix.json"
if [[ -f "${DOCS_TARGET_PATH}" ]]; then
cp "${DOCS_TARGET_PATH}" "${PUBLISHED_MATRIX}"
fi
mkdir -p "$(dirname "${DOCS_TARGET_PATH}")"
cp "${MATRIX_JSON}" "${DOCS_TARGET_PATH}"
git add "${DOCS_TARGET_PATH}"
if git diff --cached --quiet; then
log "matrix JSON unchanged from ${DOCS_BRANCH}; skipping PR"
exit 0
fi
# --- Auto-merge regression gate --------------------------------------------
# Only auto-merge when the new matrix is improvement-or-equal: every cell
# transition is red→green, green→green, or red→red. If any cell flips
# green→red (a `pass` that became `fail`), we still open/refresh the PR but
# leave auto-merge OFF so a human reviews the regression before it lands on
# the public docs table. A pre-existing red cell (e.g. Anthropic out of API
# credits) is red→red and does NOT block, so the daily PR keeps flowing.
log "checking for green->red regressions vs the published matrix"
set +e
REGRESSION_REPORT="$(
cd "${WORKTREE}" \
&& "${WORKTREE_UV}" run python "${POPULATOR_DIR}/check_regressions.py" \
--old "${PUBLISHED_MATRIX}" \
--new "${MATRIX_JSON}"
)"
REGRESSION_EXIT=$?
set -e
printf '%s\n' "${REGRESSION_REPORT}" | sed 's/^/ /' >&2
# Exit 0 = clean. Exit 3 = green→red regression(s) found. Any other code
# means the checker itself errored; fail *closed* (withhold auto-merge) so a
# bug in the gate can never silently auto-merge a regression.
if [[ ${REGRESSION_EXIT} -eq 0 ]]; then
ALLOW_AUTOMERGE=1
elif [[ ${REGRESSION_EXIT} -eq 3 ]]; then
ALLOW_AUTOMERGE=0
log "WARN: green->red regression(s) detected; auto-merge will be left OFF for review"
else
ALLOW_AUTOMERGE=0
log "WARN: regression check errored (exit ${REGRESSION_EXIT}); withholding auto-merge to be safe"
fi
GENERATED_AT="$(jq -r '.generated_at' "${MATRIX_JSON}")"
COMMIT_MSG="$(cat <<EOF
Update Claude Code compatibility matrix
litellm_version: ${LITELLM_VERSION}
claude_code_version: ${CLAUDE_CODE_VERSION}
generated_at: ${GENERATED_AT}
EOF
)"
git commit -m "${COMMIT_MSG}"
# Push the branch straight to BerriAI/litellm-docs. mateo-berri has write
# access on the docs repo, so there's no fork hop: the PR is a same-repo
# branch PR. The temp remote carries the token in its URL, so we add it,
# push, then immediately remove it so the token never lingers in
# ${DOCS_CLONE}/.git/config. (${DOCS_CLONE} is also rm -rf'd by the
# cleanup trap on exit.)
#
# Plain --force (not --force-with-lease) is acceptable here: the
# compat-matrix/* branch is bot-owned, only this script ever writes to
# it, and runs are serialized by the systemd timer. --force-with-lease
# would require a fetch to populate the remote-tracking ref before each
# push and adds no safety in this single-writer setup.
PUBLISH_PUSH_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${DOCS_REPO}.git"
git remote remove publish 2>/dev/null || true
git remote add publish "${PUBLISH_PUSH_URL}"
git push --force --set-upstream publish "${BRANCH_NAME}"
git remote remove publish
unset PUBLISH_PUSH_URL
# Per-feature status table for the PR body. Reviewers triage from this.
PR_FEATURE_TABLE="$(jq -r '
.features[] as $f
| "- **\($f.name)**: " +
([ .providers[] as $p
| "\($p)=\($f.providers[$p].status // "not_tested")"
] | join(", "))
' "${MATRIX_JSON}")"
# When the gate withheld auto-merge, call it out at the top of the PR body
# (with the offending cells) so a reviewer knows this PR needs a human and
# why. On the clean path this section is empty. Note `$(...)` strips the
# trailing newline, so the body below puts explicit blank lines *around*
# the placeholder rather than relying on the heredoc's own spacing.
if [[ "${ALLOW_AUTOMERGE}" != "1" ]]; then
PR_REGRESSION_SECTION="$(cat <<EOF
> [!WARNING]
> **Auto-merge disabled:** one or more cells regressed green→red versus the
> currently-published matrix. Review the diff before merging.
\`\`\`
${REGRESSION_REPORT}
\`\`\`
EOF
)"
else
PR_REGRESSION_SECTION=""
fi
PR_TITLE="chore(compat-matrix): refresh for ${LITELLM_VERSION} + claude-code ${CLAUDE_CODE_VERSION}"
PR_BODY="$(cat <<EOF
Automated daily refresh of the Claude Code compatibility matrix.
${PR_REGRESSION_SECTION}
| Field | Value |
| --- | --- |
| litellm_version | \`${LITELLM_VERSION}\` |
| claude_code_version | \`${CLAUDE_CODE_VERSION}\` |
| generated_at | \`${GENERATED_AT}\` |
## Per-feature results
${PR_FEATURE_TABLE}
---
Generated by \`tests/e2e/claude_code/cron_vm/run_daily.sh\`. Close without merging if the diff looks wrong; the next cron run will reopen with fresh results.
EOF
)"
log "opening PR from ${BRANCH_NAME} -> ${DOCS_REPO}:${DOCS_BRANCH} (as mateo-berri)"
# GH_TOKEN is mateo-berri's write-scoped token, the same identity used
# for release-listing above. The branch lives on ${DOCS_REPO} itself, so
# --head is a bare branch name (a same-repo PR), not `OWNER:BRANCH`.
set +e
PR_OUT="$(
GH_TOKEN="${GITHUB_TOKEN}" gh pr create \
--repo "${DOCS_REPO}" \
--base "${DOCS_BRANCH}" \
--head "${BRANCH_NAME}" \
--title "${PR_TITLE}" \
--body "${PR_BODY}" 2>&1
)"
PR_EXIT=$?
set -e
echo "${PR_OUT}"
if [[ ${PR_EXIT} -ne 0 ]]; then
if grep -q "a pull request for branch.*already exists" <<<"${PR_OUT}"; then
log "PR already exists for ${BRANCH_NAME}; updated branch in place"
else
die "gh pr create failed (exit ${PR_EXIT})"
fi
fi
# Enable auto-merge so the PR merges itself once the docs repo's required
# checks pass -- we no longer gate these bot PRs on a second human
# approval. mateo-berri authors and merges them directly. The repo only
# permits squash merges and has auto-merge enabled at the repo level
# (${AUTO_MERGE_METHOD} defaults to squash accordingly).
#
# This only fires when the regression gate above is satisfied
# (${ALLOW_AUTOMERGE}==1): a green→red regression — or a gate error —
# leaves auto-merge OFF so a human triages the PR.
#
# `gh pr merge --auto` is idempotent: re-enabling auto-merge on a PR that
# already has it set is a no-op, so same-day reruns stay clean. It's
# non-fatal: if auto-merge can't be enabled (e.g. the PR is already in a
# clean/mergeable state with nothing left to wait on, or branch
# protection isn't configured), the matrix JSON has still landed on the
# PR and the worst case is a manual merge click.
if [[ "${ALLOW_AUTOMERGE}" == "1" ]]; then
log "enabling ${AUTO_MERGE_METHOD} auto-merge on ${BRANCH_NAME}"
set +e
GH_TOKEN="${GITHUB_TOKEN}" gh pr merge \
"${BRANCH_NAME}" \
--repo "${DOCS_REPO}" \
--auto \
"--${AUTO_MERGE_METHOD}" 2>&1 | sed 's/^/ /'
AUTOMERGE_EXIT=${PIPESTATUS[0]}
set -e
if [[ ${AUTOMERGE_EXIT} -ne 0 ]]; then
log "WARN: gh pr merge --auto exited ${AUTOMERGE_EXIT} (non-fatal)"
fi
else
# Regression (or gate error): make sure auto-merge is OFF. A same-day
# rerun may have enabled it on an earlier, clean pass, so explicitly
# disable rather than just skipping. The disable call itself is allowed
# to error (`--disable-auto` fails harmlessly when auto-merge was never
# enabled), but the read-back below is authoritative: a regressed matrix
# must never be left armed to merge, so a still-armed PR is fatal.
log "leaving ${BRANCH_NAME} for manual review; disabling any prior auto-merge"
set +e
GH_TOKEN="${GITHUB_TOKEN}" gh pr merge \
"${BRANCH_NAME}" \
--repo "${DOCS_REPO}" \
--disable-auto 2>&1 | sed 's/^/ /'
set -e
AUTOMERGE_ARMED="$(
GH_TOKEN="${GITHUB_TOKEN}" gh pr view \
"${BRANCH_NAME}" \
--repo "${DOCS_REPO}" \
--json autoMergeRequest \
--jq '.autoMergeRequest.enabledAt // empty'
)" || die "could not read back the auto-merge state on ${BRANCH_NAME}"
[[ -z "${AUTOMERGE_ARMED}" ]] \
|| die "auto-merge still armed on ${BRANCH_NAME} (enabled ${AUTOMERGE_ARMED}) after --disable-auto"
fi
# --- Stale-PR sweep ----------------------------------------------------------
# Keep at most ONE compat-matrix PR open: today's. Any other open
# `compat-matrix/*` PR is a leftover from a day whose regression gate
# withheld auto-merge and nobody triaged it; the PR we just opened or
# refreshed above carries strictly fresher results, so the old one is
# pure queue noise. Closing is non-destructive — the PR record and its
# regression report stay browsable; only the bot-owned branch is
# deleted. This runs only after today's PR exists (a `die` above skips
# it), so a failed publish can never close the queue down to zero.
#
# Non-fatal: a sweep failure (rate limit, transient API error) leaves
# stale PRs for the next run to retry; it must not fail the pipeline.
log "sweeping stale compat-matrix PRs (keeping ${BRANCH_NAME})"
set +e
STALE_PRS="$(
GH_TOKEN="${GITHUB_TOKEN}" gh pr list \
--repo "${DOCS_REPO}" \
--state open \
--limit 100 \
--json number,headRefName \
--jq '.[] | select(.headRefName | startswith("compat-matrix/")) | "\(.number)\t\(.headRefName)"'
)"
while IFS=$'\t' read -r stale_pr stale_head; do
[[ -z "${stale_pr}" ]] && continue
[[ "${stale_head}" == "${BRANCH_NAME}" ]] && continue
GH_TOKEN="${GITHUB_TOKEN}" gh pr close "${stale_pr}" \
--repo "${DOCS_REPO}" \
--delete-branch \
--comment "Superseded by the newer daily compat-matrix PR from \`${BRANCH_NAME}\`; the populator keeps only the most recent compat-matrix PR open." 2>&1 | sed 's/^/ /'
if [[ ${PIPESTATUS[0]} -eq 0 ]]; then
log "closed stale compat-matrix PR #${stale_pr} (${stale_head})"
else
log "WARN: could not close stale compat-matrix PR #${stale_pr} (non-fatal)"
fi
done <<<"${STALE_PRS}"
set -e
log "done"

80
tests/e2e/claude_code/matrix_builder.py
View file

@ -174,6 +174,86 @@ def _aggregate_cell(results: Sequence[Mapping[str, Any]]) -> Dict[str, Any]:
return {"status": "not_tested"}
def _index_cells(matrix: Mapping[str, Any]) -> dict[tuple[str, str], dict[str, Any]]:
"""Map ``(feature_id, provider) -> cell dict`` for a built matrix.
Cells are keyed by the *stable* feature ``id`` (not the display
``name``, which can be reworded without changing the underlying row)
and the provider key, so two matrices built at different times line up
even if feature names drift.
"""
out: dict[tuple[str, str], dict[str, Any]] = {}
for feature in matrix.get("features", []) or []:
if not isinstance(feature, Mapping):
continue
feature_id = feature.get("id")
if not feature_id:
continue
providers = feature.get("providers", {}) or {}
if not isinstance(providers, Mapping):
continue
for provider, cell in providers.items():
if isinstance(cell, Mapping):
out[(feature_id, provider)] = dict(cell)
return out
def find_regressions(
old_matrix: Mapping[str, Any],
new_matrix: Mapping[str, Any],
) -> list[dict[str, str]]:
"""Return the cells that flipped green→red (``pass`` → ``fail``).
A *regression* is defined strictly: a cell that was ``pass`` in
``old_matrix`` and is ``fail`` in ``new_matrix``. Every other
transition is intentionally *not* a regression:
* ``red → green`` / ``green → green`` — the happy path.
* ``red → red`` — a cell that is *already* failing for an unrelated
reason (e.g. Anthropic out of API credits) must not block
publishing, otherwise the daily PR would never auto-merge until
that independent issue is fixed.
* ``green → not_tested`` / ``green → not_applicable`` — a cell going
grey is a degradation but not a *red* regression; treating a
skipped/flaky run as a hard block would create false positives.
Cells present only in ``new_matrix`` (a newly added feature or
provider) have no baseline and therefore cannot be regressions.
Each returned item is a flat str→str mapping so callers (the cron's
``check_regressions.py``) can render it without further lookups:
``feature_id``, ``feature_name``, ``provider``, ``old_status``,
``new_status``, ``error``.
"""
old_cells = _index_cells(old_matrix)
feature_names = {
f.get("id"): str(f.get("name", f.get("id")))
for f in new_matrix.get("features", []) or []
if isinstance(f, Mapping) and f.get("id")
}
regressions: list[dict[str, str]] = []
for (feature_id, provider), new_cell in sorted(
_index_cells(new_matrix).items(), key=lambda kv: (kv[0][0], kv[0][1])
):
if new_cell.get("status") != "fail":
continue
old_cell = old_cells.get((feature_id, provider))
if old_cell is None or old_cell.get("status") != "pass":
continue
regressions.append(
{
"feature_id": str(feature_id),
"feature_name": feature_names.get(feature_id, str(feature_id)),
"provider": str(provider),
"old_status": "pass",
"new_status": "fail",
"error": str(new_cell.get("error", "")),
}
)
return regressions
def build_from_paths(
*,
manifest_path: Path,

12
tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py
View file

@ -1,5 +1,6 @@
"""Per-request multi-tenant credential routing (V1 parity)."""
import base64
import os
import sys
@ -42,6 +43,17 @@ def test_langfuse_dynamic_headers_need_both_keys():
assert headers is not None and "Authorization" in headers
def test_langfuse_dynamic_headers_carry_v4_ingestion_version():
headers = dynamic_otlp_headers(
"langfuse_otel", {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}
)
expected_auth = "Basic " + base64.b64encode(b"pk:sk").decode()
assert headers == {
"Authorization": expected_auth,
"x-langfuse-ingestion-version": "4",
}
def test_weave_dynamic_headers():
headers = dynamic_otlp_headers(
"weave_otel", {"wandb_api_key": "w", "weave_project_id": "p"}

51
tests/test_litellm/integrations/test_langfuse_otel.py
View file

@ -211,7 +211,7 @@ class TestLangfuseOtelIntegration:
LangfuseSpanAttributes.GENERATION_NAME.value: "gen-name",
LangfuseSpanAttributes.GENERATION_ID.value: "gen-id",
LangfuseSpanAttributes.PARENT_OBSERVATION_ID.value: "parent-id",
LangfuseSpanAttributes.GENERATION_VERSION.value: "v1",
LangfuseSpanAttributes.VERSION.value: "t-ver",
LangfuseSpanAttributes.MASK_INPUT.value: True,
LangfuseSpanAttributes.MASK_OUTPUT.value: False,
LangfuseSpanAttributes.TRACE_USER_ID.value: "user-123",
@ -221,8 +221,7 @@ class TestLangfuseOtelIntegration:
LangfuseSpanAttributes.TRACE_NAME.value: "trace-name",
LangfuseSpanAttributes.TRACE_ID.value: "traceid", # stripped dashes
LangfuseSpanAttributes.TRACE_METADATA.value: json.dumps({"k": "v"}),
LangfuseSpanAttributes.TRACE_VERSION.value: "t-ver",
LangfuseSpanAttributes.TRACE_RELEASE.value: "rel-1",
LangfuseSpanAttributes.RELEASE.value: "rel-1",
LangfuseSpanAttributes.EXISTING_TRACE_ID.value: "existing-id",
LangfuseSpanAttributes.UPDATE_TRACE_KEYS.value: json.dumps(
["key1", "key2"]
@ -240,6 +239,52 @@ class TestLangfuseOtelIntegration:
actual == expected
), "Mismatch between expected and actual OTEL attribute mapping."
@pytest.mark.parametrize(
"metadata, expected_version",
[
(
{"version": "v-observation", "trace_version": "v-trace"},
"v-trace",
),
({"trace_version": "v-trace"}, "v-trace"),
({"version": "v-observation"}, "v-observation"),
({"version": "v-observation", "trace_version": ""}, ""),
({}, None),
],
ids=[
"trace-version-wins-as-documented",
"trace-only",
"observation-version-is-the-fallback",
"empty-trace-version-is-not-absent",
"neither-key-emits-nothing",
],
)
def test_version_emitted_on_langfuse_v4_key(self, metadata, expected_version):
kwargs = {"litellm_params": {"metadata": {"trace_release": "rel-9", **metadata}}}
with patch(
"litellm.integrations.arize._utils.safe_set_attribute"
) as mock_safe_set_attribute:
LangfuseOtelLogger._set_langfuse_specific_attributes(
MagicMock(), kwargs, None
)
emitted = {
call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list
}
if expected_version is None:
assert "langfuse.version" not in emitted
else:
assert emitted["langfuse.version"] == expected_version
assert emitted["langfuse.release"] == "rel-9"
for retired_key in (
"langfuse.generation.version",
"langfuse.trace.version",
"langfuse.trace.release",
):
assert retired_key not in emitted
def test_set_langfuse_specific_attributes_with_content(self):
"""Test that _set_langfuse_specific_attributes correctly sets observation.output with regular content response."""
from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes

21
tests/test_litellm/interactions/test_google_interactions_integration.py
View file

@ -55,17 +55,10 @@ class TestGoogleInteractionsCreate:
print(f"Usage: {response.usage}")
def test_create_with_content_list(self, api_key):
"""Test creating an interaction with a structured content list (Turn format)."""
"""Test creating an interaction with a structured content list (Content[] input)."""
response = interactions.create(
model="gemini/gemini-2.5-flash",
input=[
{
"role": "user",
"content": [
{"type": "text", "text": "What is the capital of France?"}
],
}
],
input=[{"type": "text", "text": "What is the capital of France?"}],
api_key=api_key,
)
@ -169,25 +162,25 @@ class TestGoogleInteractionsStreaming:
class TestGoogleInteractionsMultiTurn:
"""Tests for multi-turn conversations using Turn[] input."""
"""Tests for multi-turn conversations using Step[] input."""
def test_multi_turn_conversation(self, api_key):
"""Test a multi-turn conversation per OpenAPI spec (Turn[] format)."""
"""Test a multi-turn conversation per OpenAPI spec (Step[] format)."""
response = interactions.create(
model="gemini/gemini-2.5-flash",
input=[
{
"role": "user",
"type": "user_input",
"content": [{"type": "text", "text": "My name is Alice."}],
},
{
"role": "model",
"type": "model_output",
"content": [
{"type": "text", "text": "Hello Alice! Nice to meet you."}
],
},
{
"role": "user",
"type": "user_input",
"content": [{"type": "text", "text": "What is my name?"}],
},
],

40
tests/test_litellm/interactions/test_openapi_compliance.py
View file

@ -167,17 +167,39 @@ class TestRequestCompliance:
assert text_schema["properties"]["type"].get("const") == "text"
print("✓ TextContent schema is correct")
def test_turn_schema(self, spec_dict):
"""Verify Turn schema for multi-turn conversations."""
turn_schema = spec_dict["components"]["schemas"]["Turn"]
def test_step_schema(self, spec_dict):
"""Verify step-based multi-turn input.
assert "role" in turn_schema["properties"]
assert "content" in turn_schema["properties"]
Google replaced the role-carrying `Turn` schema with typed steps
(spec update of Aug 13, 2026): conversation history is now a `Step[]`
where `UserInputStep`/`ModelOutputStep` pin `type` values that our
transformations read to recover the role. Assert exactly what our code
depends on: `InteractionsInput` accepts a Step array, both step kinds
are part of the `Step` union, each pins its `type` const, and each
carries a `Content[]` content field.
"""
input_schema = spec_dict["components"]["schemas"]["InteractionsInput"]
step_array_items = [
option["items"]["$ref"].split("/")[-1]
for option in input_schema["oneOf"]
if option.get("type") == "array" and "$ref" in option.get("items", {})
]
assert "Step" in step_array_items, f"InteractionsInput should accept Step[], got arrays of {step_array_items}"
# Content can be string or Content[]
content_prop = turn_schema["properties"]["content"]
assert "oneOf" in content_prop
print("✓ Turn schema supports role + content")
step_variants = {
option["$ref"].split("/")[-1]
for option in spec_dict["components"]["schemas"]["Step"]["oneOf"]
if "$ref" in option
}
assert {"UserInputStep", "ModelOutputStep"} <= step_variants, f"Step union is missing role steps: {step_variants}"
for step_name, type_value in [("UserInputStep", "user_input"), ("ModelOutputStep", "model_output")]:
step_schema = spec_dict["components"]["schemas"][step_name]
assert step_schema["properties"]["type"].get("const") == type_value
assert "type" in step_schema["required"]
content_items = step_schema["properties"]["content"]["items"]
assert content_items["$ref"].split("/")[-1] == "Content"
print(f"✓ {step_name} pins type '{type_value}' with Content[] content")
class TestResponseCompliance:

64
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -485,6 +485,70 @@ def test_generic_cost_per_token_minimax_m3_above_512k_tokens():
assert round(completion_cost, 10) == round(expected_completion, 10)
@pytest.mark.parametrize(
"model",
[
"bedrock_mantle/openai.gpt-5.6-sol",
"bedrock_mantle/openai.gpt-5.6-terra",
"bedrock_mantle/openai.gpt-5.6-luna",
],
)
def test_generic_cost_per_token_bedrock_mantle_gpt56_long_context(model):
"""Bedrock GPT-5.6 supports a 1M context window, billed at the long-context rates above 272K."""
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
model_cost_map = litellm.model_cost[model]
assert model_cost_map["max_input_tokens"] == 1000000
cached_tokens = 100000
completion_tokens = 1000
short_prompt_tokens = 272000
short_usage = Usage(
prompt_tokens=short_prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=short_prompt_tokens + completion_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens),
)
short_prompt_cost, short_completion_cost = generic_cost_per_token(
model=model,
usage=short_usage,
custom_llm_provider="bedrock_mantle",
)
assert round(short_prompt_cost, 10) == round(
model_cost_map["input_cost_per_token"] * (short_prompt_tokens - cached_tokens)
+ model_cost_map["cache_read_input_token_cost"] * cached_tokens,
10,
)
assert round(short_completion_cost, 10) == round(
model_cost_map["output_cost_per_token"] * completion_tokens, 10
)
long_prompt_tokens = 900000
long_usage = Usage(
prompt_tokens=long_prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=long_prompt_tokens + completion_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens),
)
long_prompt_cost, long_completion_cost = generic_cost_per_token(
model=model,
usage=long_usage,
custom_llm_provider="bedrock_mantle",
)
assert round(long_prompt_cost, 10) == round(
model_cost_map["input_cost_per_token_above_272k_tokens"]
* (long_prompt_tokens - cached_tokens)
+ model_cost_map["cache_read_input_token_cost_above_272k_tokens"]
* cached_tokens,
10,
)
assert round(long_completion_cost, 10) == round(
model_cost_map["output_cost_per_token_above_272k_tokens"] * completion_tokens, 10
)
def test_generic_cost_per_token_honors_non_standard_above_threshold():
"""Regression for #30344: get_model_info must keep arbitrary
input/output_cost_per_token_above_<N>_tokens thresholds, not only the hard-coded

6
tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py
View file

@ -1526,7 +1526,11 @@ class TestBedrockMantleResponsesPricing:
assert info["cache_creation_input_token_cost"] == pytest.approx(cache_creation_cost)
assert info["cache_read_input_token_cost"] == pytest.approx(cache_read_cost)
assert info["output_cost_per_token"] == pytest.approx(output_cost)
assert info["max_input_tokens"] == 272000
assert info["max_input_tokens"] == 1000000
assert info["input_cost_per_token_above_272k_tokens"] == pytest.approx(input_cost * 2)
assert info["cache_creation_input_token_cost_above_272k_tokens"] == pytest.approx(cache_creation_cost * 2)
assert info["cache_read_input_token_cost_above_272k_tokens"] == pytest.approx(cache_read_cost * 2)
assert info["output_cost_per_token_above_272k_tokens"] == pytest.approx(output_cost * 1.5)
@pytest.mark.parametrize(
"model, input_cost, output_cost",

28
tests/test_litellm/proxy/auth/test_unmapped_model_budget_enforcement.py
View file

@ -162,6 +162,34 @@ class TestUnmappedModelBudgetEnforcement:
# Subsequent call sees the new pricing and enforces budget.
assert _is_model_cost_zero(model="ramping-model", llm_router=router) is False
def test_strategy_router_alias_with_zero_pricing_enforces_budget(self):
"""An auto-router alias is never the deployment that gets called or
billed, so zero pricing configured on it must not waive budget checks
for requests that route to (and bill as) a real paid deployment."""
router = Router(
model_list=[
{
"model_name": "smart-router",
"litellm_params": {
"model": "auto_router/complexity_router/smart-router",
"complexity_router_default_model": "paid-model",
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
"complexity_router_config": {"tiers": {"simple": "paid-model"}},
},
"model_info": {"id": "alias-id"},
},
{
"model_name": "paid-model",
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"},
"model_info": {"id": "paid-id"},
},
]
)
assert "input_cost_per_token" not in litellm.model_cost.get("alias-id", {})
assert _is_model_cost_zero(model="smart-router", llm_router=router) is False
def test_handles_router_without_zero_cost_cache_attribute(self):
"""Tolerate router-like objects (e.g. ``MagicMock`` stand-ins) that
do not expose ``_zero_cost_cache`` — the auth check must still

89
tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py
View file

@ -21,6 +21,7 @@ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.health_endpoints._health_endpoints import (
_db_health_readiness_check,
_show_no_redis_warning,
get_callback_identifier,
health_license_endpoint,
health_services_endpoint,
@ -2457,3 +2458,91 @@ class TestConfigBaseForHealthCheck:
)
assert base["litellm_credential_name"] == "OpenAI-prod"
assert base["api_key"] == "sk-configured"
class TestNoRedisWarning:
"""`show_no_redis_warning` drives the Admin UI's default-on "no Redis" banner."""
@staticmethod
def _router(redis_cache):
return SimpleNamespace(cache=SimpleNamespace(redis_cache=redis_cache))
def test_warns_when_no_redis_is_configured(self, monkeypatch):
monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False)
with (
patch("litellm.proxy.proxy_server.redis_usage_cache", None),
patch("litellm.proxy.proxy_server.llm_router", self._router(None)),
):
assert _show_no_redis_warning() is True
def test_warns_when_there_is_no_router_at_all(self, monkeypatch):
monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False)
with (
patch("litellm.proxy.proxy_server.redis_usage_cache", None),
patch("litellm.proxy.proxy_server.llm_router", None),
):
assert _show_no_redis_warning() is True
def test_stays_quiet_when_a_coordination_redis_is_configured(self, monkeypatch):
monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False)
with (
patch("litellm.proxy.proxy_server.redis_usage_cache", MagicMock()),
patch("litellm.proxy.proxy_server.llm_router", self._router(None)),
):
assert _show_no_redis_warning() is False
def test_stays_quiet_when_only_the_router_has_redis(self, monkeypatch):
"""router_settings.redis_host alone backs cooldowns and usage-based routing."""
monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False)
with (
patch("litellm.proxy.proxy_server.redis_usage_cache", None),
patch("litellm.proxy.proxy_server.llm_router", self._router(MagicMock())),
):
assert _show_no_redis_warning() is False
@pytest.mark.parametrize("value", ["true", "True"])
def test_env_var_suppresses_the_warning(self, monkeypatch, value):
monkeypatch.setenv("LITELLM_DISABLE_NO_REDIS_WARNING", value)
with (
patch("litellm.proxy.proxy_server.redis_usage_cache", None),
patch("litellm.proxy.proxy_server.llm_router", self._router(None)),
):
assert _show_no_redis_warning() is False
def test_env_var_set_false_keeps_the_warning(self, monkeypatch):
monkeypatch.setenv("LITELLM_DISABLE_NO_REDIS_WARNING", "false")
with (
patch("litellm.proxy.proxy_server.redis_usage_cache", None),
patch("litellm.proxy.proxy_server.llm_router", self._router(None)),
):
assert _show_no_redis_warning() is True
@pytest.mark.asyncio
@pytest.mark.parametrize("has_prisma_client", [True, False])
async def test_readiness_details_carries_the_flag(self, monkeypatch, has_prisma_client):
monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False)
prisma_client = MagicMock() if has_prisma_client else None
with (
patch("litellm.proxy.proxy_server.prisma_client", prisma_client),
patch("litellm.proxy.proxy_server.redis_usage_cache", None),
patch("litellm.proxy.proxy_server.llm_router", self._router(None)),
patch.object(
_health_endpoints_module,
"_db_health_readiness_check",
AsyncMock(return_value={"status": "connected"}),
),
):
details = await _health_endpoints_module._get_health_readiness_details()
assert details["show_no_redis_warning"] is True
with (
patch("litellm.proxy.proxy_server.prisma_client", prisma_client),
patch("litellm.proxy.proxy_server.redis_usage_cache", MagicMock()),
patch.object(
_health_endpoints_module,
"_db_health_readiness_check",
AsyncMock(return_value={"status": "connected"}),
),
):
details = await _health_endpoints_module._get_health_readiness_details()
assert details["show_no_redis_warning"] is False

44
tests/test_litellm/proxy/proxy_server/test_lifecycle.py
View file

@ -205,6 +205,50 @@ async def test_proxy_shutdown_event_prisma_disconnect_raises_error(monkeypatch):
await proxy_shutdown_event()
# ---------------------------------------------------------------------------
# _flush_spend_logs_queue_on_shutdown
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_flush_spend_logs_queue_on_shutdown_drains_before_disconnect(monkeypatch):
fake_prisma = MagicMock()
monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False)
monkeypatch.setattr(ps, "db_writer_client", None, raising=False)
drain = AsyncMock()
import litellm.proxy.utils as utils_mod
monkeypatch.setattr(utils_mod, "drain_spend_logs_queue", drain)
await ps._flush_spend_logs_queue_on_shutdown()
observed = {
"drain_calls": drain.await_count,
"drain_prisma": drain.await_args.kwargs["prisma_client"] is fake_prisma,
}
assert observed == {
"drain_calls": 1,
"drain_prisma": True,
}
@pytest.mark.asyncio
async def test_flush_spend_logs_queue_on_shutdown_swallows_drain_errors(monkeypatch):
monkeypatch.setattr(ps, "prisma_client", MagicMock(), raising=False)
monkeypatch.setattr(ps, "db_writer_client", None, raising=False)
import litellm.proxy.utils as utils_mod
monkeypatch.setattr(
utils_mod,
"drain_spend_logs_queue",
AsyncMock(side_effect=RuntimeError("db gone")),
)
await ps._flush_spend_logs_queue_on_shutdown()
# ---------------------------------------------------------------------------
# _initialize_shared_aiohttp_session
# ---------------------------------------------------------------------------

1
tests/test_litellm/proxy/utils/prisma_and_spend/conftest.py
View file

@ -128,6 +128,7 @@ def mock_prisma_client() -> MagicMock:
client.proxy_logging_obj.failure_handler = AsyncMock()
client.spend_log_transactions = []
client._spend_log_transactions_lock = asyncio.Lock()
client.spend_logs_queue_monitor_task = None
client.tool_usage_transactions = []
client._tool_usage_transactions_lock = asyncio.Lock()
client.jsonify_object = lambda data: dict(data)

194
tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py
View file

@ -17,8 +17,10 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from litellm.proxy.utils import (
MAX_SPEND_LOG_DRAIN_ITERATIONS,
_monitor_spend_logs_queue,
_raise_failed_update_spend_exception,
drain_spend_logs_queue,
update_daily_tag_spend,
update_spend,
update_spend_logs_job,
@ -263,6 +265,198 @@ async def test_update_spend_logs_job_processes_and_clears_queue(
}
@pytest.mark.asyncio
async def test_update_spend_logs_job_requeues_popped_rows_when_write_cancelled(
mock_prisma_client: Any, make_spend_log_row: Any
) -> None:
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [
make_spend_log_row(request_id="r1"),
make_spend_log_row(request_id="r2"),
]
row_arriving_mid_flush = make_spend_log_row(request_id="r3")
async def _cancel_mid_write(*args: Any, **kwargs: Any) -> None:
mock_prisma_client.spend_log_transactions.append(row_arriving_mid_flush)
raise asyncio.CancelledError()
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(
side_effect=_cancel_mid_write
)
with pytest.raises(asyncio.CancelledError):
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert [
row["request_id"] for row in mock_prisma_client.spend_log_transactions
] == ["r1", "r2", "r3"]
@pytest.mark.asyncio
async def test_update_spend_logs_job_does_not_requeue_when_cancelled_after_write(
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Rows are already committed once guardrail tracking runs, so replaying
them would double-count the non-idempotent daily guardrail increments.
"""
import litellm.proxy.guardrails.usage_tracking as guard_mod
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock()
monkeypatch.setattr(
guard_mod,
"process_spend_logs_guardrail_usage",
AsyncMock(side_effect=asyncio.CancelledError()),
raising=False,
)
with pytest.raises(asyncio.CancelledError):
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert mock_prisma_client.spend_log_transactions == []
@pytest.mark.asyncio
async def test_drain_spend_logs_queue_flushes_rows_queued_while_draining(
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
import litellm.proxy.db.spend_log_tool_index as tool_mod
import litellm.proxy.guardrails.usage_tracking as guard_mod
monkeypatch.setattr(
guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False
)
monkeypatch.setattr(
tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False
)
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
written: list[str] = []
async def _write(*args: Any, **kwargs: Any) -> None:
written.extend(row["request_id"] for row in kwargs["data"])
if len(written) == 1:
mock_prisma_client.spend_log_transactions.append(
make_spend_log_row(request_id="r2")
)
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write)
await drain_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert written == ["r1", "r2"]
assert mock_prisma_client.spend_log_transactions == []
@pytest.mark.asyncio
async def test_drain_spend_logs_queue_stops_monitor_and_keeps_its_popped_rows(
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
import litellm.proxy.db.spend_log_tool_index as tool_mod
import litellm.proxy.guardrails.usage_tracking as guard_mod
monkeypatch.setattr(
guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False
)
monkeypatch.setattr(
tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False
)
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
write_started = asyncio.Event()
written: list[str] = []
write_calls = {"n": 0}
async def _write(*args: Any, **kwargs: Any) -> None:
write_calls["n"] += 1
if write_calls["n"] == 1:
write_started.set()
await asyncio.Event().wait()
written.extend(row["request_id"] for row in kwargs["data"])
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write)
async def _monitor() -> None:
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
mock_prisma_client.spend_logs_queue_monitor_task = asyncio.create_task(_monitor())
await write_started.wait()
await drain_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert written == ["r1"]
assert mock_prisma_client.spend_log_transactions == []
assert mock_prisma_client.spend_logs_queue_monitor_task is None
@pytest.mark.asyncio
async def test_drain_spend_logs_queue_gives_up_after_max_passes(
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
import litellm.proxy.db.spend_log_tool_index as tool_mod
import litellm.proxy.guardrails.usage_tracking as guard_mod
monkeypatch.setattr(
guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False
)
monkeypatch.setattr(
tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False
)
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
async def _write_and_refill(*args: Any, **kwargs: Any) -> None:
mock_prisma_client.spend_log_transactions.append(make_spend_log_row())
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(
side_effect=_write_and_refill
)
await drain_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert (
mock_prisma_client.db.litellm_spendlogs.create_many.await_count
== MAX_SPEND_LOG_DRAIN_ITERATIONS
)
@pytest.mark.asyncio
async def test_monitor_spend_logs_queue_invokes_job_when_queue_nonempty(
mock_prisma_client: Any,

246
tests/test_litellm/router_strategy/test_complexity_router.py
View file

@ -1176,6 +1176,41 @@ class TestPreRoutingStrategyRegistry:
}
assert router._select_pre_routing_strategy("smart", {}).strategy is cn
@staticmethod
def _router_with_plain_smart_deployment(enable_tag_filtering: bool) -> Router:
return Router(
model_list=[{"model_name": "smart", "litellm_params": {"model": "openai/gpt-4o-mini"}}],
enable_tag_filtering=enable_tag_filtering,
)
def test_select_falls_through_to_plain_deployments_when_no_tag_matches_under_tag_filtering(self):
router = self._router_with_plain_smart_deployment(enable_tag_filtering=True)
cn, us = object(), object()
router.complexity_routers = {"smart": [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]}
assert router._select_pre_routing_strategy("smart", {}) is None
assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["cn"]}}).strategy is cn
router.complexity_routers = {
"smart": [
TaggedPreRoutingStrategy(tags=("cn",), strategy=cn),
TaggedPreRoutingStrategy(tags=("us",), strategy=us),
]
}
assert router._select_pre_routing_strategy("smart", {}) is None
assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["row"]}}) is None
assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["us"]}}).strategy is us
router.complexity_routers["router-only"] = [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]
assert router._select_pre_routing_strategy("router-only", {}).strategy is cn
def test_select_keeps_capturing_when_tag_filtering_is_disabled(self):
router = self._router_with_plain_smart_deployment(enable_tag_filtering=False)
cn = object()
router.complexity_routers = {"smart": [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]}
assert router._select_pre_routing_strategy("smart", {}).strategy is cn
class TestAsyncPreRoutingHookMultiFormat:
"""Test async_pre_routing_hook with multiple input formats."""
@ -2041,14 +2076,54 @@ class TestRouterPreRoutingAliasOverrides:
assert request_kwargs["cache_control_injection_points"] == [{"location": "message", "role": "system"}]
@pytest.mark.asyncio
async def test_alias_overrides_exclude_only_model(self):
"""`model` (the alias marker, e.g. auto_router/complexity_router) is
excluded since it's never a real provider model. Router-only fields
like complexity_router_config DO flow through into request_kwargs at
this layer - they're filtered from the actual outbound LLM call
downstream by litellm.types.utils.all_litellm_params instead, not by
the router's pre-routing hook. See test_router_init_only_params_are_
never_sent_to_a_provider for the guard on that downstream filter."""
async def test_alias_custom_pricing_is_not_applied_to_request_kwargs(self):
"""Custom pricing on the alias prices the alias, not the tier deployment
the hook picked. Unlike the router-only fields, pricing fields are real
call params, so forwarding them would re-register the routed deployment
at the alias's price - an explicit 0 billing every request as free."""
router = Router(
model_list=[
{
"model_name": "smart-router",
"litellm_params": {
"model": "auto_router/complexity_router",
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
"input_cost_per_second": 0.0,
"drop_params": True,
"complexity_router_config": {"tiers": {"SIMPLE": "gpt-4o-mini"}},
"complexity_router_default_model": "gpt-4o",
},
},
{"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini"}},
{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o"}},
]
)
request_kwargs: dict = {}
result = await router.async_pre_routing_hook(
model="smart-router",
request_kwargs=request_kwargs,
messages=[{"role": "user", "content": "hi"}],
)
assert result is not None
# Non-pricing alias params still carry over.
assert request_kwargs["drop_params"] is True
for field in ("input_cost_per_token", "output_cost_per_token", "input_cost_per_second"):
assert field not in request_kwargs
@pytest.mark.asyncio
async def test_alias_overrides_exclude_only_marker_and_connection_params(self):
"""`model` (the alias marker, e.g. auto_router/complexity_router) and
provider-connection params (api_base/api_key/api_version) are excluded
since they never describe the tier deployment actually called.
Router-only fields like complexity_router_config DO flow through into
request_kwargs at this layer - they're filtered from the actual
outbound LLM call downstream by litellm.types.utils.all_litellm_params
instead, not by the router's pre-routing hook. See
test_router_init_only_params_are_never_sent_to_a_provider for the
guard on that downstream filter."""
router = self._make_router()
request_kwargs: Dict = {}
@ -2068,9 +2143,10 @@ class TestRouterPreRoutingAliasOverrides:
assert request_kwargs["complexity_router_default_model"] == "gpt-4o"
def test_router_init_only_params_are_never_sent_to_a_provider(self):
"""The router's pre-routing hook only excludes `model` (see
test_alias_overrides_exclude_only_model above) - every other alias
litellm_param, including router-init-only fields like
"""The router's pre-routing hook only excludes `model` and
provider-connection params (see test_alias_overrides_exclude_only_
marker_and_connection_params above) - every other alias litellm_param,
including router-init-only fields like
complexity_router_config, flows into request_kwargs unfiltered. That's
only safe because litellm.completion()/acompletion() itself strips
anything listed in all_litellm_params before building the provider
@ -2163,6 +2239,154 @@ class TestRouterPreRoutingAliasOverrides:
assert request_kwargs["drop_params"] is True
class TestRouterPreRoutingSharedAliasName:
"""
Regression tests for https://github.com/BerriAI/litellm/issues/36619.
A plain deployment and an `auto_router/` marker can share a `model_name`.
The alias-param forwarding after a pre-routing rewrite must read the
marker entry, never whichever same-name entry happens to sit first in
`model_list` - otherwise the plain entry's api_base/api_key get grafted
onto the routed tier's call (a Gemini path under api.openai.com, 404).
"""
@staticmethod
def _plain_entry() -> dict:
return {
"model_name": "gpt4o",
"litellm_params": {
"model": "openai/gpt-4o",
"api_key": "sk-plain-entry",
"api_base": "https://plain-entry.example/v1",
},
}
@staticmethod
def _marker_entry() -> dict:
return {
"model_name": "gpt4o",
"litellm_params": {
"model": "auto_router/complexity_router",
"drop_params": True,
"complexity_router_config": {"tiers": {"SIMPLE": "gemini-flash", "MEDIUM": "gemini-flash"}},
"complexity_router_default_model": "gemini-flash",
},
}
@staticmethod
def _tier_entry() -> dict:
return {
"model_name": "gemini-flash",
"litellm_params": {"model": "gemini/gemini-3.6-flash", "api_key": "sk-tier"},
}
@pytest.mark.asyncio
@pytest.mark.parametrize("plain_entry_first", [True, False], ids=["plain_entry_first", "marker_entry_first"])
async def test_marker_params_forwarded_regardless_of_model_list_order(self, plain_entry_first):
"""In either config order the routed call gets the marker's own params
(drop_params) and never the plain sibling's api_base/api_key."""
shared_name_entries = (
[self._plain_entry(), self._marker_entry()]
if plain_entry_first
else [self._marker_entry(), self._plain_entry()]
)
router = Router(model_list=[*shared_name_entries, self._tier_entry()])
request_kwargs: Dict = {}
result = await router.async_pre_routing_hook(
model="gpt4o",
request_kwargs=request_kwargs,
messages=[{"role": "user", "content": "What is the capital of France?"}],
)
assert result is not None
assert result.model == "gemini-flash"
assert "api_base" not in request_kwargs
assert "api_key" not in request_kwargs
assert request_kwargs["drop_params"] is True
@pytest.mark.asyncio
async def test_connection_params_on_the_marker_itself_are_not_forwarded(self):
"""Even when the marker entry carries api_base/api_key/api_version,
they describe no real deployment and must not reach the routed call,
while the marker's other params still do."""
marker_with_connection_params = {
"model_name": "smart",
"litellm_params": {
**self._marker_entry()["litellm_params"],
"api_key": "sk-marker",
"api_base": "https://marker.example/v1",
"api_version": "2024-01-01",
},
}
router = Router(model_list=[marker_with_connection_params, self._tier_entry()])
request_kwargs: Dict = {}
result = await router.async_pre_routing_hook(
model="smart",
request_kwargs=request_kwargs,
messages=[{"role": "user", "content": "hi"}],
)
assert result is not None
assert "api_base" not in request_kwargs
assert "api_key" not in request_kwargs
assert "api_version" not in request_kwargs
assert request_kwargs["drop_params"] is True
@pytest.mark.asyncio
async def test_tag_scoped_markers_forward_the_selected_markers_params(self):
"""With two tag-scoped markers under one name, the forwarded params
come from the marker whose tags matched the request, not from the
first marker in the list."""
def tagged_marker(routed_model: str, tags: list, drop_params: bool | None) -> dict:
return {
"model_name": "smart",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_default_model": routed_model,
"complexity_router_config": {"tiers": {"SIMPLE": [routed_model], "MEDIUM": [routed_model]}},
"tags": tags,
**({"drop_params": drop_params} if drop_params is not None else {}),
},
}
router = Router(
model_list=[
tagged_marker("gpt-cn", ["cn"], None),
tagged_marker("gpt-us", ["us"], True),
]
)
us_kwargs: Dict = {"metadata": {"tags": ["us"]}}
us_result = await router.async_pre_routing_hook(
model="smart",
request_kwargs=us_kwargs,
messages=[{"role": "user", "content": "hi"}],
)
assert us_result is not None and us_result.model == "gpt-us"
assert us_kwargs["drop_params"] is True
cn_kwargs: Dict = {"metadata": {"tags": ["cn"]}}
cn_result = await router.async_pre_routing_hook(
model="smart",
request_kwargs=cn_kwargs,
messages=[{"role": "user", "content": "hi"}],
)
assert cn_result is not None and cn_result.model == "gpt-cn"
assert "drop_params" not in cn_kwargs
def test_forwardable_alias_marker_params_reads_the_marker_entry_only(self):
router = Router(model_list=[self._plain_entry(), self._marker_entry(), self._tier_entry()])
forwarded = dict(router._forwardable_alias_marker_params(model="gpt4o", strategy_tags=()))
assert forwarded["drop_params"] is True
assert "api_key" not in forwarded and "api_base" not in forwarded
assert router._forwardable_alias_marker_params(model="gemini-flash", strategy_tags=()) == ()
class TestAdaptiveSoftFloors:
def test_adaptive_defaults_use_cost_weighted_cold_policy(self):
config = ComplexityRouterConfig(

27
tests/test_litellm/router_strategy/test_router_tag_routing.py
View file

@ -423,6 +423,33 @@ def test_get_tags_from_request_kwargs_various_inputs():
assert _get_tags_from_request_kwargs({"foo": "bar"}) == []
@pytest.mark.parametrize(
"request_kwargs",
[
{"metadata": "not-a-dict"},
{"litellm_metadata": "not-a-dict"},
{"litellm_metadata": ["not", "a", "dict"]},
{"litellm_params": "not-a-dict"},
{"litellm_params": {"metadata": "not-a-dict"}},
{"metadata": {"tags": "free"}},
{"metadata": {"tags": {"free": "paid"}}},
],
)
def test_get_tags_from_request_kwargs_reads_no_tags_from_a_non_dict_shape(request_kwargs):
"""Metadata and `tags` are request-controlled, so a client can send either as a
string, a list or null. Every shape that cannot hold string tags reads as untagged
instead of raising, because callers run on the hot request path."""
from litellm.router_strategy.tag_based_routing import _get_tags_from_request_kwargs
assert _get_tags_from_request_kwargs(request_kwargs) == []
def test_get_tags_from_request_kwargs_keeps_only_string_tags():
from litellm.router_strategy.tag_based_routing import _get_tags_from_request_kwargs
assert _get_tags_from_request_kwargs({"metadata": {"tags": ["free", 7, None, "paid"]}}) == ["free", "paid"]
# --- _split_tags unit tests ---

98
tests/test_litellm/test_router.py
View file

@ -7617,6 +7617,104 @@ class TestAutoRouterMaxInputCharsWiring:
assert self._registered_auto_router(router).max_input_chars == DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS
class TestTaggedAutoRouterOnSharedModelName:
"""A tagged auto-router marker sharing its model_name with a plain deployment must not
capture requests whose tags don't match it when tag filtering is enabled (#36620)."""
class _FixedRouteLayer:
def __call__(self, text: str):
from semantic_router.schema import RouteChoice
return RouteChoice(name="gemini-flash")
@classmethod
def _router(cls, marker_tags, include_plain_sibling: bool, enable_tag_filtering: bool) -> "litellm.Router":
pytest.importorskip("semantic_router", reason="auto-router needs the semantic-router extra")
marker = {
"model_name": "gpt4o",
"litellm_params": {
"model": "auto_router/gpt4o-router",
"auto_router_config": json.dumps(
{"routes": [{"name": "gemini-flash", "utterances": ["capital city questions"]}]}
),
"auto_router_default_model": "gemini-flash",
"auto_router_embedding_model": "text-embedding-3-small",
**({"tags": marker_tags} if marker_tags else {}),
},
}
plain = {"model_name": "gpt4o", "litellm_params": {"model": "openai/gpt-4o"}}
tier = {"model_name": "gemini-flash", "litellm_params": {"model": "gemini/gemini-3.6-flash"}}
router = litellm.Router(
model_list=[plain, marker, tier] if include_plain_sibling else [marker, tier],
enable_tag_filtering=enable_tag_filtering,
)
router.auto_routers["gpt4o"][0].strategy.routelayer = cls._FixedRouteLayer()
return router
@staticmethod
async def _hook_response(router: "litellm.Router", request_kwargs: dict):
return await router.async_pre_routing_hook(
model="gpt4o",
request_kwargs=request_kwargs,
messages=[{"role": "user", "content": "What is the capital of France?"}],
)
@pytest.mark.asyncio
async def test_untagged_request_bypasses_the_tagged_marker_when_a_plain_deployment_shares_the_name(self):
router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True)
assert await self._hook_response(router, {}) is None
@pytest.mark.asyncio
async def test_request_tagged_for_the_marker_is_still_semantically_routed(self):
router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True)
response = await self._hook_response(router, {"metadata": {"tags": ["route"]}})
assert response is not None
assert response.model == "gemini-flash"
@pytest.mark.asyncio
async def test_marker_only_alias_still_captures_untagged_requests(self):
router = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True)
response = await self._hook_response(router, {})
assert response is not None
assert response.model == "gemini-flash"
@pytest.mark.asyncio
async def test_untagged_marker_sharing_the_name_still_captures_untagged_requests(self):
router = self._router(marker_tags=None, include_plain_sibling=True, enable_tag_filtering=True)
response = await self._hook_response(router, {})
assert response is not None
assert response.model == "gemini-flash"
@pytest.mark.asyncio
async def test_untagged_selection_never_lands_on_the_marker_deployment(self):
router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True)
for _ in range(20):
deployment = await router.async_get_available_deployment(
model="gpt4o",
request_kwargs={},
messages=[{"role": "user", "content": "What is the capital of France?"}],
)
assert deployment["litellm_params"]["model"] == "openai/gpt-4o"
def test_deployment_without_litellm_params_mapping_is_not_a_marker(self):
assert litellm.Router._is_strategy_marker_deployment({"model_name": "gpt4o"}) is False
def test_model_name_has_plain_deployments_reflects_the_pool(self):
mixed = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True)
marker_only = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True)
assert mixed._model_name_has_plain_deployments("gpt4o") is True
assert marker_only._model_name_has_plain_deployments("gpt4o") is False
class TestGetAllowedFailsFromPolicy:
def _make_router(self, **policy_kwargs) -> litellm.Router:
from litellm.types.router import AllowedFailsPolicy

58
tests/test_litellm/test_router_model_cost_isolation.py
View file

@ -1471,3 +1471,61 @@ def test_replay_live_router_model_cost_rebuilds_every_live_router():
finally:
litellm.model_cost = saved_model_cost
_invalidate_model_cost_lowercase_map()
def test_strategy_router_alias_pricing_never_enters_model_cost(monkeypatch):
"""
A strategy-router alias is never the deployment actually called or billed,
so custom pricing configured on it must not be registered under its
model_id - an explicit zero there makes the budget check treat the alias
as a genuinely free model while requests bill as a real deployment. The
strip must also survive a price-data reload, which rebuilds entries by
walking the live routers.
"""
from litellm import utils as litellm_utils
monkeypatch.setattr(
litellm_utils,
"_runtime_registered_model_cost",
dict(litellm_utils._runtime_registered_model_cost),
)
router = Router(
model_list=[
{
"model_name": "smart-router",
"litellm_params": {
"model": "auto_router/complexity_router/smart-router",
"complexity_router_default_model": "paid-model",
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
"complexity_router_config": {"tiers": {"simple": "paid-model"}},
},
"model_info": {"id": "strategy-alias-id", "max_input_tokens": 128000},
},
{
"model_name": "paid-model",
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"},
"model_info": {"id": "strategy-alias-paid-id"},
},
],
)
def _assert_alias_unpriced():
entry = litellm.model_cost.get("strategy-alias-id")
assert entry is not None, "Alias metadata should still be registered"
assert entry["max_input_tokens"] == 128000
assert "input_cost_per_token" not in entry
assert "output_cost_per_token" not in entry
_assert_alias_unpriced()
saved_model_cost = litellm.model_cost
try:
_simulate_price_data_reload(
{"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}},
)
_assert_alias_unpriced()
assert router.model_list
finally:
litellm.model_cost = saved_model_cost
_invalidate_model_cost_lowercase_map()

1
ui/litellm-dashboard/src/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails.ts
View file

@ -13,6 +13,7 @@ export interface HealthReadinessDetailsResponse {
use_aiohttp_transport?: boolean;
log_level?: string;
is_detailed_debug?: boolean;
show_no_redis_warning?: boolean;
}
const fetchHealthReadinessDetails = async (accessToken: string): Promise<HealthReadinessDetailsResponse> => {

4
ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx
View file

@ -25,6 +25,10 @@ vi.mock("@/components/DebugWarningBanner", () => ({
DebugWarningBanner: () => null,
}));
vi.mock("@/components/NoRedisWarningBanner", () => ({
NoRedisWarningBanner: () => null,
}));
vi.mock("@/components/LicenseExpiryBanner", () => ({
LicenseExpiryBanner: () => null,
}));

3
ui/litellm-dashboard/src/app/(dashboard)/layout.tsx
View file

@ -9,6 +9,7 @@ import { useAuth } from "@/contexts/AuthContext";
import SidebarProvider from "@/app/(dashboard)/components/SidebarProvider";
import { useRouter, useSearchParams, usePathname } from "next/navigation";
import { DebugWarningBanner } from "@/components/DebugWarningBanner";
import { NoRedisWarningBanner } from "@/components/NoRedisWarningBanner";
import { LicenseExpiryBanner } from "@/components/LicenseExpiryBanner";
import { UserBanner } from "@/components/UserBanner";
import { MIGRATED_PAGES, migratedHref, legacyPageHref, legacyKeyForPathname } from "@/utils/migratedPages";
@ -120,6 +121,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) {
<div className="flex h-screen flex-col overflow-hidden bg-background">
<Navbar accessToken={accessToken} isPublicPage={false} />
<DebugWarningBanner accessToken={accessToken} />
<NoRedisWarningBanner accessToken={accessToken} />
<LicenseExpiryBanner accessToken={accessToken} />
<UserBanner accessToken={accessToken} />
<main className="flex min-h-0 flex-1 overflow-hidden">
@ -143,6 +145,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) {
<div className="flex min-w-0 flex-1 flex-col overflow-hidden">
<DashboardHeader page={page} />
<DebugWarningBanner accessToken={accessToken} />
<NoRedisWarningBanner accessToken={accessToken} />
<LicenseExpiryBanner accessToken={accessToken} />
<UserBanner accessToken={accessToken} />
<main className="min-w-0 flex-1 overflow-y-auto">{children}</main>

57
ui/litellm-dashboard/src/components/NoRedisWarningBanner.test.tsx Normal file
View file

@ -0,0 +1,57 @@
import { renderWithProviders, screen } from "../../tests/test-utils";
import { vi } from "vitest";
import { NoRedisWarningBanner } from "./NoRedisWarningBanner";
import type { HealthReadinessDetailsResponse } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails";
import type { UseQueryResult } from "@tanstack/react-query";
vi.mock("@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails", () => ({
useHealthReadinessDetails: vi.fn(),
}));
import { useHealthReadinessDetails } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails";
const mockDetails = (data: Partial<HealthReadinessDetailsResponse> | undefined) => {
vi.mocked(useHealthReadinessDetails).mockReturnValue({ data } as UseQueryResult<HealthReadinessDetailsResponse>);
};
describe("NoRedisWarningBanner", () => {
it("should warn that Redis is recommended when the proxy reports no Redis", () => {
mockDetails({ status: "healthy", show_no_redis_warning: true });
renderWithProviders(<NoRedisWarningBanner accessToken="token" />);
expect(screen.getByRole("alert")).toBeInTheDocument();
expect(screen.getByText(/No Redis configured\. Redis is highly recommended/i)).toBeInTheDocument();
});
it("should link to the docs page listing what breaks without Redis", () => {
mockDetails({ status: "healthy", show_no_redis_warning: true });
renderWithProviders(<NoRedisWarningBanner accessToken="token" />);
expect(screen.getByRole("link", { name: /does not work without Redis/i })).toHaveAttribute(
"href",
"https://docs.litellm.ai/docs/proxy/redis_requirements",
);
});
it("should name the env var that suppresses it", () => {
mockDetails({ status: "healthy", show_no_redis_warning: true });
renderWithProviders(<NoRedisWarningBanner accessToken="token" />);
expect(screen.getByText("LITELLM_DISABLE_NO_REDIS_WARNING=true")).toBeInTheDocument();
});
it("should render nothing when the proxy reports the warning is not needed", () => {
mockDetails({ status: "healthy", show_no_redis_warning: false });
const { container } = renderWithProviders(<NoRedisWarningBanner accessToken="token" />);
expect(container).toBeEmptyDOMElement();
});
it("should render nothing when readiness details are unavailable", () => {
mockDetails(undefined);
const { container } = renderWithProviders(<NoRedisWarningBanner accessToken={null} />);
expect(container).toBeEmptyDOMElement();
});
it("should pass the access token to the readiness hook", () => {
mockDetails(undefined);
renderWithProviders(<NoRedisWarningBanner accessToken="my-token" />);
expect(useHealthReadinessDetails).toHaveBeenCalledWith("my-token");
});
});

40
ui/litellm-dashboard/src/components/NoRedisWarningBanner.tsx Normal file
View file

@ -0,0 +1,40 @@
"use client";
import React from "react";
import { TriangleAlert } from "lucide-react";
import { useHealthReadinessDetails } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails";
const REDIS_DOCS_URL = "https://docs.litellm.ai/docs/proxy/redis_requirements";
interface NoRedisWarningBannerProps {
accessToken: string | null;
}
export const NoRedisWarningBanner: React.FC<NoRedisWarningBannerProps> = ({ accessToken }) => {
const { data: healthData } = useHealthReadinessDetails(accessToken);
if (!healthData?.show_no_redis_warning) {
return null;
}
return (
<div
role="alert"
className="flex items-start gap-3 border-b border-destructive/40 bg-destructive/10 px-4 py-3 text-sm text-destructive"
>
<TriangleAlert className="mt-0.5 size-5 shrink-0" aria-hidden="true" />
<div>
<p className="font-semibold">No Redis configured. Redis is highly recommended</p>
<p>
Rate limits, budgets, router state, and cache invalidation are per worker without Redis, so limits are
enforced once per worker and spend can overshoot.{" "}
<a className="underline" href={REDIS_DOCS_URL} target="_blank" rel="noreferrer">
See everything that does not work without Redis
</a>
. If you run a single worker and this is intentional, set{" "}
<code className="font-mono">LITELLM_DISABLE_NO_REDIS_WARNING=true</code> to hide this banner.
</p>
</div>
</div>
);
};

28
ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.test.tsx Normal file
View file

@ -0,0 +1,28 @@
import { render, screen } from "@testing-library/react";
import { describe, expect, it } from "vitest";
import RouterSettingsSummary from "./RouterSettingsSummary";
describe("RouterSettingsSummary", () => {
it("should list each configured fallback mapping", () => {
render(
<RouterSettingsSummary
routerSettings={{
fallbacks: [{ "gpt-4": ["gpt-4o", "claude-sonnet"] }, { "gpt-4o": ["gpt-4o-mini"] }],
num_retries: 3,
}}
/>,
);
expect(screen.getByText("gpt-4")).toBeInTheDocument();
expect(screen.getByText("gpt-4o, claude-sonnet")).toBeInTheDocument();
expect(screen.getByText("gpt-4o")).toBeInTheDocument();
expect(screen.getByText("gpt-4o-mini")).toBeInTheDocument();
expect(screen.getByText("Number of Retries: 3")).toBeInTheDocument();
});
it("should show the empty state when every setting is null", () => {
render(<RouterSettingsSummary routerSettings={{ fallbacks: null, num_retries: null }} />);
expect(screen.getByText("No router settings configured")).toBeInTheDocument();
});
});

56
ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx Normal file
View file

@ -0,0 +1,56 @@
import { Badge } from "@/components/ui/badge";
import { hasRouterSettings } from "./routerSettingsPayload";
interface RouterSettingsSummaryProps {
routerSettings: Record<string, unknown> | null | undefined;
emptyText?: string;
}
const fallbackEntries = (fallbacks: unknown): Array<[string, string[]]> => {
if (!Array.isArray(fallbacks)) return [];
return fallbacks.flatMap((entry) =>
entry && typeof entry === "object" ? (Object.entries(entry) as Array<[string, string[]]>) : [],
);
};
export default function RouterSettingsSummary({
routerSettings,
emptyText = "No router settings configured",
}: RouterSettingsSummaryProps) {
if (!hasRouterSettings(routerSettings)) {
return <div className="text-gray-400">{emptyText}</div>;
}
const settings = routerSettings as Record<string, unknown>;
const fallbacks = fallbackEntries(settings.fallbacks);
return (
<div className="space-y-1 text-sm">
{settings.routing_strategy != null && (
<div>
Routing Strategy: <Badge variant="secondary">{String(settings.routing_strategy)}</Badge>
</div>
)}
{settings.num_retries != null && <div>Number of Retries: {String(settings.num_retries)}</div>}
{settings.allowed_fails != null && <div>Allowed Failures: {String(settings.allowed_fails)}</div>}
{settings.cooldown_time != null && <div>Cooldown Time: {String(settings.cooldown_time)}s</div>}
{settings.timeout != null && <div>Timeout: {String(settings.timeout)}s</div>}
{settings.retry_after != null && <div>Retry After: {String(settings.retry_after)}s</div>}
{Boolean(settings.enable_tag_filtering) && <div>Tag Filtering: Enabled</div>}
{fallbacks.length > 0 && (
<div>
<div>Fallbacks:</div>
<div className="mt-1 space-y-1">
{fallbacks.map(([model, targets]) => (
<div key={model} className="text-xs text-gray-600">
<span className="font-medium">{model}</span>
<span className="mx-1 text-gray-400">-&gt;</span>
{Array.isArray(targets) ? targets.join(", ") : String(targets)}
</div>
))}
</div>
</div>
)}
</div>
);
}

115
ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts Normal file
View file

@ -0,0 +1,115 @@
import { describe, expect, it } from "vitest";
import { hasRouterSettings, routerSettingsEditorValue, routerSettingsUpdate } from "./routerSettingsPayload";
describe("hasRouterSettings", () => {
it("should treat unset, empty and all-null settings as absent", () => {
expect(hasRouterSettings(undefined)).toBe(false);
expect(hasRouterSettings(null)).toBe(false);
expect(hasRouterSettings({})).toBe(false);
expect(
hasRouterSettings({ num_retries: null, fallbacks: [], model_group_alias: {}, enable_tag_filtering: false }),
).toBe(false);
});
it("should detect configured settings", () => {
expect(hasRouterSettings({ num_retries: 3 })).toBe(true);
expect(hasRouterSettings({ fallbacks: [{ "gpt-4": ["gpt-4o"] }] })).toBe(true);
expect(hasRouterSettings({ enable_tag_filtering: true })).toBe(true);
expect(hasRouterSettings({ num_retries: 0 })).toBe(true);
});
});
describe("routerSettingsEditorValue", () => {
it("should hand the editor only the fields it renders", () => {
expect(
routerSettingsEditorValue({
num_retries: 2,
tag_routing_prefix: "team-",
fallbacks: [{ "gpt-4": ["gpt-4o"] }],
}),
).toStrictEqual({ router_settings: { num_retries: 2, fallbacks: [{ "gpt-4": ["gpt-4o"] }] } });
});
it("should keep an explicitly stored null so the editor renders it as empty", () => {
expect(routerSettingsEditorValue({ num_retries: null })).toStrictEqual({ router_settings: { num_retries: null } });
});
it("should leave the editor uninitialised when the key has no stored settings", () => {
expect(routerSettingsEditorValue(null)).toBeUndefined();
expect(routerSettingsEditorValue(undefined)).toBeUndefined();
});
});
describe("routerSettingsUpdate", () => {
const fallbacks = [{ "gpt-4": ["gpt-4o"] }];
// Accepted by UpdateRouterConfig on /key/update but not rendered by the accordion.
const unsupported = {
tag_routing_prefix: "team-",
model_group_retry_policy: { "gpt-4": { TimeoutErrorRetries: 2 } },
};
// The editor is a fixed-field form, so the merge has to hold two opposing guarantees at once:
// routing fields it cannot render survive, and a field it does render that the admin emptied
// is still sent as null. Orderings vary because an object spread resolves collisions by position.
const storedOrderings: Array<[string, Record<string, unknown>]> = [
["unsupported fields first", { ...unsupported, num_retries: 2, fallbacks }],
["unsupported fields last", { num_retries: 2, fallbacks, ...unsupported }],
[
"unsupported fields interleaved",
{
tag_routing_prefix: unsupported.tag_routing_prefix,
num_retries: 2,
model_group_retry_policy: unsupported.model_group_retry_policy,
fallbacks,
},
],
];
it.each(storedOrderings)(
"should keep unsupported stored fields and still clear an emptied editor field (%s)",
(_ordering, stored) => {
const result = routerSettingsUpdate({ num_retries: 4, fallbacks: null }, stored);
expect(result).toMatchObject({ ...unsupported, num_retries: 4, fallbacks: null });
},
);
it("should send an empty object, not a null blob, when the last stored setting is cleared", () => {
expect(routerSettingsUpdate({ fallbacks: null, num_retries: null }, { fallbacks, num_retries: 2 })).toEqual({});
});
it("should keep clearing owned fields explicitly while an unsupported setting still stands", () => {
expect(routerSettingsUpdate({ fallbacks: null, num_retries: null }, { fallbacks, ...unsupported })).toMatchObject({
...unsupported,
fallbacks: null,
num_retries: null,
});
});
it("should null every editor-owned field the editor left out", () => {
expect(routerSettingsUpdate({ fallbacks: null }, { timeout: 30, ...unsupported })).toEqual({
...unsupported,
routing_strategy: null,
allowed_fails: null,
cooldown_time: null,
num_retries: null,
timeout: null,
retry_after: null,
fallbacks: null,
context_window_fallbacks: null,
retry_policy: null,
model_group_alias: null,
enable_tag_filtering: null,
routing_strategy_args: null,
});
});
it("should send the edited settings when the user configured something", () => {
expect(routerSettingsUpdate({ fallbacks }, null)).toMatchObject({ fallbacks });
});
it("should leave the field off when nothing is stored and nothing was configured", () => {
expect(routerSettingsUpdate({ fallbacks: null, num_retries: null }, {})).toBeUndefined();
expect(routerSettingsUpdate(undefined, { fallbacks })).toBeUndefined();
});
});

63
ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts Normal file
View file

@ -0,0 +1,63 @@
import { RouterSettingsAccordionValue } from "./RouterSettingsAccordion";
export type RouterSettings = RouterSettingsAccordionValue["router_settings"];
const EDITOR_OWNED_FIELDS: Record<keyof RouterSettings, true> = {
routing_strategy: true,
allowed_fails: true,
cooldown_time: true,
num_retries: true,
timeout: true,
retry_after: true,
fallbacks: true,
context_window_fallbacks: true,
retry_policy: true,
model_group_alias: true,
enable_tag_filtering: true,
routing_strategy_args: true,
};
const EDITOR_OWNED_KEYS = Object.keys(EDITOR_OWNED_FIELDS) as Array<keyof RouterSettings>;
const isMeaningfulRouterSetting = (value: unknown): boolean => {
if (value === null || value === undefined || value === "" || value === false) return false;
if (Array.isArray(value)) return value.length > 0;
if (typeof value === "object") return Object.keys(value).length > 0;
return true;
};
export const hasRouterSettings = (settings: Record<string, unknown> | null | undefined): boolean =>
settings != null && Object.values(settings).some(isMeaningfulRouterSetting);
/**
* The stored settings narrowed to what the editor renders. The stored blob is untyped JSON
* from /key/info, so this projection is the one place it is read as RouterSettings.
*/
export const routerSettingsEditorValue = (
stored: Record<string, unknown> | null | undefined,
): RouterSettingsAccordionValue | undefined =>
stored
? {
router_settings: Object.fromEntries(
EDITOR_OWNED_KEYS.filter((key) => key in stored).map((key) => [key, stored[key]]),
) as RouterSettings,
}
: undefined;
/**
* Router settings to put on a /key/update payload, or undefined to leave the field off.
* The editor only renders EDITOR_OWNED_FIELDS, so its value is merged over the stored object
* rather than replacing it, and routing fields the editor cannot show survive an unrelated edit.
* Emptying every field sends {}, which the proxy reads as "no key-level override" so the key
* falls back to its team and global settings, where an all-null blob would pin it to nulls.
*/
export const routerSettingsUpdate = (
edited: RouterSettings | null | undefined,
stored: Record<string, unknown> | null | undefined,
): Record<string, unknown> | undefined => {
if (!edited) return undefined;
const editorOwned = Object.fromEntries(EDITOR_OWNED_KEYS.map((key) => [key, edited[key] ?? null]));
const merged: Record<string, unknown> = { ...stored, ...editorOwned };
if (hasRouterSettings(merged)) return merged;
return hasRouterSettings(stored) ? {} : undefined;
};

79
ui/litellm-dashboard/src/components/common_components/routerSettingsWiring.test.tsx Normal file
View file

@ -0,0 +1,79 @@
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import { render, screen, waitFor } from "@testing-library/react";
import type { ReactElement, ReactNode } from "react";
import { describe, expect, it, vi } from "vitest";
import type { FallbackGroup } from "../Settings/RouterSettings/Fallbacks/FallbackGroupConfig";
import type { RouterSettingsFormValue } from "../router_settings/RouterSettingsForm";
import RouterSettingsAccordion from "./RouterSettingsAccordion";
import { routerSettingsEditorValue } from "./routerSettingsPayload";
vi.mock("../networking", () => ({
getRouterSettingsCall: vi.fn().mockResolvedValue({}),
}));
vi.mock("@/components/llm_calls/fetch_models", () => ({
fetchAvailableModels: vi.fn().mockResolvedValue([{ model_group: "gpt-5.5" }, { model_group: "gpt-4o-mini" }]),
fetchAvailableModelsForTeam: vi.fn().mockResolvedValue([]),
}));
vi.mock("@tremor/react", () => ({
TabGroup: ({ children }: { children: ReactNode }) => <div>{children}</div>,
TabList: ({ children }: { children: ReactNode }) => <div>{children}</div>,
Tab: ({ children }: { children: ReactNode }) => <div>{children}</div>,
TabPanels: ({ children }: { children: ReactNode }) => <div>{children}</div>,
TabPanel: ({ children }: { children: ReactNode }) => <div>{children}</div>,
}));
vi.mock("../router_settings/RouterSettingsForm", () => ({
default: ({ value }: { value: RouterSettingsFormValue }) => (
<div data-testid="loadbalancing">{JSON.stringify(value.routerSettings)}</div>
),
}));
vi.mock("../Settings/RouterSettings/Fallbacks/FallbackSelectionForm", () => ({
FallbackSelectionForm: ({ groups }: { groups: FallbackGroup[] }) => (
<div data-testid="fallbacks">
{groups.map((g) => `${g.primaryModel ?? "none"}->${g.fallbackModels.join("|") || "none"}`).join(" ")}
</div>
),
}));
// Captured verbatim from GET /key/info on a live proxy for a key created through the UI.
// tag_routing_prefix is stored on the key and accepted by /key/update, but the accordion
// has no control for it, so the projection must drop it without losing the rest.
const KEY_INFO_ROUTER_SETTINGS: Record<string, unknown> = {
fallbacks: [{ "gpt-5.5": ["gpt-4o-mini"] }],
num_retries: 3,
tag_routing_prefix: "team-",
};
const renderAccordion = (stored: Record<string, unknown> | null): ReactElement => (
<QueryClientProvider client={new QueryClient({ defaultOptions: { queries: { retry: false } } })}>
<RouterSettingsAccordion accessToken="test-token" value={routerSettingsEditorValue(stored)} />
</QueryClientProvider>
);
describe("key router settings wiring, /key/info payload through to rendered output", () => {
it("should prefill both tabs from the stored settings", async () => {
render(renderAccordion(KEY_INFO_ROUTER_SETTINGS));
await waitFor(() => {
expect(screen.getByTestId("fallbacks")).toHaveTextContent("gpt-5.5->gpt-4o-mini");
});
expect(JSON.parse(screen.getByTestId("loadbalancing").textContent ?? "{}")).toMatchObject({ num_retries: 3 });
});
it("should not leak a field the accordion has no control for into the editor", async () => {
render(renderAccordion(KEY_INFO_ROUTER_SETTINGS));
await waitFor(() => expect(screen.getByTestId("loadbalancing")).toBeInTheDocument());
expect(screen.getByTestId("loadbalancing").textContent).not.toContain("tag_routing_prefix");
});
it("should render an empty editor for a key holding only fields it cannot show", async () => {
render(renderAccordion({ tag_routing_prefix: "team-" }));
await waitFor(() => expect(screen.getByTestId("fallbacks")).toHaveTextContent("none->none"));
expect(JSON.parse(screen.getByTestId("loadbalancing").textContent ?? "null")).toEqual({});
});
});

1
ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx
View file

@ -95,6 +95,7 @@ export interface KeyResponse {
object_permission?: ObjectPermission | null;
access_group_ids?: string[];
budget_fallbacks?: Record<string, string[]>;
router_settings?: Record<string, unknown> | null;
budget_limits?: Array<{ budget_duration: string; max_budget: number; reset_at?: string }>;
auto_rotate?: boolean;
rotation_interval?: string;

95
ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx
View file

@ -59,6 +59,24 @@ vi.mock("../organisms/create_key_button", () => ({
fetchTeamModels: vi.fn().mockResolvedValue(["team-model-1", "team-model-2"]),
}));
const routerSettingsMocks = vi.hoisted(() => ({
receivedValue: undefined as { router_settings: Record<string, unknown> } | undefined,
editedValue: null as Record<string, unknown> | null,
}));
vi.mock("../common_components/RouterSettingsAccordion", async () => {
const { forwardRef, useImperativeHandle } = await import("react");
return {
default: forwardRef(({ value }: { value?: { router_settings: Record<string, unknown> } }, ref) => {
routerSettingsMocks.receivedValue = value;
useImperativeHandle(ref, () => ({
getValue: () => ({ router_settings: routerSettingsMocks.editedValue ?? value?.router_settings ?? {} }),
}));
return <div data-testid="router-settings-accordion" />;
}),
};
});
vi.mock("@/app/(dashboard)/hooks/organizations/useOrganizations", () => ({
useOrganizations: vi.fn().mockReturnValue({
data: [
@ -160,6 +178,83 @@ describe("KeyEditView", () => {
last_rotation_at: undefined,
key_rotation_at: undefined,
};
describe("router settings", () => {
const UNSUPPORTED_STORED_FIELD = { tag_routing_prefix: "team-" };
const STORED_ROUTER_SETTINGS = {
num_retries: 2,
fallbacks: [{ "gpt-4": ["gpt-4o"] }],
...UNSUPPORTED_STORED_FIELD,
};
const renderWithRouterSettings = (onSubmit: (values: Record<string, unknown>) => Promise<void>) =>
renderWithProviders(
<KeyEditView
keyData={{ ...MOCK_KEY_DATA, router_settings: STORED_ROUTER_SETTINGS }}
onCancel={() => {}}
onSubmit={onSubmit}
accessToken="test-token"
userID="test-user"
userRole="proxy_admin"
premiumUser={true}
/>,
);
beforeEach(() => {
routerSettingsMocks.receivedValue = undefined;
routerSettingsMocks.editedValue = null;
});
it("should load the fields it renders into the editor and withhold the ones it does not", async () => {
renderWithRouterSettings(async () => {});
await waitFor(() => {
expect(routerSettingsMocks.receivedValue).toStrictEqual({
router_settings: { num_retries: 2, fallbacks: [{ "gpt-4": ["gpt-4o"] }] },
});
});
});
it("should submit edited fallbacks alongside routing fields the editor cannot show", async () => {
const onSubmit = vi.fn().mockResolvedValue(undefined);
renderWithRouterSettings(onSubmit);
routerSettingsMocks.editedValue = { num_retries: 2, fallbacks: [{ "gpt-4": ["gpt-4o", "gpt-4o-mini"] }] };
fireEvent.click(screen.getByText("Save Changes"));
await waitFor(() => {
expect(onSubmit).toHaveBeenCalledWith(
expect.objectContaining({
router_settings: expect.objectContaining({
...UNSUPPORTED_STORED_FIELD,
num_retries: 2,
fallbacks: [{ "gpt-4": ["gpt-4o", "gpt-4o-mini"] }],
}),
}),
);
});
});
it("should submit cleared router settings so removing every fallback is persisted", async () => {
const onSubmit = vi.fn().mockResolvedValue(undefined);
renderWithRouterSettings(onSubmit);
routerSettingsMocks.editedValue = { num_retries: null, fallbacks: null };
fireEvent.click(screen.getByText("Save Changes"));
await waitFor(() => {
expect(onSubmit).toHaveBeenCalledWith(
expect.objectContaining({
router_settings: expect.objectContaining({
...UNSUPPORTED_STORED_FIELD,
num_retries: null,
fallbacks: null,
}),
}),
);
});
});
});
it("should render", async () => {
const { getByText } = renderWithProviders(
<KeyEditView

22
ui/litellm-dashboard/src/components/templates/key_edit_view.tsx
View file

@ -6,7 +6,7 @@ import PolicySelector from "@/components/policies/PolicySelector";
import { InfoCircleOutlined } from "@ant-design/icons";
import { TextInput, Button as TremorButton } from "@tremor/react";
import { Form, Input, Select, Switch, Tooltip } from "antd";
import { useEffect, useState } from "react";
import { useEffect, useRef, useState } from "react";
import { hasCapability } from "../../utils/capabilities";
import { isProxyAdminRole, rolesWithWriteAccess } from "../../utils/roles";
import AgentSelector from "../agent_management/AgentSelector";
@ -17,6 +17,8 @@ import KeyLifecycleSettings from "../common_components/KeyLifecycleSettings";
import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector";
import RateLimitTypeFormItem from "../common_components/RateLimitTypeFormItem";
import OrganizationDropdown from "../common_components/OrganizationDropdown";
import RouterSettingsAccordion, { RouterSettingsAccordionRef } from "../common_components/RouterSettingsAccordion";
import { routerSettingsEditorValue, routerSettingsUpdate } from "../common_components/routerSettingsPayload";
import { extractLoggingSettings, formatMetadataForDisplay, stripTagsFromMetadata } from "../key_info_utils";
import { estimateFields, estimateRules, estimateTooltips, withNormalizedEstimates } from "./estimatedOutputTokens";
import { canonicalBudgetDuration, keyTypeFromRoutes } from "./keyEditFieldNormalizers";
@ -91,6 +93,7 @@ export function KeyEditView({
const [budgetFallbacks, setBudgetFallbacks] = useState<Record<string, string[]>>(
keyData.budget_fallbacks && typeof keyData.budget_fallbacks === "object" ? keyData.budget_fallbacks : {},
);
const routerSettingsRef = useRef<RouterSettingsAccordionRef>(null);
const { data: organizations, isLoading: isOrganizationsLoading } = useOrganizations();
const { data: projects } = useProjects();
const { data: uiSettingsData } = useUISettings();
@ -286,6 +289,14 @@ export function KeyEditView({
values.budget_fallbacks = {};
}
const routerSettings = routerSettingsUpdate(
routerSettingsRef.current?.getValue()?.router_settings,
keyData.router_settings,
);
if (routerSettings) {
values.router_settings = routerSettings;
}
await onSubmit(withNormalizedEstimates(values));
} finally {
setIsKeySaving(false);
@ -799,6 +810,15 @@ export function KeyEditView({
<Input value={projectDisplay ?? ""} disabled />
</Form.Item>
)}
<Form.Item label="Router Settings">
<RouterSettingsAccordion
ref={routerSettingsRef}
accessToken={accessToken || ""}
teamId={keyData.team_id}
value={routerSettingsEditorValue(keyData.router_settings)}
/>
</Form.Item>
<Form.Item label="Logging Settings" name="logging_settings">
<EditLoggingSettings
value={form.getFieldValue("logging_settings")}

19
ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx
View file

@ -189,6 +189,25 @@ describe("KeyInfoView", () => {
});
});
it("should render the key's saved router fallbacks", async () => {
vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock);
renderWithProviders(
<KeyInfoView
keyData={{ ...MOCK_KEY_DATA, router_settings: { num_retries: 2, fallbacks: [{ "gpt-4": ["gpt-4o"] }] } }}
onClose={() => {}}
keyId={"test-key-id"}
onKeyDataUpdate={() => {}}
teams={[]}
/>,
);
expect(await screen.findByText("Router Settings")).toBeInTheDocument();
expect(screen.getByText("gpt-4")).toBeInTheDocument();
expect(screen.getByText("gpt-4o")).toBeInTheDocument();
expect(screen.getByText("Number of Retries: 2")).toBeInTheDocument();
});
it("should render tags", async () => {
vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock);

11
ui/litellm-dashboard/src/components/templates/key_info_view.tsx
View file

@ -13,6 +13,8 @@ import { isProxyAdminRole, isUserTeamAdminForSingleTeam, rolesWithWriteAccess }
import { mapDisplayToInternalNames, mapInternalToDisplayNames } from "../callback_info_helpers";
import AutoRotationView from "../common_components/AutoRotationView";
import DeleteResourceModal from "../common_components/DeleteResourceModal";
import RouterSettingsSummary from "../common_components/RouterSettingsSummary";
import { hasRouterSettings } from "../common_components/routerSettingsPayload";
import { extractLoggingSettings, formatMetadataForDisplay, stripTagsFromMetadata } from "../key_info_utils";
import { KeyResponse } from "../key_team_helpers/key_list";
import LoggingSettingsView from "../logging_settings_view";
@ -837,6 +839,15 @@ export default function KeyInfoView({
</div>
)}
{hasRouterSettings(currentKeyData.router_settings) && (
<div>
<Text className="font-medium">Router Settings</Text>
<div className="mt-1">
<RouterSettingsSummary routerSettings={currentKeyData.router_settings} />
</div>
</div>
)}
<div>
<Text className="font-medium">Tags</Text>
<div className="flex flex-wrap gap-2 mt-1">