Merge remote-tracking branch 'origin/main' into litellm_pr49188_push_merge

This commit is contained in:
mateo-berri 2026-11-02 15:59:45 -08:00
commit d0cbfa460b
338 changed files with 19197 additions and 6476 deletions

View file

@ -104,6 +104,8 @@ attributes! {
/// Cloud regions the model is available in ('global' or region ids).
supported_regions: Vec<String>,
thinking_always_on: bool,
/// Whether the model thinks when the request carries no thinking param.
thinking_on_by_default: bool,
token_count_framing: u64 as NumericCount,
token_count_model: TokenCountModel,
token_count_responses_framing: u64 as NumericCount,
@ -301,6 +303,8 @@ mod tests {
#[case::known_mode("mode", json!("chat"), true)]
#[case::unknown_mode("mode", json!("chatt"), false)]
#[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)]
#[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)]
#[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)]
#[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)]
#[case::reasoning_modes("reasoning_modes", json!(["none", "adaptive"]), true)]
#[case::reasoning_context_not_a_list("reasoning_contexts", json!("reasoning"), false)]

View file

@ -673,7 +673,7 @@ class DualCache(BaseCache):
if self.redis_cache is not None:
self.redis_cache.flush_cache()
def delete_cache(self, key):
def delete_cache(self, key: str) -> None:
"""
Delete a key from the cache
"""

View file

@ -1,7 +1,8 @@
import asyncio
import os
import time
from collections.abc import Callable
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from functools import cache
from typing import Final
@ -15,6 +16,7 @@ from litellm.constants import (
AZURE_STORAGE_MSFT_VERSION,
)
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.azure.common_utils import get_azure_ad_token_from_entra_id
from litellm.llms.custom_httpx.http_handler import (
@ -47,6 +49,20 @@ def adls_safe_file_name(payload_id: str | None) -> str:
return f"{name.translate(_ADLS_SAFE_NAME)}.json"
@dataclass(frozen=True, slots=True)
class AzureStorageLogQueueItem:
payload: StandardLoggingPayload
file_name: str
def _queued_log(kwargs: Mapping[str, object], payload: StandardLoggingPayload) -> AzureStorageLogQueueItem:
payload_id: Final = payload.get("id")
return AzureStorageLogQueueItem(
payload=payload,
file_name=adls_safe_file_name(payload_id and logged_event_key(kwargs, payload_id)),
)
@cache
def _cached_credential_chain_token_provider() -> Callable[[], str]:
return get_azure_ad_token_provider(
@ -99,7 +115,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
asyncio.create_task(self.periodic_flush())
self.flush_lock = asyncio.Lock()
self.log_queue: list[StandardLoggingPayload] = []
self.log_queue: list[AzureStorageLogQueueItem] = []
super().__init__(**kwargs, flush_lock=self.flush_lock)
except Exception as e:
verbose_logger.exception(
@ -115,7 +131,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
def azure_storage_blob_endpoint(self) -> str:
return f"https://{self.azure_storage_account_name}.blob.{self.azure_storage_endpoint_suffix}"
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
async def async_log_success_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime):
"""
Async Log success events to Azure Blob Storage
@ -133,12 +149,12 @@ class AzureBlobStorageLogger(CustomBatchLogger):
if standard_logging_payload is None:
raise ValueError("standard_logging_payload is not set")
self.log_queue.append(standard_logging_payload)
self.log_queue.append(_queued_log(kwargs, standard_logging_payload))
except Exception as e:
verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
async def async_log_failure_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime):
"""
Async Log failure events to Azure Blob Storage
@ -156,7 +172,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
if standard_logging_payload is None:
raise ValueError("standard_logging_payload is not set")
self.log_queue.append(standard_logging_payload)
self.log_queue.append(_queued_log(kwargs, standard_logging_payload))
except Exception as e:
verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e)
@ -195,13 +211,15 @@ class AzureBlobStorageLogger(CustomBatchLogger):
len(self.log_queue),
)
for payload in self.log_queue:
await self.async_upload_payload_to_azure_blob_storage(payload=payload)
for item in self.log_queue:
await self.async_upload_payload_to_azure_blob_storage(payload=item.payload, file_name=item.file_name)
except Exception as e:
verbose_logger.exception("AzureBlobStorageLogger Error sending batch API - %s", e)
async def async_upload_payload_to_azure_blob_storage(self, payload: StandardLoggingPayload):
async def async_upload_payload_to_azure_blob_storage(
self, payload: StandardLoggingPayload, file_name: str | None = None
):
"""
Uploads the payload to Azure Blob Storage using a 3-step process:
1. Create file resource
@ -210,14 +228,14 @@ class AzureBlobStorageLogger(CustomBatchLogger):
"""
try:
if self.azure_storage_account_key:
await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload)
await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload, file_name=file_name)
else:
# Get a valid token instead of always requesting a new one
await self.set_valid_azure_ad_token()
async_client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
json_payload: Final = safe_dumps(payload) + "\n" # Add newline for each log entry
payload_bytes: Final = json_payload.encode("utf-8")
filename: Final = adls_safe_file_name(payload.get("id"))
filename: Final = file_name or adls_safe_file_name(payload.get("id"))
base_url = f"{self.azure_storage_dfs_endpoint}/{self.azure_storage_file_system}/{filename}"
# Execute the 3-step upload process
@ -377,7 +395,9 @@ class AzureBlobStorageLogger(CustomBatchLogger):
self._service_client_timeout = self._clock() + _DEFAULT_TTL_FOR_HTTPX_CLIENTS
return self._service_client
async def upload_to_azure_data_lake_with_azure_account_key(self, payload: StandardLoggingPayload):
async def upload_to_azure_data_lake_with_azure_account_key(
self, payload: StandardLoggingPayload, file_name: str | None = None
):
"""
Uploads the payload to Azure Data Lake using the Azure SDK
@ -386,11 +406,11 @@ class AzureBlobStorageLogger(CustomBatchLogger):
try:
today: Final = datetime.now().strftime("%Y-%m-%d")
file_name: Final = adls_safe_file_name(payload.get("id"))
resolved_file_name: Final = file_name or adls_safe_file_name(payload.get("id"))
await self._upload_bytes_with_account_key(
file_path=f"{today}/{file_name}", content=safe_dumps(payload).encode("utf-8")
file_path=f"{today}/{resolved_file_name}", content=safe_dumps(payload).encode("utf-8")
)
verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, file_name)
verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, resolved_file_name)
except Exception as e:
verbose_logger.exception("Error occurred: %s", e)

View file

@ -19,6 +19,7 @@ from litellm._logging import verbose_logger
from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger
from litellm.integrations.clickhouse.context import is_lens_analysis
from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
from litellm.tracing.types import SpendLogPayload, SpendLogRecord
@ -134,7 +135,7 @@ def spend_log_row_from_payload(
usage: Mapping[str, Any] = metadata.get("usage_object") or hidden_params.get("usage_object") or MappingProxyType({})
cache_read_tokens, cache_write_tokens = _cache_tokens(usage)
trace_id, span_id = _find_traceparent(metadata, kwargs)
request_id = str(payload.get("id") or "")
response_id: Final = str(payload.get("id") or "")
redact = litellm.turn_off_message_logging is True
track_end_user: Final = not litellm.disable_end_user_cost_tracking
completion_start_ms = _to_ms(payload.get("completionStartTime"))
@ -157,8 +158,8 @@ def spend_log_row_from_payload(
)
)
return SpendLogRecord(
request_id=request_id,
response_id=strip_cache_hit_suffix(request_id),
request_id=logged_event_key(kwargs, response_id),
response_id=strip_cache_hit_suffix(response_id),
provider_request_id=(
get_provider_request_id(kwargs.get("response_headers"))
or get_provider_request_id(hidden_params.get("additional_headers"))

View file

@ -15,6 +15,7 @@ from litellm.integrations.deepeval.utils import (
to_zod_compatible_iso,
validate_environment,
)
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
# This file includes the custom callbacks for LiteLLM Proxy
@ -109,8 +110,10 @@ class DeepEvalLogger(CustomLogger):
)
else:
output = str(standard_logging_object.get("error_string", ""))
logged_id: Final = standard_logging_object.get("id", uuid.uuid4())
span_id: Final = logged_event_key(kwargs, logged_id) if isinstance(logged_id, str) else logged_id
return BaseApiSpan(
uuid=standard_logging_object.get("id", uuid.uuid4()),
uuid=span_id,
name=("litellm_success_callback" if is_success else "litellm_failure_callback"),
status=(TraceSpanApiStatus.SUCCESS if is_success else TraceSpanApiStatus.ERRORED),
type=SpanApiType.LLM,

View file

@ -9,6 +9,8 @@ from typing import Final, Protocol
import litellm
from litellm._uuid import uuid
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
class _DynamoTable(Protocol):
@ -22,11 +24,13 @@ class _DynamoResource(Protocol):
class DyanmoDBLogger:
# Class variables or attributes
def __init__(self):
def __init__(self, dynamodb: _DynamoResource | None = None):
# Instance variables
import boto3
self.dynamodb: Final[_DynamoResource] = boto3.resource("dynamodb", region_name=os.environ["AWS_REGION_NAME"])
self.dynamodb: Final[_DynamoResource] = dynamodb or boto3.resource(
"dynamodb", region_name=os.environ["AWS_REGION_NAME"]
)
if litellm.dynamodb_table_name is None:
raise ValueError(
"LiteLLM Error, trying to use DynamoDB but not table name passed. Create a table and set `litellm.dynamodb_table_name=<your-table>`"
@ -45,23 +49,23 @@ class DyanmoDBLogger:
_async_log_event = async_log_event
def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
def log_event(self, kwargs: Mapping[str, object], response_obj, start_time, end_time, print_verbose):
try:
print_verbose(f"DynamoDB Logging - Enters logging function for model {kwargs}")
# construct payload to send to DynamoDB
# follows the same params as langfuse.py
litellm_params: Final = kwargs.get("litellm_params", {})
metadata: Final = litellm_params.get("metadata", {}) or {} # if litellm_params['metadata'] == None
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
messages: Final = kwargs.get("messages")
optional_params: Final = kwargs.get("optional_params", {})
call_type: Final = kwargs.get("call_type", "litellm.completion")
usage: Final = response_obj["usage"]
id: Final = response_obj.get("id", str(uuid.uuid4()))
response_id: Final = response_obj.get("id", str(uuid.uuid4()))
# Build the initial payload
payload: Final[dict[str, object]] = {
"id": id,
"id": logged_event_key(kwargs, response_id) if isinstance(response_id, str) else response_id,
"call_type": call_type,
"startTime": start_time,
"endTime": end_time,

View file

@ -17,6 +17,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
sanitize_cloud_object_component,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.gcs_bucket import *
from litellm.types.utils import StandardBatchLineItemsPayload, StandardLoggingPayload
@ -296,30 +297,28 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
else:
await self._send_individual_logs(items_to_process)
def _get_object_name(self, kwargs: dict, logging_payload: StandardLoggingPayload, response_obj: Any) -> str:
def _get_object_name(
self, kwargs: Mapping[str, object], logging_payload: StandardLoggingPayload, response_obj: Any
) -> str:
"""
Get the object name to use for the current payload
"""
current_date: Final = self._get_object_date_from_datetime(datetime.now(timezone.utc))
metadata: Final = as_str_mapping((as_str_mapping(kwargs.get("litellm_params")) or {}).get("metadata")) or {}
gcs_log_id: Final = metadata.get("gcs_log_id")
safe_log_id: Final = sanitize_cloud_object_component(
gcs_log_id if isinstance(gcs_log_id, str) else None, fallback=""
)
if safe_log_id:
return f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
if logging_payload.get("error_str", None) is not None:
object_name = self._generate_failure_object_name(
request_date_str=current_date,
)
else:
object_name = self._generate_success_object_name(
request_date_str=current_date,
response_id=response_obj.get("id", ""),
)
# used for testing
_litellm_params: Final = kwargs.get("litellm_params", None) or {}
_metadata: Final = _litellm_params.get("metadata", None) or {}
if "gcs_log_id" in _metadata:
safe_log_id: Final = sanitize_cloud_object_component(_metadata.get("gcs_log_id"), fallback="")
if safe_log_id:
object_name = f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
return object_name
return self._generate_failure_object_name(request_date_str=current_date)
raw_response_id: Final[object] = response_obj.get("id", "")
response_id: Final = raw_response_id if isinstance(raw_response_id, str) else ""
return self._generate_success_object_name(
request_date_str=current_date,
response_id=logged_event_key(kwargs, response_id),
)
async def get_request_response_payload(
self,

View file

@ -3,12 +3,15 @@
import json
import os
from collections.abc import Mapping
from typing import Final
import httpx
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.llms.custom_httpx.http_handler import (
HTTPHandler,
get_async_httpx_client,
@ -48,7 +51,7 @@ class OpenMeterLogger(CustomLogger):
if len(missing_keys) > 0:
raise Exception(f"Missing keys={missing_keys} in environment.")
def _common_logic(self, kwargs: dict, response_obj):
def _common_logic(self, kwargs: Mapping[str, object], response_obj):
call_id: Final = response_obj.get("id", kwargs.get("litellm_call_id"))
dt: Final = get_utc_datetime().isoformat()
cost: Final = kwargs.get("response_cost", None)
@ -74,8 +77,8 @@ class OpenMeterLogger(CustomLogger):
# If no user provided directly, try to get it from token user_id
if user_param is None:
# Check if user_id is available from the API key metadata
litellm_params: Final = kwargs.get("litellm_params", {})
metadata: Final = litellm_params.get("metadata", {})
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
user_api_key_user_id: Final = metadata.get("user_api_key_user_id", None)
if user_api_key_user_id is not None:
@ -89,7 +92,7 @@ class OpenMeterLogger(CustomLogger):
return {
"specversion": "1.0",
"type": os.getenv("OPENMETER_EVENT_TYPE", "litellm_tokens"),
"id": call_id,
"id": logged_event_key(kwargs, call_id) if isinstance(call_id, str) else call_id,
"time": dt,
"subject": subject,
"source": "litellm-proxy",

View file

@ -26,7 +26,12 @@ from litellm.constants import (
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
NON_INFERENCE_CALL_TYPES,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.initialize_dynamic_callback_params import initialize_standard_callback_dynamic_params
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
BackgroundResponseSettlement,
)
from litellm.types.utils import (
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN,
@ -35,6 +40,8 @@ from litellm.types.utils import (
BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX: Final = "_background_cost"
ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved"
OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved"
ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key"
@ -151,6 +158,26 @@ def is_unbilled_non_inference_call_from_params(
return is_unbilled_non_inference_call(call_type, metadata, response)
def settles_background_response(call_type: str, metadata: Mapping[str, object], kwargs: Mapping[str, object]) -> bool:
return kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES or (
call_type in NON_INFERENCE_CALL_TYPES
and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
)
def logged_event_key(kwargs: Mapping[str, object], event_id: str) -> str:
call_type: Final = kwargs.get("call_type")
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
metadata: Final = (
as_str_mapping(litellm_params.get("litellm_metadata")) or as_str_mapping(litellm_params.get("metadata")) or {}
)
if not event_id or not settles_background_response(
call_type if isinstance(call_type, str) else "", metadata, kwargs
):
return event_id
return f"{event_id}{BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX}"
def sanitize_user_api_key_auth(auth: object) -> object:
"""Copy of the auth object with its budget reservation removed; the cost callback
falls back to reading the reservation from inside the auth object."""

View file

@ -872,6 +872,7 @@ class Logging(LiteLLMLoggingBaseClass):
# The model the proxy shows the client on streamed chunks. The logged streamed response carries it
# once that response is priced, the same way a non-streamed response is logged
self.client_facing_stream_model: str | None = None
self.client_facing_response_id: str | None = None
self.zero_cost_warned: bool = False
self.llm_caching_handler: LLMCachingHandler | None = None
@ -3238,6 +3239,23 @@ class Logging(LiteLLMLoggingBaseClass):
return response
return response.model_copy(update={"id": response_id})
def _with_client_facing_response_id(
self,
response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
response_id: Final = self.client_facing_response_id
if response_id is None or getattr(response, "id", None) == response_id:
return response
return response.model_copy(update={"id": response_id})
def _client_facing_streaming_response(
self,
response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
return self._with_client_facing_response_id(
self._with_continued_response_id(self._with_client_facing_stream_model(response))
)
def _success_handler_helper_fn(
self,
result: object = None,
@ -3641,9 +3659,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
self._add_continued_partial_spend(complete_streaming_response)
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
logged_streaming_response: Final = self._with_continued_response_id(
self._with_client_facing_stream_model(complete_streaming_response)
)
logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response)
self.model_call_details["complete_streaming_response"] = logged_streaming_response
self.capture_usage_before_logging(logged_streaming_response, start_time, end_time)
## STANDARDIZED LOGGING PAYLOAD
@ -4240,9 +4256,7 @@ class Logging(LiteLLMLoggingBaseClass):
await self._prepare_baseline_cache_estimate(complete_streaming_response)
logged_streaming_response: Final = self._with_continued_response_id(
self._with_client_facing_stream_model(complete_streaming_response)
)
logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response)
self.model_call_details["async_complete_streaming_response"] = logged_streaming_response
## STANDARDIZED LOGGING PAYLOAD

View file

@ -13835,7 +13835,7 @@
"input_cost_per_token_flex": 3.7e-07,
"input_cost_per_token_priority": 1.295e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
@ -13849,7 +13849,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/ap-northeast-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -14006,7 +14006,7 @@
"input_cost_per_token_flex": 3.7e-07,
"input_cost_per_token_priority": 1.295e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
@ -14020,7 +14020,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/ap-south-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -14144,12 +14144,12 @@
"input_cost_per_token": 7.4e-07,
"input_cost_per_token_flex": 3.7e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
"output_cost_per_token_flex": 1.11e-06,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html",
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md",
"supports_audio_input": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
@ -14254,7 +14254,7 @@
"input_cost_per_token_flex": 3.7e-07,
"input_cost_per_token_priority": 1.295e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
@ -14268,7 +14268,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/eu-north-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -14790,7 +14790,7 @@
"input_cost_per_token_flex": 3.7e-07,
"input_cost_per_token_priority": 1.295e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
@ -14804,7 +14804,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/sa-east-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -15039,7 +15039,7 @@
"input_cost_per_token_flex": 3.1e-07,
"input_cost_per_token_priority": 1.085e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
@ -15053,7 +15053,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/us-east-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -15160,7 +15160,7 @@
"input_cost_per_token_flex": 3.1e-07,
"input_cost_per_token_priority": 1.085e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
@ -15174,7 +15174,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/us-east-2/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -15556,6 +15556,7 @@
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md"
},
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 2.65e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 8000,
@ -15566,6 +15567,7 @@
"supports_pdf_input": true
},
"bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 8000,
@ -15712,7 +15714,7 @@
"input_cost_per_token_flex": 3.1e-07,
"input_cost_per_token_priority": 1.085e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
@ -15726,7 +15728,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/us-west-2/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -23523,12 +23525,12 @@
]
},
"deepgram/streaming/nova-3": {
"input_cost_per_second": 8e-05,
"input_cost_per_second": 0.00012833,
"litellm_provider": "deepgram",
"metadata": {
"calculation": "$0.0048/60 seconds = $0.00008000 per second",
"calculation": "$0.0077/60 seconds = $0.00012833 per second",
"note": "Nova-3 monolingual streaming, pay as you go",
"original_pricing_per_minute": 0.0048
"original_pricing_per_minute": 0.0077
},
"mode": "audio_transcription",
"output_cost_per_second": 0.0,
@ -23537,13 +23539,28 @@
"/v1/listen"
]
},
"deepgram/streaming/nova-3-multilingual": {
"input_cost_per_second": 9.667e-05,
"deepgram/streaming/nova-3-medical": {
"input_cost_per_second": 0.00012833,
"litellm_provider": "deepgram",
"metadata": {
"calculation": "$0.0058/60 seconds = $0.00009667 per second",
"calculation": "$0.0077/60 seconds = $0.00012833 per second",
"note": "Nova-3 Medical streaming, launch price from Deepgram's 2025-03-03 announcement. deepgram.com/pricing does not list Nova-3 Medical, and this matches its current Nova-3 monolingual streaming pay as you go rate",
"original_pricing_per_minute": 0.0077
},
"mode": "audio_transcription",
"output_cost_per_second": 0.0,
"source": "https://www.businesswire.com/news/home/20250303470439/en",
"supported_endpoints": [
"/v1/listen"
]
},
"deepgram/streaming/nova-3-multilingual": {
"input_cost_per_second": 0.00015333,
"litellm_provider": "deepgram",
"metadata": {
"calculation": "$0.0092/60 seconds = $0.00015333 per second",
"note": "Nova-3 multilingual (language=multi) streaming, pay as you go",
"original_pricing_per_minute": 0.0058
"original_pricing_per_minute": 0.0092
},
"mode": "audio_transcription",
"output_cost_per_second": 0.0,
@ -24523,7 +24540,7 @@
"input_cost_per_token_flex": 3.1e-07,
"input_cost_per_token_priority": 1.085e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 164000,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
@ -24534,7 +24551,7 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json",
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md",
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false

View file

@ -127,6 +127,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
NO_TEAM_MEMBERSHIP_SENTINEL,
TAG_REGISTRY_OVERFLOW_SENTINEL,
UserApiKeyCache,
budget_windows_absent_cache_key,
end_user_cache_key,
end_user_restricted_registry_cache_key,
get_management_object_ttl,
@ -3480,11 +3481,22 @@ async def delete_cache_key_object(
)
await publish_auth_cache_invalidation(cache_key=key)
await evict_budget_windows_absent_marker(
entity_type=Litellm_EntityType.KEY, entity_id=hashed_token, user_api_key_cache=user_api_key_cache
)
_delete_cache_key_object: Final = delete_cache_key_object
async def evict_budget_windows_absent_marker(
entity_type: Litellm_EntityType, entity_id: str, user_api_key_cache: UserApiKeyCache
) -> None:
marker: Final = budget_windows_absent_cache_key(entity_type.value, entity_id)
evict_local(cache_keys=(marker,), user_api_key_cache=user_api_key_cache)
await publish_auth_cache_invalidation(cache_key=marker)
async def delete_cache_key_objects(
hashed_tokens: Sequence[str],
user_api_key_cache: UserApiKeyCache,

View file

@ -183,7 +183,7 @@ def evict_local(cache_keys: Sequence[str], user_api_key_cache: "UserApiKeyCache"
def _drop_from_memory(cache_key: str, user_api_key_cache: "UserApiKeyCache") -> None:
user_api_key_cache.in_memory_cache_for(cache_key).delete_cache(cache_key)
user_api_key_cache.delete_from_memory(cache_key)
@with_service_target(AUTH_OBJECTS_TARGET)

View file

@ -24,6 +24,7 @@ T = TypeVar("T", bound=BaseModel)
from litellm.constants import AUTH_OBJECTS_TARGET as AUTH_OBJECTS_TARGET # noqa: E402, PLC0414 # public re-export
_HASHED_TOKEN_CACHE_KEY: Final = re.compile(r"[0-9a-f]{64}")
_BUDGET_WINDOWS_ABSENT_PREFIX: Final = "budget_windows_absent:"
def is_user_key_cache_key(key: str) -> bool:
@ -70,6 +71,7 @@ class UserApiKeyCache(DualCache):
default_in_memory_ttl: float | None = None,
default_redis_ttl: float | None = None,
key_object_in_memory_cache: InMemoryCache | None = None,
budget_windows_absent_in_memory_cache: InMemoryCache | None = None,
) -> None:
super().__init__(
in_memory_cache=in_memory_cache,
@ -83,19 +85,33 @@ class UserApiKeyCache(DualCache):
default_in_memory_ttl=default_in_memory_ttl,
default_redis_ttl=default_redis_ttl,
)
self.budget_windows_absent_cache: Final = DualCache(
in_memory_cache=budget_windows_absent_in_memory_cache
or InMemoryCache(default_ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL),
default_in_memory_ttl=default_in_memory_ttl,
)
def in_memory_cache_for(self, key: str) -> InMemoryCache:
return self.key_object_cache.in_memory_cache if is_user_key_cache_key(key) else self.in_memory_cache
def delete_from_memory(self, key: str) -> None:
self.in_memory_cache_for(key).delete_cache(key)
if key.startswith(_BUDGET_WINDOWS_ABSENT_PREFIX):
self.budget_windows_absent_cache.delete_cache(key)
def update_cache_ttl(self, default_in_memory_ttl: float | None, default_redis_ttl: float | None) -> None:
super().update_cache_ttl(default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl)
self.key_object_cache.update_cache_ttl(
default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl
)
self.budget_windows_absent_cache.update_cache_ttl(
default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=None
)
def update_in_memory_max_size(self, max_size: int | None) -> None:
super().update_in_memory_max_size(max_size)
self.key_object_cache.update_in_memory_max_size(max_size)
self.budget_windows_absent_cache.update_in_memory_max_size(max_size)
def attach_redis_cache(
self, redis_cache: RedisCache | None = None, *, default_redis_ttl: float | None = None
@ -272,6 +288,7 @@ class UserApiKeyCache(DualCache):
def flush_cache(self) -> None:
super().flush_cache()
self.key_object_cache.in_memory_cache.flush_cache()
self.budget_windows_absent_cache.flush_cache()
async def async_set_cache_pipeline(
self, cache_list: Sequence[tuple[str, object]], local_only: bool = False, **kwargs: object
@ -400,6 +417,11 @@ def team_membership_reservation_cache_key(user_id: str, team_id: str) -> str:
NO_TEAM_MEMBERSHIP_SENTINEL: Final = "__no_team_membership__"
def budget_windows_absent_cache_key(entity_type: str, entity_id: str) -> str:
"""Cache key marking a key or team that holds no budget windows, so the spend fallback skips its row read."""
return f"{_BUDGET_WINDOWS_ABSENT_PREFIX}{entity_type}:{entity_id}"
def get_management_object_ttl(cache: DualCache) -> float:
"""
In-memory TTL for management-object cache writes (keys, teams, users, budgets, ...).

View file

@ -42,10 +42,12 @@ _EMPTY_ROWS: Final[Mapping[DbRow, Mapping[str, JsonValue]]] = MappingProxyType({
class SettingsStore(MutableMapping[str, JsonValue]):
def __init__(self, section: Section) -> None:
def __init__(self, section: Section, *, managed_db_keys: frozenset[str] | None = None) -> None:
self._section: Final = section
self._managed_db_keys: Final = managed_db_keys
self._yaml_values: Mapping[str, JsonValue] = _EMPTY_VALUES
self._database_rows: Mapping[DbRow, Mapping[str, JsonValue]] = _EMPTY_ROWS
self._unmanaged_db_keys: frozenset[str] = frozenset()
self._runtime_values: Mapping[str, JsonValue] = _EMPTY_VALUES
self._deleted_runtime_keys: frozenset[str] = frozenset()
@ -77,15 +79,24 @@ class SettingsStore(MutableMapping[str, JsonValue]):
return self.owned_by_config(key) and self._db_value_is_shadowed(key)
def apply_db_row(self, row: DbRow, db_row: Mapping[str, JsonValue]) -> None:
applied_row: Final = MappingProxyType({key: db_row[key] for key in db_row if self._db_manages(row, key)})
previous_row: Final = self._database_rows.get(row, _EMPTY_VALUES)
changed: Final = frozenset(
key
for key in (*previous_row, *db_row)
if previous_row.get(key, ABSENT) != db_row.get(key, ABSENT) # pyright: ignore[reportUnknownArgumentType] # JsonValue vs Absent compare
for key in (*previous_row, *applied_row)
if previous_row.get(key, ABSENT) != applied_row.get(key, ABSENT) # pyright: ignore[reportUnknownArgumentType] # JsonValue vs Absent compare
)
self._database_rows = MappingProxyType({**self._database_rows, row: MappingProxyType(dict(db_row))})
self._database_rows = MappingProxyType({**self._database_rows, row: applied_row})
if row == self._section:
self._unmanaged_db_keys = frozenset(key for key in db_row if key not in applied_row)
self._clear_runtime_keys(changed)
def unmanaged_db_keys(self) -> tuple[str, ...]:
return tuple(sorted(self._unmanaged_db_keys))
def _db_manages(self, row: DbRow, key: str) -> bool:
return row != self._section or self._managed_db_keys is None or key in self._managed_db_keys
def resolved(self) -> Mapping[str, JsonValue]:
return MappingProxyType(dict(self))

View file

@ -7,22 +7,25 @@ instead of writing immediately on each request.
from collections.abc import AsyncGenerator, Callable, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, TypeVar
from fastapi import HTTPException
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy._types import LitellmUserRoles
from litellm.proxy.common_utils.encrypt_decrypt_utils import (
decrypt_value_helper,
encrypt_value_helper,
)
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import (
BaseLiteLLMOpenAIResponseObject,
ResponsesAPIResponse,
)
from litellm.types.responses.main import DeleteResponseResult
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypesLiteral, LLMResponseTypes, SpecialEnums
if TYPE_CHECKING:
@ -67,6 +70,8 @@ def _proxy_signing_key() -> str | None:
_RESPONSE_PAYLOAD_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_ResponseObjectT = TypeVar("_ResponseObjectT", bound=BaseLiteLLMOpenAIResponseObject)
_AddressedResultT = TypeVar("_AddressedResultT", bound=ResponsesAPIResponse | DeleteResponseResult)
def _response_payload(response_obj: object) -> Mapping[str, object] | None:
@ -77,9 +82,9 @@ def _response_payload(response_obj: object) -> Mapping[str, object] | None:
def _rewrite_advertised_id(
event: BaseLiteLLMOpenAIResponseObject,
event: _ResponseObjectT,
rewrite: Callable[[str], str],
) -> BaseLiteLLMOpenAIResponseObject:
) -> _ResponseObjectT:
event_id: Final = getattr(event, "id", None)
if isinstance(event_id, str) and event_id.startswith("resp_"):
setattr(event, "id", rewrite(event_id))
@ -116,6 +121,35 @@ def _streams_response_ids(request_route: str | None) -> bool:
return canonical in _RESPONSES_API_ROUTES or (parent in _RESPONSES_API_ROUTES and response_id != "")
_NO_HELD_IDS: Final[Mapping[str, str]] = MappingProxyType({})
def _upstream_response_id(response_id: str) -> str:
return ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(response_id)
def _names_the_same_response(response_id: str | None, original_response_id: str) -> bool:
return response_id is not None and _upstream_response_id(response_id) == _upstream_response_id(original_response_id)
def _held_id_naming(response_id: str | None, held_ids: Mapping[str, str]) -> str | None:
return next(
(
held_id
for original_response_id, held_id in held_ids.items()
if _names_the_same_response(response_id, original_response_id)
),
None,
)
def _id_sent_back(response: ResponsesAPIResponse | DeleteResponseResult, held_ids: Mapping[str, str]) -> str | None:
held_id: Final = _held_id_naming(response.id, held_ids)
if held_id is not None:
return held_id
return response.id if isinstance(response, DeleteResponseResult) else None
class ResponsesIDSecurity(CustomLogger):
def __init__(
self,
@ -275,10 +309,18 @@ class ResponsesIDSecurity(CustomLogger):
def _encrypt_response_id(
self,
response: BaseLiteLLMOpenAIResponseObject,
response: _ResponseObjectT,
user_api_key_dict: "UserAPIKeyAuth",
request_cache: dict[str, str] | None = None,
) -> BaseLiteLLMOpenAIResponseObject:
) -> _ResponseObjectT:
encrypt: Final = self._response_id_encrypter(user_api_key_dict, request_cache)
return response if encrypt is None else _rewrite_advertised_id(response, encrypt)
def _response_id_encrypter(
self,
user_api_key_dict: "UserAPIKeyAuth",
request_cache: dict[str, str] | None,
) -> Callable[[str], str] | None:
signing_key: Final = self._get_signing_key()
if signing_key is None:
verbose_proxy_logger.debug(
@ -287,7 +329,7 @@ class ResponsesIDSecurity(CustomLogger):
"Skipping response ID encryption. "
"See: https://docs.litellm.ai/docs/proxy/prod#5-set-litellm-salt-key"
)
return response
return None
def encrypt(original_id: str) -> str:
cached: Final = request_cache.get(original_id) if request_cache is not None else None
@ -304,47 +346,73 @@ class ResponsesIDSecurity(CustomLogger):
request_cache[original_id] = encrypted_id
return encrypted_id
return _rewrite_advertised_id(response, encrypt)
return encrypt
def _streamed_response_id_advertiser(
self,
user_api_key_dict: "UserAPIKeyAuth",
request_data: Mapping[str, object],
) -> Callable[[str], str] | None:
if not _streams_response_ids(user_api_key_dict.request_route):
return None
if self._general_settings_reader().get("disable_responses_id_security", False):
return None
encrypt: Final = self._response_id_encrypter(user_api_key_dict, request_cache={})
if encrypt is None:
return None
logging_obj: Final = request_data.get("litellm_logging_obj")
held_ids: Final = self._ids_the_caller_holds(request_data.get(ADDRESSED_RESPONSE_ID_FIELD))
def advertise(original_id: str) -> str:
logged_id: Final = encrypt(original_id)
if isinstance(logging_obj, LiteLLMLoggingObj):
logging_obj.client_facing_response_id = logged_id
return _held_id_naming(original_id, held_ids) or logged_id
return advertise
def _ids_the_caller_holds(self, addressed_id: object) -> Mapping[str, str]:
if not isinstance(addressed_id, str) or not self._is_encrypted_response_id(addressed_id):
return _NO_HELD_IDS
original_response_id, _, _ = self._decrypt_response_id(addressed_id)
return MappingProxyType({original_response_id: addressed_id})
def _advertised_response(
self,
response: _AddressedResultT,
user_api_key_dict: "UserAPIKeyAuth",
request_data: Mapping[str, object],
) -> _AddressedResultT:
sent_back_id: Final = _id_sent_back(
response, self._ids_the_caller_holds(request_data.get(ADDRESSED_RESPONSE_ID_FIELD))
)
logged_response: Final = self._encrypt_response_id(response, user_api_key_dict, request_cache=None)
return logged_response if sent_back_id is None else logged_response.model_copy(update={"id": sent_back_id})
async def async_post_call_success_hook(
self,
data: dict,
data: dict[str, object],
user_api_key_dict: "UserAPIKeyAuth",
response: LLMResponseTypes,
) -> LLMResponseTypes:
"""
Queue response IDs for batch processing instead of writing directly to DB.
This method adds response IDs to an in-memory queue, which are then
batch-processed by the DBSpendUpdateWriter during regular database update cycles.
"""
general_settings: Final = self._general_settings_reader()
if general_settings.get("disable_responses_id_security", False):
return response
if isinstance(response, ResponsesAPIResponse):
response = cast(
ResponsesAPIResponse,
self._encrypt_response_id(response, user_api_key_dict, request_cache=None),
)
if isinstance(response, ResponsesAPIResponse | DeleteResponseResult):
return self._advertised_response(response, user_api_key_dict, data)
return response
async def async_post_call_streaming_iterator_hook(
self, user_api_key_dict: "UserAPIKeyAuth", response: Any, request_data: dict
self, user_api_key_dict: "UserAPIKeyAuth", response: Any, request_data: dict[str, object]
) -> AsyncGenerator[BaseLiteLLMOpenAIResponseObject, None]:
general_settings: Final = self._general_settings_reader()
# Create a request-scoped cache for consistent encryption across streaming chunks.
request_encryption_cache: Final[dict[str, str]] = {}
advertise: Final = self._streamed_response_id_advertiser(user_api_key_dict, request_data)
async for chunk in response:
if (
isinstance(chunk, BaseLiteLLMOpenAIResponseObject)
and _streams_response_ids(user_api_key_dict.request_route)
and not general_settings.get("disable_responses_id_security", False)
):
chunk = self._encrypt_response_id(chunk, user_api_key_dict, request_encryption_cache)
yield chunk
yield (
_rewrite_advertised_id(chunk, advertise)
if advertise is not None and isinstance(chunk, BaseLiteLLMOpenAIResponseObject)
else chunk
)
def decrypted_response_id(response_id: str) -> str:

View file

@ -37,6 +37,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
from litellm.proxy.route_llm_request import route_request
from litellm.types.images.main import ImageEditRequestParams
from litellm.types.llms.openai import ChatCompletionUserMessage
from litellm.types.utils import CallTypes
router: Final = APIRouter()
_IMAGE_GENERATION_MODEL_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object)
@ -221,7 +222,10 @@ async def image_generation(
return response
except Exception as e:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=data,
dispatched_call_type=CallTypes.aimage_generation.value,
)
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, HTTPException):

View file

@ -109,6 +109,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
can_org_access_model,
delete_cache_key_objects,
delete_cache_team_object,
evict_budget_windows_absent_marker,
get_jwt_key_mapping_cache_keys_for_tokens,
get_org_object,
get_team_membership,
@ -612,6 +613,9 @@ async def refresh_cached_team(
await publish_auth_cache_invalidation(cache_key=f"team_id:{team_row.team_id}")
if team_table.team_alias is not None:
await publish_auth_cache_invalidation(cache_key=f"team_alias:{team_table.team_alias}")
await evict_budget_windows_absent_marker(
entity_type=Litellm_EntityType.TEAM, entity_id=team_row.team_id, user_api_key_cache=user_api_key_cache
)
_refresh_cached_team: Final = refresh_cached_team

View file

@ -26,6 +26,7 @@ from collections.abc import (
Collection,
Container,
Iterable,
Iterator,
Mapping,
MutableMapping,
Sequence,
@ -206,6 +207,7 @@ from litellm.rust_bridge.catalog import Target
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.utils import (
PRICING_OVERRIDES_KEY,
CallTypes,
CountContextManagement,
LlmProviders,
ModelResponse,
@ -588,6 +590,7 @@ from litellm.proxy.common_utils.token_count_log import (
)
from litellm.proxy.common_utils.user_api_key_cache import (
UserApiKeyCache,
budget_windows_absent_cache_key,
end_user_cache_key,
get_management_object_ttl,
model_access_group_cache_key,
@ -3636,6 +3639,52 @@ def _cached_budget_limits(cached: object) -> object:
return _UNTYPED_VALUE.validate_python(json.loads(stored)) if isinstance(stored, str) else stored
def _has_budget_windows(budget_limits: object) -> bool:
return isinstance(budget_limits, list) and budget_limits != []
def _budget_windows_absent_marker(absent_key: str) -> object:
return _UNTYPED_VALUE.validate_python(user_api_key_cache.budget_windows_absent_cache.get_cache(key=absent_key))
def _finish_pending_read(absent_key: str, pending_read: str, remember_absent: bool) -> None:
if _budget_windows_absent_marker(absent_key) != pending_read:
return
if remember_absent:
user_api_key_cache.budget_windows_absent_cache.set_cache(key=absent_key, value=True)
return
user_api_key_cache.budget_windows_absent_cache.delete_cache(absent_key)
async def _stored_budget_limits(entity_type: Litellm_EntityType, entity_id: str) -> object:
"""Read only: caching the row here could race a delete and put a just-deleted key back into the auth cache."""
if prisma_client is None:
return None
absent_key: Final = budget_windows_absent_cache_key(entity_type.value, entity_id)
if _budget_windows_absent_marker(absent_key) is True:
return None
pending_read: Final = f"reading:{secrets.token_hex(8)}"
user_api_key_cache.budget_windows_absent_cache.set_cache(key=absent_key, value=pending_read)
try:
row: Final[SupportsModelDump | None] = (
await VerificationTokenRepository(prisma_client).table.find_unique(where={"token": entity_id})
if entity_type == Litellm_EntityType.KEY
else await TeamRepository(prisma_client).table.find_unique(where={"team_id": entity_id})
)
except Exception as e: # noqa: BLE001 # spend tracking must never fail the cost callback
verbose_proxy_logger.warning(
"Unable to read budget windows for %s=%s; this request's window spend is not counted: %s",
entity_type.value,
entity_id,
e,
)
_finish_pending_read(absent_key, pending_read, remember_absent=False)
return None
budget_limits: Final[object] = None if row is None else _cached_budget_limits(row)
_finish_pending_read(absent_key, pending_read, remember_absent=not _has_budget_windows(budget_limits))
return budget_limits
async def _increment_spend_counters_batched(
token: str | None,
team_id: str | None,
@ -3666,7 +3715,68 @@ async def _increment_spend_counters_batched(
cost: Final[float] = response_cost
async def _key_scope(key_token: str) -> tuple[PendingSpendIncrement | BaseException, ...]:
async def _window_increment(
entity_type: Litellm_EntityType, entity_id: str, window: object
) -> PendingSpendIncrement | None:
duration: Final = (
window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None)
)
window_reset_at: Final = (
window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None)
)
window_counter: Final = f"spend:{entity_type.value}:{entity_id}:window:{duration}"
window_start: Final = get_budget_window_start(window)
pending_window: Final = (
await _prepare_window_spend_counter_increment(
counter_key=window_counter,
entity_type=entity_type.value.capitalize(),
entity_id=entity_id,
window_duration=duration,
window_start=window_start,
increment=cost,
)
if window_counter not in reserved_counter_keys
else None
)
await _enqueue_window_spend_row_update(
entity_type=entity_type,
entity_id=entity_id,
reset_at=window_reset_at,
window_duration=duration,
window_start=window_start,
increment=cost,
request_started_at=request_started_at,
)
return pending_window
async def _window_increments(
entity_type: Litellm_EntityType, entity_id: str, budget_limits: object
) -> tuple[PendingSpendIncrement | BaseException, ...]:
if not isinstance(budget_limits, list):
return ()
window_pending: Final = await asyncio.gather(
*(_window_increment(entity_type, entity_id, window) for window in budget_limits), return_exceptions=True
)
return tuple(item for item in window_pending if item is not None)
async def _cached_window_increments(
entity_type: Litellm_EntityType, entity_id: str, cache_key: str
) -> tuple[_SpendScopeItem, ...]:
cached: Final[object] = _UNTYPED_VALUE.validate_python(await user_api_key_cache.async_get_cache(key=cache_key))
if cached is None:
return (_UncachedBudgetWindows(entity_type=entity_type, entity_id=entity_id),)
return await _window_increments(entity_type, entity_id, _cached_budget_limits(cached))
async def _stored_window_increments(
uncached: _UncachedBudgetWindows,
) -> tuple[PendingSpendIncrement | BaseException, ...]:
return await _window_increments(
uncached.entity_type,
uncached.entity_id,
await _stored_budget_limits(uncached.entity_type, uncached.entity_id),
)
async def _key_scope(key_token: str) -> tuple[_SpendScopeItem, ...]:
# key_token arrives pre-hashed from metadata["user_api_key"] (auth flow
# hashes raw "sk-..." keys before they reach the callback). The
# startswith("sk-") check is a safety net matching update_cache —
@ -3689,52 +3799,9 @@ async def _increment_spend_counters_batched(
)
)
async def _key_window_increment(window: object) -> PendingSpendIncrement | None:
duration = (
window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None)
)
key_window_reset_at = (
window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None)
)
key_window_counter: Final = f"spend:key:{hashed_token}:window:{duration}"
key_window_start = get_budget_window_start(window)
pending_window: Final = (
await _prepare_window_spend_counter_increment(
counter_key=key_window_counter,
entity_type="Key",
entity_id=hashed_token,
window_duration=duration,
window_start=key_window_start,
increment=cost,
)
if key_window_counter not in reserved_counter_keys
else None
)
await _enqueue_window_spend_row_update(
entity_type=Litellm_EntityType.KEY,
entity_id=hashed_token,
reset_at=key_window_reset_at,
window_duration=duration,
window_start=key_window_start,
increment=cost,
request_started_at=request_started_at,
)
return pending_window
return key_pending + await _cached_window_increments(Litellm_EntityType.KEY, hashed_token, hashed_token)
key_obj: Final[object] = _UNTYPED_VALUE.validate_python(
await user_api_key_cache.async_get_cache(key=hashed_token)
)
if key_obj is None:
return key_pending
key_budget_limits: Final = _cached_budget_limits(key_obj)
if not isinstance(key_budget_limits, list):
return key_pending
window_pending: Final = await asyncio.gather(
*(_key_window_increment(window) for window in key_budget_limits), return_exceptions=True
)
return key_pending + tuple(item for item in window_pending if item is not None)
async def _team_scope(scope_team_id: str) -> tuple[PendingSpendIncrement | BaseException, ...]:
async def _team_scope(scope_team_id: str) -> tuple[_SpendScopeItem, ...]:
team_counter_key: Final = f"spend:team:{scope_team_id}"
team_pending: Final[tuple[PendingSpendIncrement, ...]] = (
()
@ -3748,50 +3815,9 @@ async def _increment_spend_counters_batched(
)
)
async def _team_window_increment(window: object) -> PendingSpendIncrement | None:
duration = (
window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None)
)
team_window_reset_at = (
window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None)
)
team_window_counter: Final = f"spend:team:{scope_team_id}:window:{duration}"
team_window_start = get_budget_window_start(window)
pending_window: Final = (
await _prepare_window_spend_counter_increment(
counter_key=team_window_counter,
entity_type="Team",
entity_id=scope_team_id,
window_duration=duration,
window_start=team_window_start,
increment=cost,
)
if team_window_counter not in reserved_counter_keys
else None
)
await _enqueue_window_spend_row_update(
entity_type=Litellm_EntityType.TEAM,
entity_id=scope_team_id,
reset_at=team_window_reset_at,
window_duration=duration,
window_start=team_window_start,
increment=cost,
request_started_at=request_started_at,
)
return pending_window
team_obj: Final[object] = _UNTYPED_VALUE.validate_python(
await user_api_key_cache.async_get_cache(key=f"team_id:{scope_team_id}")
return team_pending + await _cached_window_increments(
Litellm_EntityType.TEAM, scope_team_id, f"team_id:{scope_team_id}"
)
if team_obj is None:
return team_pending
team_budget_limits: Final = _cached_budget_limits(team_obj)
if not isinstance(team_budget_limits, list):
return team_pending
window_pending: Final = await asyncio.gather(
*(_team_window_increment(window) for window in team_budget_limits), return_exceptions=True
)
return team_pending + tuple(item for item in window_pending if item is not None)
async def _team_member_scope(
scope_user_id: str, scope_team_id: str
@ -3862,22 +3888,30 @@ async def _increment_spend_counters_batched(
# return_exceptions so a failing scope does not leave its siblings running
# as orphaned tasks that race the caller's reservation-counter invalidation;
# all scopes settle, then the first error propagates as before.
scope_results: Final = await asyncio.gather(*scope_coros, return_exceptions=True)
scope_errors: Final = tuple(
item
for scope in scope_results
for item in (scope if isinstance(scope, tuple) else (scope,))
if isinstance(item, BaseException)
scope_items: Final = tuple(_flatten_spend_scopes(await asyncio.gather(*scope_coros, return_exceptions=True)))
uncached_windows: Final = tuple(item for item in scope_items if isinstance(item, _UncachedBudgetWindows))
await _apply_spend_counter_increments(
pending=reservation_update.pending
+ tuple(item for item in scope_items if isinstance(item, PendingSpendIncrement))
)
pending: Final = tuple(
item
for scope in scope_results
if not isinstance(scope, BaseException)
for item in scope
if not isinstance(item, BaseException)
stored_window_items: Final = (
tuple(
_flatten_spend_scopes(
await asyncio.gather(
*(_stored_window_increments(uncached) for uncached in uncached_windows), return_exceptions=True
)
)
)
if uncached_windows
else ()
)
await _apply_spend_counter_increments(pending=reservation_update.pending + pending)
stored_window_pending: Final = tuple(
item for item in stored_window_items if isinstance(item, PendingSpendIncrement)
)
if stored_window_pending:
await _apply_spend_counter_increments(pending=stored_window_pending)
stamp_budget_reservation_actual_cost(budget_reservation=budget_reservation, actual_cost=response_cost)
scope_errors: Final = tuple(item for item in scope_items + stored_window_items if isinstance(item, BaseException))
if scope_errors:
raise scope_errors[0]
@ -3885,6 +3919,25 @@ async def _increment_spend_counters_batched(
budget_reservation["finalized"] = True
@dataclass(frozen=True, slots=True)
class _UncachedBudgetWindows:
entity_type: Litellm_EntityType
entity_id: str
_SpendScopeItem: TypeAlias = PendingSpendIncrement | BaseException | _UncachedBudgetWindows
def _flatten_spend_scopes(
scope_results: Sequence[tuple[_SpendScopeItem, ...] | BaseException],
) -> Iterator[_SpendScopeItem]:
for scope in scope_results:
if isinstance(scope, BaseException):
yield scope
continue
yield from scope
@dataclass(frozen=True, slots=True)
class _ReservationCounterUpdate:
"""The reserved counters the direct increment must skip, and the adjustments that settle them on the actual
@ -5665,11 +5718,18 @@ def _with_config_file_pass_through_endpoints(
def _reload_settings_store(section: Section, store: SettingsStore, section_config: object) -> None:
serving_pass_throughs: Final = store.get("pass_through_endpoints")
store.load_yaml(_as_settings_mapping(section_config))
store.apply_db_row(section, _EMPTY_SETTINGS_MAPPING)
if is_resource_list(section, "pass_through_endpoints") and serving_pass_throughs is not None:
store["pass_through_endpoints"] = serving_pass_throughs
def _general_settings_row_is_readable(row: _ConfigParamValueRow | None) -> bool:
return row is None or row.param_value is None or isinstance(row.param_value, Mapping)
def _db_managed_general_settings_keys() -> frozenset[str]:
return frozenset((*ConfigGeneralSettings.model_fields, *_CLEANUP_SCHEDULE_KEYS, "allowed_ips", "email_settings"))
def _bind_general_settings_store(settings: SettingsStore) -> None:
global general_settings
general_settings = settings # pyright: ignore[reportAssignmentType] # legacy global accepts mappings
@ -5820,7 +5880,9 @@ class ProxyConfig:
# whether an existing request predates the prices it just fetched, and re-serving one
# costs a single fetch where skipping one leaves it priced wrong indefinitely
self.model_cost_map_applied_revision: int = 0
self.settings: Final[SettingsStore] = SettingsStore("general_settings")
self.settings: Final[SettingsStore] = SettingsStore(
"general_settings", managed_db_keys=_db_managed_general_settings_keys()
)
self.router_settings: Final[SettingsStore] = SettingsStore("router_settings")
self.litellm_settings: Final[SettingsStore] = SettingsStore("litellm_settings")
self.environment_variables: Final[SettingsStore] = SettingsStore("environment_variables")
@ -5831,6 +5893,8 @@ class ProxyConfig:
self._db_environment_keys: frozenset[str] = frozenset()
self._sso_environment_removals: frozenset[str] = frozenset()
self._warned_shadowed_keys: frozenset[tuple[Section, str]] = frozenset()
self._warned_unmanaged_keys: frozenset[str] = frozenset()
self._warned_unreadable_db_general_settings: bool = False
self._db_config_callback_entries: Mapping[tuple[str, str], _DbCallbackRegistration] = MappingProxyType({})
self._settings_stores: Final[Mapping[Section, SettingsStore]] = MappingProxyType(
{
@ -6476,6 +6540,9 @@ class ProxyConfig:
prisma_client=prisma_client,
store_model_in_db=store_model_in_db,
)
else:
for section, store in self._settings_stores.items():
store.apply_db_row(section, _EMPTY_SETTINGS_MAPPING)
config = self._config_with_resolved_settings(config)
@ -8354,15 +8421,20 @@ class ProxyConfig:
general_settings.get("maximum_spend_logs_retention_interval", "1d"),
)
async def _update_general_settings(self, db_general_settings: Mapping[str, SettingsJsonValue] | None) -> None:
async def _update_general_settings(self, db_general_settings: Mapping[str, SettingsJsonValue] | str | None) -> None:
global general_settings
if db_general_settings is None:
return
if not isinstance(db_general_settings, Mapping):
self._warn_about_unreadable_db_general_settings()
return
self._warned_unreadable_db_general_settings = False
if not isinstance(general_settings, SettingsStore):
self.settings.load_yaml(_as_settings_mapping(general_settings))
cache_size_was_db: Final = self.settings.source("user_api_key_cache_max_size") == "db"
previous_cleanup_schedule: Final = self._resolved_cleanup_schedule()
self.settings.apply_db_row("general_settings", db_general_settings)
self._warn_about_unmanaged_db_settings()
_bind_general_settings_store(self.settings)
await self._apply_general_settings_side_effects(
db_general_settings,
@ -8554,12 +8626,24 @@ class ProxyConfig:
environment_read: Final = next(self._environment_read_numbers)
sections: Final = tuple(self._settings_stores)
responses: Final = await asyncio.gather(*(get_config_param(prisma_client, section) for section in sections))
general_settings_response: Final = responses[sections.index("general_settings")]
general_settings_value: Final[Mapping[str, SettingsJsonValue]] = (
self._prepared_db_settings_values("general_settings", general_settings_response.param_value)
if general_settings_response is not None and general_settings_response.param_value is not None
else _EMPTY_SETTINGS_MAPPING
)
self._settings_stores["general_settings"].apply_db_row("general_settings", general_settings_value)
response: _ConfigParamValueRow | None
for section, response in zip(sections, responses):
if section == "general_settings":
continue
if response is None:
self._settings_stores[section].apply_db_row(section, _EMPTY_SETTINGS_MAPPING)
continue
param_value: JsonValue = getattr(response, "param_value", None)
if param_value is None:
self._settings_stores[section].apply_db_row(section, _EMPTY_SETTINGS_MAPPING)
continue
verbose_proxy_logger.debug(
@ -8570,6 +8654,7 @@ class ProxyConfig:
if section == "litellm_settings":
self._apply_litellm_settings_db_values(self._prepared_db_settings_values(section, param_value))
elif section == "environment_variables":
self._settings_stores[section].apply_db_row(section, _EMPTY_SETTINGS_MAPPING)
await self._apply_db_environment_variables_row(
prisma_client, response, environment_before_db, environment_read
)
@ -8580,6 +8665,7 @@ class ProxyConfig:
)
self._warn_about_shadowed_db_settings()
self._warn_about_unmanaged_db_settings()
return self._config_with_resolved_settings(config)
async def _apply_db_environment_variables_row(
@ -8638,6 +8724,30 @@ class ProxyConfig:
)
self._warned_shadowed_keys = shadowed
def _warn_about_unmanaged_db_settings(self) -> None:
unmanaged: Final = frozenset(
key for key in self.settings.unmanaged_db_keys() if key not in _GENERAL_SETTINGS_UI_LITELLM_FIELDS
)
for key in sorted(unmanaged - self._warned_unmanaged_keys):
verbose_proxy_logger.warning(
"general_settings.%s is stored in the database, but the proxy does not apply that stored value, so it "
"is ignored. %s",
key,
"The config file's value applies. Remove the stored copy with POST /config/field/delete."
if self.settings.owned_by_config(key)
else "Set it in the config file to use it, or remove it with POST /config/field/delete.",
)
self._warned_unmanaged_keys = unmanaged
def _warn_about_unreadable_db_general_settings(self) -> None:
if self._warned_unreadable_db_general_settings:
return
verbose_proxy_logger.warning(
"general_settings is stored in the database as something other than a JSON object, so the proxy "
"ignores every stored general setting."
)
self._warned_unreadable_db_general_settings = True
def _prepared_db_settings_values(self, section: Section, value: object) -> Mapping[str, SettingsJsonValue]:
scrubbed: Final = _scrub_db_overlay_remote_module_loads(section=section, db_value=value)
return _as_settings_mapping(scrubbed)
@ -8805,7 +8915,10 @@ class ProxyConfig:
)
# initialize vector stores, guardrails, etc. table in db
await self._init_non_llm_objects_in_db(prisma_client=prisma_client)
await self._init_non_llm_objects_in_db(
prisma_client=prisma_client,
refresh_pass_through_endpoints=_general_settings_row_is_readable(db_general_settings),
)
except Exception as e:
verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - %s", e)
@ -8874,7 +8987,9 @@ class ProxyConfig:
except Exception as e: # noqa: BLE001 # best-effort: a failing stop must not break proxy shutdown
verbose_proxy_logger.error("Error stopping auth cache invalidation subscriber: %s", e)
async def _init_non_llm_objects_in_db(self, prisma_client: PrismaClient):
async def _init_non_llm_objects_in_db(
self, prisma_client: PrismaClient, *, refresh_pass_through_endpoints: bool = True
):
"""
Use this to read non-llm objects from the db and initialize them
@ -8898,7 +9013,7 @@ class ProxyConfig:
if self._should_load_db_object(object_type="agents"):
await self._init_agents_in_db(prisma_client=prisma_client)
if self._should_load_db_object(object_type="pass_through_endpoints"):
if refresh_pass_through_endpoints and self._should_load_db_object(object_type="pass_through_endpoints"):
await self._init_pass_through_endpoints_in_db()
if self._should_load_db_object(object_type="prompts"):
@ -13446,7 +13561,10 @@ async def moderations(
return response
except Exception as e:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=data,
dispatched_call_type=CallTypes.amoderation.value,
)
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, ProxyException):
@ -13591,6 +13709,7 @@ async def audio_speech(
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=data,
dispatched_call_type=CallTypes.aspeech.value,
)
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, ProxyException):
@ -13742,7 +13861,10 @@ async def audio_transcriptions(
return response
except Exception as e:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=data,
dispatched_call_type=CallTypes.atranscription.value,
)
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, HTTPException):
@ -14840,6 +14962,7 @@ from litellm.repositories.table_repositories import (
)
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
from litellm.responses.litellm_completion_transformation.count_request import (
ChatCountBridgeError,
chat_count_request,
@ -21058,53 +21181,56 @@ async def delete_config_general_settings(
if data.field_name in _GENERAL_SETTINGS_UI_LITELLM_FIELDS:
return await _reset_general_settings_ui_litellm_field(data.field_name, user_api_key_dict)
if data.field_name not in ConfigGeneralSettings.model_fields:
config_table: Final[TableActions[_ConfigParamValueRow]] = _config_param_table(prisma_client)
stored_row: Final = await config_table.find_first(where={"param_name": "general_settings"})
stored: Final = (
_SETTINGS_MAPPING.validate_python(stored_row.param_value)
if stored_row is not None and isinstance(stored_row.param_value, Mapping)
else None
)
if not _general_settings_row_is_readable(stored_row):
raise HTTPException(
status_code=400,
detail={
"error": "general_settings is stored in the database as something other than a JSON object, so no "
"field can be removed from it"
},
)
if data.field_name not in ConfigGeneralSettings.model_fields and (stored is None or data.field_name not in stored):
raise HTTPException(
status_code=400,
detail={"error": f"Invalid field={data.field_name} passed in."},
)
proxy_config.reject_config_owned_deletes(section_name="general_settings", keys=(data.field_name,))
if data.field_name in _db_managed_general_settings_keys():
proxy_config.reject_config_owned_deletes(section_name="general_settings", keys=(data.field_name,))
## get general settings from db
db_general_settings: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
where={"param_name": "general_settings"}
)
### pop the value
if db_general_settings is None or db_general_settings.param_value is None:
if stored is None:
raise HTTPException(
status_code=400,
detail={"error": f"Field name={data.field_name} not in config"},
)
else:
general_settings = dict(db_general_settings.param_value)
before_general_settings: Final = copy.deepcopy(general_settings)
## update db
general_settings.pop(data.field_name, None)
remaining: Final = {key: value for key, value in stored.items() if key != data.field_name}
response: Final = await ConfigRepository(prisma_client).table.upsert(
where={"param_name": "general_settings"},
data={
"create": {
"param_name": "general_settings",
"param_value": json.dumps(general_settings),
"param_value": json.dumps(remaining),
},
"update": {"param_value": json.dumps(general_settings)},
"update": {"param_value": json.dumps(remaining)},
},
)
await invalidate_config_param("general_settings")
proxy_config.settings.apply_db_row("general_settings", general_settings)
proxy_config.settings.apply_db_row("general_settings", remaining)
if is_resource_list("general_settings", data.field_name):
stored_endpoints: Final = general_settings.get("pass_through_endpoints")
stored_endpoints: Final = remaining.get("pass_through_endpoints")
await proxy_config.serve_pass_through_endpoints(stored_endpoints if isinstance(stored_endpoints, list) else ())
asyncio.create_task(
create_config_audit_log(
"general_settings", "deleted", before_general_settings, general_settings, user_api_key_dict
)
create_config_audit_log("general_settings", "deleted", dict(stored), remaining, user_api_key_dict)
)
return response

View file

@ -20,6 +20,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
openai_error_param,
openai_error_type,
)
from litellm.types.utils import CallTypes
router: Final = APIRouter()
@ -114,7 +115,10 @@ async def rerank(
return response
except Exception as e:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=data,
dispatched_call_type=CallTypes.arerank.value,
)
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, HTTPException):

View file

@ -20,7 +20,6 @@ from litellm.constants import (
CALLER_SESSION_ID_METADATA_KEY,
CLI_SESSION_KEY_PREFIX,
EMPTY_MAPPING,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
KUBERNETES_POD_ROUTING_KEY,
LITELLM_PROXY_MASTER_KEY_ALIAS,
LITELLM_TRUNCATED_PAYLOAD_FIELD,
@ -30,7 +29,6 @@ from litellm.constants import (
MAX_SPEND_LOG_MODEL_NAME_LENGTH,
MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB,
MCP_SPEND_LOG_MODEL_PREFIX,
NON_INFERENCE_CALL_TYPES,
REDACTED_BY_LITELM_STRING,
SESSION_ID_OMITTED_METADATA_KEY,
UNKNOWN_MODEL_SPEND_LOG_MODEL,
@ -53,7 +51,11 @@ from litellm.litellm_core_utils.core_helpers import (
)
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.litellm_core_utils.hidden_params import served_from_cache
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call
from litellm.litellm_core_utils.internal_call_metadata import (
BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX,
is_unbilled_non_inference_call,
settles_background_response,
)
from litellm.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup,
UsageObservation,
@ -79,13 +81,8 @@ from litellm.responses.litellm_completion_transformation.compaction_items import
summary_only_compaction_content,
with_compaction_summary,
)
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
BackgroundResponseSettlement,
)
from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag
from litellm.types.utils import (
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
PROMPT_CARRYING_GUARDRAIL_FIELDS,
AzureSpillover,
CallTypes,
@ -347,24 +344,21 @@ def _get_spend_logs_metadata(
BATCH_COST_REQUEST_ID_SUFFIX: Final = "_batch_cost"
BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX: Final = "_background_cost"
def _cost_row_request_id_suffix(
call_type: str, metadata: Mapping[str, object], settles_background_response: bool
) -> str:
def _cost_row_request_id_suffix(call_type: str, metadata: Mapping[str, object], kwargs: Mapping[str, object]) -> str:
if call_type == CallTypes.aretrieve_batch.value:
return BATCH_COST_REQUEST_ID_SUFFIX
if settles_background_response or (
call_type in NON_INFERENCE_CALL_TYPES
and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
):
if settles_background_response(call_type, metadata, kwargs):
return BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX
return ""
def get_spend_logs_id(
call_type: str, response_obj: dict, kwargs: Mapping[str, object], metadata: Mapping[str, object] = EMPTY_MAPPING
call_type: str,
response_obj: Mapping[str, object],
kwargs: Mapping[str, object],
metadata: Mapping[str, object] = EMPTY_MAPPING,
) -> str | None:
standard_logging_payload = kwargs.get("standard_logging_object")
candidate_ids: Final = (
@ -377,10 +371,7 @@ def get_spend_logs_id(
)
if resolved_id is None:
return None
settles_background_response: Final = (
kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES
)
return f"{resolved_id}{_cost_row_request_id_suffix(call_type, metadata, settles_background_response)}"
return f"{resolved_id}{_cost_row_request_id_suffix(call_type, metadata, kwargs)}"
_MISSING_ATTRIBUTE: Final = object()

View file

@ -1760,7 +1760,10 @@ def _proxy_only_error_log_fields(
def _failure_fields_to_lift(
request_data: Mapping[str, object], provider_billed: bool = False, dispatched_call_type: str | None = None
request_data: Mapping[str, object],
provider_billed: bool = False,
dispatched_call_type: str | None = None,
route_call_type: str | None = None,
) -> Mapping[str, object]:
"""Failure-path callbacks run after ``litellm_logging_obj`` is popped from
request_data (it is not serialisable), so the caller merges these fields
@ -1770,7 +1773,8 @@ def _failure_fields_to_lift(
attribution on failed-request spend logs."""
_logging_obj: Final = request_data.get("litellm_logging_obj")
if _logging_obj is None:
return _EMPTY_LIFT
return _EMPTY_LIFT if dispatched_call_type is None else MappingProxyType({"call_type": dispatched_call_type})
_estimate_call_type: Final = dispatched_call_type or route_call_type
_model_call_details: Final = getattr(_logging_obj, "model_call_details", {})
_first_handoff: Final = _model_call_details.get("first_api_call_start_time")
_litellm_params: Final = getattr(_logging_obj, "litellm_params", None)
@ -1785,10 +1789,10 @@ def _failure_fields_to_lift(
provider_billed=_billed,
deployment_id=_deployment_id,
base_model=_base_model,
dispatched_call_type=dispatched_call_type,
dispatched_call_type=_estimate_call_type,
)
_prompt_estimate: Final = (
_estimated_prompt_usage(_model_call_details, request_data, dispatched_call_type)
_estimated_prompt_usage(_model_call_details, request_data, _estimate_call_type)
if _billed and _first_handoff is not None and not _model_call_details.get("failed_attempts_usage")
else None
)
@ -1808,7 +1812,7 @@ def _failure_fields_to_lift(
_billed,
_deployment_id,
_base_model,
dispatched_call_type,
_estimate_call_type,
)
if Logging.usage_capture_callback is not None
and (_billed or _model_call_details.get("combined_usage_object") is not None)
@ -4104,13 +4108,15 @@ class ProxyLogging:
route=route,
original_exception=original_exception,
error_type=error_type,
dispatched_call_type=dispatched_call_type,
)
request_data.update(
await offload_token_count(_failure_fields_to_lift)(
request_data,
provider_billed=_provider_billed_the_failure(original_exception),
dispatched_call_type=dispatched_call_type or _call_type_for_route(user_api_key_dict.request_route),
dispatched_call_type=dispatched_call_type,
route_call_type=_call_type_for_route(user_api_key_dict.request_route),
)
)
@ -4199,6 +4205,7 @@ class ProxyLogging:
route: str | None = None,
original_exception: Exception | None = None,
error_type: ProxyErrorTypes | None = None,
dispatched_call_type: str | None = None,
):
"""
Handle logging for proxy only errors by calling `litellm_logging_obj.async_failure_handler`
@ -4298,7 +4305,7 @@ class ProxyLogging:
litellm_logging_obj.model_call_details["input"] = input
body_shape_call_type = CallTypes.aembedding.value
resolved_call_type: Final = _call_type_for_route(route) or (
None if set_up_by_request else body_shape_call_type
None if set_up_by_request else dispatched_call_type or body_shape_call_type
)
if resolved_call_type is not None and litellm_logging_obj.call_type != CallTypes.pass_through.value:
litellm_logging_obj.call_type = resolved_call_type

View file

@ -1,14 +1,16 @@
import asyncio
import json
from collections.abc import Mapping, Sequence
from collections.abc import Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from itertools import chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, cast
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER
from litellm.litellm_core_utils.cold_storage_handler import ColdStorageHandler
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.responses.litellm_completion_transformation.client_tool_search import (
bridge_client_tool_search,
client_tool_search_loaded_tools,
@ -69,6 +71,90 @@ def _is_a_conversation_turn(spend_log: "SpendLogsPayload") -> bool:
return spend_log.get("call_type") not in _MCP_TOOL_CALL_TYPES and not _is_a_failed_turn(spend_log)
_RESPONSE_READ_CALL_TYPES: Final = frozenset(("aget_responses", "acancel_responses"))
def _output_item_id(item: object) -> str | None:
match item:
case {"id": str() as item_id}:
return item_id
case _:
return None
def _is_a_response_read(spend_log: "SpendLogsPayload") -> bool:
return "call_type" in spend_log and spend_log["call_type"] in _RESPONSE_READ_CALL_TYPES
def _stored_response(spend_log: "SpendLogsPayload") -> object:
return spend_log.get("response")
def _output_items(spend_log: "SpendLogsPayload") -> Iterator[tuple[str | None, object]]:
output: Final = _stored_responses_api_output(_stored_response(spend_log)) or []
return zip(map(_output_item_id, output), output)
def _identified_output_items(spend_log: "SpendLogsPayload") -> Iterator[tuple[str, object]]:
return ((item_id, item) for item_id, item in _output_items(spend_log) if item_id is not None)
def _output_item_ids(spend_log: "SpendLogsPayload") -> frozenset[str]:
return frozenset(item_id for item_id, _ in _identified_output_items(spend_log))
def _first_index_holding_each_item(indexed_turns: Iterable[tuple[int, "SpendLogsPayload"]]) -> Mapping[str, int]:
return dict(
chain.from_iterable(
((item_id, index) for item_id in _output_item_ids(turn)) for index, turn in reversed(tuple(indexed_turns))
)
)
def _turn_replaying_each_item(turns: Sequence["SpendLogsPayload"]) -> Mapping[str, int]:
indexed_turns: Final = tuple(enumerate(turns))
first_read_holding: Final = _first_index_holding_each_item(
(index, turn) for index, turn in indexed_turns if _is_a_response_read(turn)
)
first_other_turn_holding: Final = _first_index_holding_each_item(
(index, turn) for index, turn in indexed_turns if not _is_a_response_read(turn)
)
return MappingProxyType({**first_read_holding, **first_other_turn_holding})
def _latest_read_copy_of_each_item(turns: Sequence["SpendLogsPayload"]) -> Mapping[str, object]:
return MappingProxyType(
dict(chain.from_iterable(_identified_output_items(turn) for turn in turns if _is_a_response_read(turn)))
)
def _with_only_new_output(
index: int, turn: "SpendLogsPayload", turn_replaying: Mapping[str, int], latest_copy: Mapping[str, object]
) -> "SpendLogsPayload | None":
if not _is_a_response_read(turn):
return turn
stored_response: Final = as_str_mapping(_stored_response(turn))
new_output: Final = [
item if item_id is None else latest_copy[item_id]
for item_id, item in _output_items(turn)
if item_id is None or turn_replaying.get(item_id) == index
]
if stored_response is None or not new_output:
return None
replayed_turn: Final[SpendLogsPayload] = {**turn, "response": {**stored_response, "output": new_output}}
return replayed_turn
def _conversation_turns(spend_logs: Sequence["SpendLogsPayload"]) -> tuple["SpendLogsPayload", ...]:
turns: Final = tuple(filter(_is_a_conversation_turn, spend_logs))
turn_replaying: Final = _turn_replaying_each_item(turns)
latest_copy: Final = _latest_read_copy_of_each_item(turns)
replayed: Final = (
_with_only_new_output(index, turn, turn_replaying, latest_copy) for index, turn in enumerate(turns)
)
return tuple(turn for turn in replayed if turn is not None)
_MCP_AUTO_EXECUTE_OUTPUT_TYPES: Final = frozenset(("mcp_tools_fetched", "tool_execution_results"))
@ -101,7 +187,7 @@ class StoredTurns:
def _compaction_point(
index: int, spend_log: "SpendLogsPayload", proxy_server_request: Mapping[str, object] | None
) -> _ReplayStart | None:
if response_holds_compaction_block(spend_log.get("response")):
if response_holds_compaction_block(_stored_response(spend_log)):
return _ReplayStart(index=index, at_output=True)
if proxy_server_request is not None and holds_compaction_summary(proxy_server_request.get("input")):
return _ReplayStart(index=index, at_output=False)
@ -271,12 +357,10 @@ class ResponsesSessionHandler:
previous_response_id: str,
client_response_ids: tuple[str, ...] = (),
) -> StoredTurns:
spend_logs: Final = tuple(
spend_log
for spend_log in await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id(
spend_logs: Final = _conversation_turns(
await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id(
previous_response_id, client_response_ids
)
if _is_a_conversation_turn(spend_log)
)
return StoredTurns(
spend_logs=spend_logs,
@ -378,7 +462,7 @@ class ResponsesSessionHandler:
############################################################
# Add Output messages for this Spend Log
############################################################
_response_output: Final[object] = spend_log.get("response", "{}")
_response_output: Final = _stored_response(spend_log)
if (responses_api_output := _stored_responses_api_output(_response_output)) is not None:
chat_completion_message_history.extend(
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(

View file

@ -13835,7 +13835,7 @@
"input_cost_per_token_flex": 3.7e-07,
"input_cost_per_token_priority": 1.295e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
@ -13849,7 +13849,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/ap-northeast-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -14006,7 +14006,7 @@
"input_cost_per_token_flex": 3.7e-07,
"input_cost_per_token_priority": 1.295e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
@ -14020,7 +14020,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/ap-south-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -14144,12 +14144,12 @@
"input_cost_per_token": 7.4e-07,
"input_cost_per_token_flex": 3.7e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
"output_cost_per_token_flex": 1.11e-06,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html",
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md",
"supports_audio_input": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
@ -14254,7 +14254,7 @@
"input_cost_per_token_flex": 3.7e-07,
"input_cost_per_token_priority": 1.295e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
@ -14268,7 +14268,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/eu-north-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -14790,7 +14790,7 @@
"input_cost_per_token_flex": 3.7e-07,
"input_cost_per_token_priority": 1.295e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
@ -14804,7 +14804,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/sa-east-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -15039,7 +15039,7 @@
"input_cost_per_token_flex": 3.1e-07,
"input_cost_per_token_priority": 1.085e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
@ -15053,7 +15053,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/us-east-1/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -15160,7 +15160,7 @@
"input_cost_per_token_flex": 3.1e-07,
"input_cost_per_token_priority": 1.085e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
@ -15174,7 +15174,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/us-east-2/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -15556,6 +15556,7 @@
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md"
},
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 2.65e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 8000,
@ -15566,6 +15567,7 @@
"supports_pdf_input": true
},
"bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 8000,
@ -15712,7 +15714,7 @@
"input_cost_per_token_flex": 3.1e-07,
"input_cost_per_token_priority": 1.085e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
@ -15726,7 +15728,7 @@
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
},
"bedrock/us-west-2/minimax.minimax-m2.1": {
"deprecation_date": "2027-03-30",
@ -23523,12 +23525,12 @@
]
},
"deepgram/streaming/nova-3": {
"input_cost_per_second": 8e-05,
"input_cost_per_second": 0.00012833,
"litellm_provider": "deepgram",
"metadata": {
"calculation": "$0.0048/60 seconds = $0.00008000 per second",
"calculation": "$0.0077/60 seconds = $0.00012833 per second",
"note": "Nova-3 monolingual streaming, pay as you go",
"original_pricing_per_minute": 0.0048
"original_pricing_per_minute": 0.0077
},
"mode": "audio_transcription",
"output_cost_per_second": 0.0,
@ -23537,13 +23539,28 @@
"/v1/listen"
]
},
"deepgram/streaming/nova-3-multilingual": {
"input_cost_per_second": 9.667e-05,
"deepgram/streaming/nova-3-medical": {
"input_cost_per_second": 0.00012833,
"litellm_provider": "deepgram",
"metadata": {
"calculation": "$0.0058/60 seconds = $0.00009667 per second",
"calculation": "$0.0077/60 seconds = $0.00012833 per second",
"note": "Nova-3 Medical streaming, launch price from Deepgram's 2025-03-03 announcement. deepgram.com/pricing does not list Nova-3 Medical, and this matches its current Nova-3 monolingual streaming pay as you go rate",
"original_pricing_per_minute": 0.0077
},
"mode": "audio_transcription",
"output_cost_per_second": 0.0,
"source": "https://www.businesswire.com/news/home/20250303470439/en",
"supported_endpoints": [
"/v1/listen"
]
},
"deepgram/streaming/nova-3-multilingual": {
"input_cost_per_second": 0.00015333,
"litellm_provider": "deepgram",
"metadata": {
"calculation": "$0.0092/60 seconds = $0.00015333 per second",
"note": "Nova-3 multilingual (language=multi) streaming, pay as you go",
"original_pricing_per_minute": 0.0058
"original_pricing_per_minute": 0.0092
},
"mode": "audio_transcription",
"output_cost_per_second": 0.0,
@ -24523,7 +24540,7 @@
"input_cost_per_token_flex": 3.1e-07,
"input_cost_per_token_priority": 1.085e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 164000,
"max_input_tokens": 156000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
@ -24534,7 +24551,7 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json",
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md",
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false

View file

@ -26,7 +26,7 @@ dependencies = [
"tiktoken>=0.12.0,<1.0; python_version == '3.14'",
"tiktoken>=0.14.0,<1.0; python_version >= '3.15'",
"tokenizers>=0.21.0,<1.0",
"huggingface-hub>=0.34.0,<2.0",
"huggingface-hub>=0.34.0,<3.0",
"click>=8.0.0,<9.0",
"jinja2>=3.1.6,<4.0",
"aiohttp>=3.14.2,<4.0",

View file

@ -69,9 +69,6 @@ def _retrieve_response(proxy: ProxyClient, key: str, response_id: str) -> Respon
class TestResponsesRetrieve:
@pytest.mark.skip(
reason="stage red: product gap (LIT-5446), retrieve returns a different id than the stored response (non-idempotent response-id re-encryption)"
)
@pytest.mark.covers("llm.responses.openai.basic.nonstream.works")
@meta(
Subject(

View file

@ -630,19 +630,6 @@ def _spend_row(identity: str) -> Mapping[str, JsonValue]:
return rows[0]
def _spend_row_keyed_by_any(identities: frozenset[str]) -> Mapping[str, JsonValue]:
keys: Final = tuple(sorted(identities))
placeholders: Final = ", ".join("%s" for _ in keys)
rows: Final = eventually(
lambda: read_rows(
f'SELECT request_id, call_type, model FROM "LiteLLM_SpendLogs" WHERE request_id IN ({placeholders})', keys
),
lambda rows: len(rows) == 1,
seconds=70,
)
return rows[0]
def _spend_request_ids(identities: tuple[str, str, str]) -> frozenset[str]:
rows: Final = eventually(
lambda: read_rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE request_id IN (%s, %s, %s)', identities),
@ -815,7 +802,7 @@ def test_r07_responses_stream_through_the_async_openai_sdk(gateway: Gateway) ->
served: Final = _served(wire)
assert _routes(served) == (("POST", "/v1/responses"),), served
assert _body(served[0])["stream"] is True, served[0].body
assert _spend_row_keyed_by_any(response_identities(last.response.id))["model"] == DEPLOYMENT_MODEL
assert _spend_row(last.response.id)["model"] == DEPLOYMENT_MODEL
def test_r08_completions_non_stream_through_the_sync_openai_sdk(gateway: Gateway) -> None:

View file

@ -20,7 +20,7 @@ from pydantic import BaseModel, JsonValue, TypeAdapter
from tests.integration._support.client import Gateway, eventually, object_value, string_value
from tests.integration._support.database import read_rows, scratch_database
from tests.integration._support.process import owned_proxy
from tests.integration._support.process import owned_proxy, owned_proxy_process
from tests.integration._support.wire import Reply, Request, Wire, wire_server
pytestmark: Final = pytest.mark.shares_worker_state
@ -1701,3 +1701,57 @@ def test_db_environment_variables_saved_from_a_session_outside_the_tables_schema
assert isinstance(first, int) and isinstance(second, int)
assert 0 < first < second
def test_a_stored_general_setting_no_settings_api_manages_stays_off_and_the_rest_of_the_row_applies(
gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
alias: Final = "stale-database-setting-" + uuid.uuid4().hex
with scratch_database() as database_url, wire_server(lambda _request: _completion_reply("pong")) as provider:
monkeypatch.setenv("DATABASE_URL", database_url)
_push_schema()
with psycopg.connect(database_url, autocommit=True) as writer:
writer.execute(
'INSERT INTO "LiteLLM_Config" (param_name, param_value) VALUES (%s, %s)',
("general_settings", Jsonb({"enforce_user_param": True, "store_prompts_in_spend_logs": True})),
)
config: Final = tmp_path / "stale-database-setting.yaml"
config.write_text(
json.dumps(
{
"model_list": [
{
"model_name": alias,
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_base": provider.url + "/v1",
"api_key": "integration-provider-key",
},
}
],
"general_settings": {"master_key": gateway.key, "store_model_in_db": True},
}
)
)
with owned_proxy_process(
gateway,
tmp_path / "stale-database-setting",
{"DATABASE_URL": database_url, "DISABLE_SCHEMA_UPDATE": "true"},
config=config,
database_setup=(),
) as owned:
answer: Final = owned.gateway.request(
"POST", "/v1/chat/completions", {"model": alias, "messages": [{"role": "user", "content": "ping"}]}
)
listed: Final = owned.gateway.request("GET", "/config/list", params={"config_type": "general_settings"})
log: Final = owned.log.read_text()
assert answer.status_code == 200, answer.text
assert listed.status_code == 200, listed.text
stored_prompts: Final = next(
entry
for entry in TypeAdapter(list[dict[str, JsonValue]]).validate_json(listed.content)
if entry["field_name"] == "store_prompts_in_spend_logs"
)
assert (stored_prompts["field_value"], stored_prompts["source"]) == (True, "db")
assert "general_settings.enforce_user_param is stored in the database, but the proxy does not apply" in log

View file

@ -107,9 +107,11 @@ def test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once(gatewa
)
payloads: Final = tuple(json.loads(line) for line in events)
found: Final = next(
payload.get("response_id") or (payload.get("message") or {}).get("id") or payload.get("id")
(payload.get("response") or {}).get("id")
or (payload.get("message") or {}).get("id")
or payload.get("id")
for payload in payloads
if payload.get("response_id") or payload.get("message") or payload.get("id")
if payload.get("response") or payload.get("message") or payload.get("id")
)
return str(found)
response: Final = rig.candidate.client.post(path, json=body, headers={"Authorization": f"Bearer {key}"})

View file

@ -162,8 +162,13 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_responses_sse_stream(gatewa
for line in stream.iter_lines()
if line.startswith("data: ") and line != "data: [DONE]"
)
response_id: Final = next(
event["response_id"] for event in events if event.get("type") == "response.output_text.delta"
advertised: Final = frozenset(
event["response"]["id"]
for event in events
if event.get("type") in ("response.created", "response.completed")
)
assert len(advertised) == 1, events
response_id: Final = next(iter(advertised))
attrs: Final = exported_llm_span(rig.collector, response_id)
assert attrs["gen_ai.response.id"] == response_id, attrs
assert_identity(attrs, chat_identity_fields(identity))

View file

@ -1,5 +1,4 @@
import asyncio
import base64
import json
import re
import signal
@ -118,17 +117,6 @@ def _responses_reply(identity: str, stream: bool) -> Reply:
)
def _decoded_responses_id(identity: str) -> str:
try:
return base64.b64decode(identity.removeprefix("resp_").encode()).decode()
except (ValueError, UnicodeDecodeError):
return identity
def _canonical_id(identity: str) -> str:
return _decoded_responses_id(identity).rpartition("response_id:")[2]
def _sse_events(text: str) -> tuple[dict[str, JsonValue], ...]:
return tuple(
json.loads(line[6:]) for line in text.splitlines() if line.startswith("data: ") and line != "data: [DONE]"
@ -170,8 +158,7 @@ class Collector:
return tuple(
attributes
for attributes in self.attributes()
if isinstance(logged := attributes.get("gen_ai.response.id", {}).get("stringValue"), str)
and _canonical_id(logged) == _canonical_id(response_id)
if attributes.get("gen_ai.response.id", {}).get("stringValue") == response_id
)
def conversation_ids(self, response_id: str) -> tuple[str | None, ...]:
@ -183,10 +170,6 @@ class Collector:
def single_span(self, response_id: str) -> str | None:
return eventually(lambda: self.conversation_ids(response_id), lambda values: len(values) == 1, seconds=30)[0]
def logged_id(self, response_id: str) -> str:
spans: Final = eventually(lambda: self.spans(response_id), lambda values: len(values) == 1, seconds=30)
return str(spans[0]["gen_ai.response.id"]["stringValue"])
@pytest.fixture(scope="session")
def collector() -> Iterator[Collector]:
@ -437,12 +420,13 @@ def test_responses_stream_raw_metadata_session_id_lands_as_conversation_id(rig:
completed: Final = tuple(event for event in events if event["type"] == "response.completed")
assert len(completed) == 1, events
assert completed[0]["response"]["output"][0]["id"] == f"msg_resp_{marker}", completed
assert str(completed[0]["response"]["id"]).startswith("resp_"), completed
client_id: Final = str(completed[0]["response"]["id"])
assert client_id.startswith("resp_"), completed
assert rig.upstream_bodies(marker) == (
{"model": "gpt-4o-mini", "input": marker, "metadata": {"session_id": session}, "stream": True},
)
assert rig.sink.single_span(f"resp_{marker}") == session
assert rig.spend_session(rig.sink.logged_id(f"resp_{marker}")) == session
assert rig.sink.single_span(client_id) == session
assert rig.spend_session(client_id) == session
@pytest.mark.covers("other.observability.otel.conversation_id_from_metadata_chat_raw")
@ -689,15 +673,11 @@ def _burst(rig: Rig, count: int, session_for: Mapping[int, str]) -> tuple[tuple[
return tuple(pool.map(one, range(count)))
def _is_encrypted_responses_id(identity: str) -> bool:
return identity.startswith("resp_") and _decoded_responses_id(identity) == identity
def _landed(rig: Rig, expected: Mapping[str, str]) -> dict[str, tuple[str, ...]]:
spans: Final = rig.sink.attributes()
return {
session: tuple(
_canonical_id(str(attributes["gen_ai.response.id"]["stringValue"]))
str(attributes["gen_ai.response.id"]["stringValue"])
for attributes in spans
if attributes.get(CONVERSATION, {}).get("stringValue") == session and "gen_ai.response.id" in attributes
)
@ -707,21 +687,13 @@ def _landed(rig: Rig, expected: Mapping[str, str]) -> dict[str, tuple[str, ...]]
def _assert_exactly_once(rig: Rig, expected: Mapping[str, str], landed: Mapping[str, tuple[str, ...]]) -> None:
spend: Final = eventually(
lambda: {
session: tuple(_canonical_id(identity) for identity in rig.spend_request_ids(session))
for session in expected.values()
},
lambda: {session: rig.spend_request_ids(session) for session in expected.values()},
lambda rows: all(len(values) >= 1 for values in rows.values()),
seconds=70,
)
assert landed == spend, (landed, spend)
assert all(len(values) == 1 for values in landed.values()), landed
caller_visible: Final = {
session: (_canonical_id(identity),)
for identity, session in expected.items()
if not _is_encrypted_responses_id(identity)
}
assert {session: landed[session] for session in caller_visible} == caller_visible, landed
assert landed == {session: (identity,) for identity, session in expected.items()}, landed
@pytest.mark.covers("other.observability.otel.conversation_id_sink_outage_recovers_exactly_once")

View file

@ -1,5 +1,4 @@
import asyncio
import base64
import json
import re
import signal
@ -9,7 +8,6 @@ from collections import deque
from collections.abc import Iterator, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from functools import partial
from pathlib import Path
from typing import Final
@ -137,17 +135,6 @@ def _upstream(request: Request) -> Reply:
return _chat_reply(f"chatcmpl-{marker}", stream)
def _decoded_responses_id(identity: str) -> str:
try:
return base64.b64decode(identity.removeprefix("resp_").encode()).decode()
except (ValueError, UnicodeDecodeError):
return identity
def _canonical_id(identity: str) -> str:
return _decoded_responses_id(identity).rpartition("response_id:")[2]
def _sse_events(text: str) -> tuple[dict[str, JsonValue], ...]:
return tuple(
object_value(JSON.validate_json(line[6:]))
@ -232,8 +219,7 @@ class Collector:
return tuple(
span
for span in self.spans()
if RESPONSE_ID in span.attributes
and _canonical_id(span.attributes[RESPONSE_ID]) == _canonical_id(response_id)
if RESPONSE_ID in span.attributes and span.attributes[RESPONSE_ID] == response_id
)
def single_span(self, response_id: str, *, elsewhere: "Collector | None" = None) -> Span:
@ -249,11 +235,8 @@ class Collector:
def landed(self, response_ids: Sequence[str]) -> dict[str, int]:
spans: Final = self.spans()
return {
_canonical_id(identity): sum(
1
for span in spans
if RESPONSE_ID in span.attributes
and _canonical_id(span.attributes[RESPONSE_ID]) == _canonical_id(identity)
identity: sum(
1 for span in spans if RESPONSE_ID in span.attributes and span.attributes[RESPONSE_ID] == identity
)
for identity in response_ids
}
@ -448,7 +431,7 @@ def _assert_operator_span(rig: Rig, response_id: str, marker: str) -> Span:
span: Final = rig.sink.single_span(response_id)
assert span.target == "/v1/traces", span
assert span.ingestion_key == OPERATOR_KEY, span
assert rig.tenant_sink.landed((response_id,)) == {_canonical_id(response_id): 0}
assert rig.tenant_sink.landed((response_id,)) == {response_id: 0}
bodies: Final = rig.upstream_bodies(marker)
assert len(bodies) == 1, bodies
assert "signoz" not in json.dumps(bodies[0]).replace(marker, ""), bodies[0]
@ -522,7 +505,7 @@ def test_responses_stream_raw_httpx_span_lands_once_after_the_stream_is_consumed
marker: Final = _marker()
response: Final = rig.proxy.request("POST", "/v1/responses", {"model": rig.model, "input": marker, "stream": True})
assert response.status_code == 200, response.text
_assert_operator_span(rig, _responses_id(response, marker), marker)
_assert_operator_span(rig, _responses_id(response), marker)
def test_v2_flag_on_still_delivers_the_operator_span_with_the_ingestion_key(v2_rig: Rig) -> None:
@ -821,7 +804,7 @@ def _chat_id(response: httpx.Response) -> str:
return next(iter(identities))
def _responses_id(response: httpx.Response, marker: str) -> str:
def _responses_id(response: httpx.Response) -> str:
if not _is_event_stream(response):
return _body_id(response)
completed: Final = tuple(
@ -830,7 +813,7 @@ def _responses_id(response: httpx.Response, marker: str) -> str:
if event.get("type") == "response.completed"
)
assert len(completed) == 1 and completed[0].startswith("resp_"), response.text
return f"resp_{marker}"
return completed[0]
def _message_id(response: httpx.Response) -> str:
@ -852,7 +835,7 @@ def _burst(rig: Rig, count: int) -> tuple[tuple[int, str, str | None], ...]:
stream: Final = index % 2 == 0
path, body, identity_of = (
("/v1/chat/completions", {"model": rig.model, "messages": [{"role": "user", "content": marker}]}, _chat_id),
("/v1/responses", {"model": rig.model, "input": marker}, partial(_responses_id, marker=marker)),
("/v1/responses", {"model": rig.model, "input": marker}, _responses_id),
(
"/v1/messages",
{"model": rig.model, "max_tokens": 16, "messages": [{"role": "user", "content": marker}]},
@ -874,8 +857,8 @@ def _assert_exactly_once(rig: Rig, identities: Sequence[str]) -> None:
landed: Final = eventually(
lambda: rig.sink.landed(identities), lambda seen: all(count >= 1 for count in seen.values()), seconds=80
)
assert landed == {_canonical_id(identity): 1 for identity in identities}, landed
charged: Final = frozenset(identity for identity in identities if not identity.startswith("resp_"))
assert landed == {identity: 1 for identity in identities}, landed
charged: Final = frozenset(identities)
spend: Final = eventually(
lambda: read_rows(
'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE request_id = ANY(%s::text[])',

View file

@ -62,12 +62,12 @@ def _body(request: Request) -> dict[str, JsonValue]:
# ciphertext the caller received, because the spend row id is read from response_obj["id"] before the
# ResponsesIDSecurity hook rewrites it in place; the row is looked up under both ids until that ordering is fixed on
# main
def _spend_row(client_id: str, issued_id: str) -> dict[str, JsonValue]:
def _spend_row(*request_ids: str) -> dict[str, JsonValue]:
rows: Final = eventually(
lambda: read_rows(
'SELECT model_group, status, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" '
"WHERE request_id = ANY(%s)",
([client_id, issued_id],), # pyright: ignore[reportArgumentType] # psycopg adapts the list to a text array
(list(request_ids),), # pyright: ignore[reportArgumentType] # psycopg adapts the list to a text array
),
lambda found: len(found) == 1,
seconds=70,
@ -120,4 +120,4 @@ async def test_async_openai_sdk_responses_stream_is_served_by_the_native_respons
assert issued.upstream == f"resp_upstream_{marker}", completed.response.id
request: Final = _native_request(wire)
assert _body(request) == {"model": GPT, "input": _prompt(marker), "stream": True}, request.body
assert _spend_row(completed.response.id, issued.issued) == _success_row(model)
assert _spend_row(completed.response.id) == _success_row(model)

View file

@ -516,7 +516,7 @@ def test_responses_max_output_tokens_is_forwarded_natively_on_both_legs(gateway:
for attempt in attempts:
assert attempt.body["max_output_tokens"] == 1000 and _limits(attempt.body) == (None, None), attempt.body
_assert_logged(issued, model, "success")
_assert_logged(_managed_response_id(streamed_issued), model, "success")
_assert_logged(streamed_issued, model, "success")
def test_region_prefixed_deployment_on_a_v1_base_sends_the_bare_model_with_max_completion_tokens(

View file

@ -301,11 +301,7 @@ def _spend_row_id(rows: Sequence[Mapping[str, JsonValue]], tag: str) -> str:
def _landed_once(served: tuple[tuple[str, str, bool], ...], tags: tuple[str, ...]) -> None:
"""One spend row per tagged call, carrying the id the client saw.
FIXME: a streamed /v1/responses row carries the managed `resp_` id LiteLLM built before the proxy encrypted the
advertised one, so that row is matched by tag alone until the spend log reads the id the client received.
"""
"""One spend row per tagged call, carrying the id the client saw."""
rows: Final = eventually(
lambda: read_rows(
'SELECT request_id, request_tags FROM "LiteLLM_SpendLogs" WHERE request_tags ?| %s',
@ -316,9 +312,6 @@ def _landed_once(served: tuple[tuple[str, str, bool], ...], tags: tuple[str, ...
)
for (identity, path, stream), tag in zip(served, tags, strict=True):
row_id: Final = _spend_row_id(rows, tag)
if path == "/v1/responses" and stream:
assert row_id.startswith("resp_"), (row_id, identity)
continue
assert row_id == identity, (path, stream, row_id, identity)

View file

@ -172,7 +172,11 @@ def _upstream_id_inside(row_id: str) -> str | None:
def _row_belongs_to(row_id: str, served: _Served) -> bool:
if row_id == _response_id(served):
return True
return served.call.endpoint == "responses" and _upstream_id_inside(row_id) == f"resp_upstream_{served.call.marker}"
return (
served.call.endpoint == "responses"
and not served.call.stream
and _upstream_id_inside(row_id) == f"resp_upstream_{served.call.marker}"
)
def _assert_each_success_landed_once(rows: list[dict[str, JsonValue]], served: tuple[_Served, ...]) -> None:

View file

@ -380,7 +380,9 @@ def test_cut_chat_stream_hands_its_token_on_a_chunk_with_empty_vertex_ai_lists_a
chunks: Final = payloads(post(gateway, "chat", request_body("chat", model, True)).text, True)
carriers: Final = [chunk for chunk in chunks if "continuation_token" in json.dumps(chunk)]
assert [_vertex_lists(chunk) for chunk in carriers] == [dict(_EMPTY_VERTEX_LISTS)], chunks
assert [_vertex_lists(chunk) for chunk in chunks if chunk not in carriers] == [{}] * (len(chunks) - 1), chunks
assert [_vertex_lists(chunk) for chunk in chunks] == [
dict(_EMPTY_VERTEX_LISTS) if at(chunk, "choices", 0, "delta", "content") else {} for chunk in chunks
], chunks
metered: Final = [chunk for chunk in chunks if isinstance(chunk.get("usage"), dict)]
assert [at(chunk, "usage", "completion_tokens_details") for chunk in metered] == [
{"text_tokens": CUT.output_tokens}

View file

@ -203,9 +203,6 @@ def test_retrieve_routes_to_the_deployment_that_created_the_response(
@pytest.mark.parametrize("prefix", _ALIASES)
def test_retrieve_returns_the_id_the_client_holds(gateway: Gateway, prefix: str) -> None:
pytest.skip(
"BUG: GET /v1/responses/{id} returns a re-encrypted id that differs byte-for-byte from the id the client sent"
)
raw_id: Final = f"resp_retrieve_id_{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
@ -697,7 +694,6 @@ def test_delete_returns_the_client_held_id_not_the_provider_id(
gateway: Gateway,
prefix: str,
) -> None:
pytest.skip("BUG: DELETE /v1/responses/{id} returns the raw provider id instead of the proxy-issued id")
raw_id: Final = f"resp_delete_id_{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:

View file

@ -0,0 +1,465 @@
import json
from collections.abc import Mapping
from datetime import datetime, timedelta, timezone
from hashlib import sha256
from pathlib import Path
from types import MappingProxyType
from typing import Final
from uuid import uuid4
import httpx
import pytest
import yaml
from integration._support.client import JSON_OBJECT, Gateway, eventually, list_value, object_value, string_value
from integration._support.process import owned_proxy
from integration._support.responses_stream import frame, response_object
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_NATIVE_MODEL: Final = "openai/gpt-5.6"
_BRIDGED_MODEL: Final = "deepseek/deepseek-v4-pro"
_ANSWER: Final = "streamed answer"
_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": 30, "output_tokens": 5, "total_tokens": 35}
_JSON_LIST: Final = TypeAdapter(list[dict[str, JsonValue]])
def _sse(payload: dict[str, JsonValue]) -> bytes:
return f"data: {json.dumps(payload)}\n\n".encode()
def _native_completed(upstream_id: str) -> dict[str, JsonValue]:
message: Final[dict[str, JsonValue]] = {
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": _ANSWER, "annotations": []}],
}
return response_object(upstream_id, "completed", output=[message], usage=_USAGE)
def _native_stream(upstream_id: str) -> Reply:
return Reply(
content_type="text/event-stream",
chunks=(
frame(
{
"type": "response.created",
"sequence_number": 0,
"response": response_object(upstream_id, "in_progress"),
}
),
frame(
{
"type": "response.output_text.delta",
"sequence_number": 1,
"item_id": "msg_1",
"output_index": 0,
"content_index": 0,
"delta": _ANSWER,
}
),
frame({"type": "response.completed", "sequence_number": 2, "response": _native_completed(upstream_id)}),
),
)
def _chat_stream(upstream_id: str, model: str) -> Reply:
def chunk(delta: dict[str, JsonValue], finish_reason: str | None) -> dict[str, JsonValue]:
return {
"id": upstream_id,
"object": "chat.completion.chunk",
"created": 1,
"model": model,
"choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}],
}
return Reply(
content_type="text/event-stream",
chunks=(
_sse(chunk({"role": "assistant", "content": _ANSWER}, None)),
_sse(
{
**chunk({}, "stop"),
"usage": {"prompt_tokens": 30, "completion_tokens": 5, "total_tokens": 35},
}
),
b"data: [DONE]\n\n",
),
)
def _respond(request: Request) -> Reply:
if request.method == "GET" and request.target == "/v1/models":
return Reply(body=json.dumps({"object": "list", "data": []}).encode())
if request.method == "GET":
stored_id: Final = request.target.removeprefix("/v1/responses/")
assert stored_id.startswith("resp_upstream-"), request.target
return Reply(body=json.dumps(_native_completed(stored_id)).encode())
if request.target.endswith("/cancel"):
cancelled_id: Final = request.target.removeprefix("/v1/responses/").removesuffix("/cancel")
assert cancelled_id.startswith("resp_upstream-"), request.target
return Reply(body=json.dumps(response_object(cancelled_id, "cancelled")).encode())
body: Final = JSON_OBJECT.validate_json(request.body)
upstream_id: Final = f"upstream-{uuid4().hex}"
if request.target == "/v1/responses" and body.get("stream") is not True:
return Reply(body=json.dumps(_native_completed(f"resp_{upstream_id}")).encode())
assert body.get("stream") is True, body
if request.target == "/v1/responses":
return _native_stream(f"resp_{upstream_id}")
assert request.target == "/v1/chat/completions", request.target
return _chat_stream(f"chatcmpl-{upstream_id}", string_value(body["model"]))
def _terminal_response_id(
gateway: Gateway, key: str, body: dict[str, JsonValue], headers: Mapping[str, str] | None = None
) -> str:
response: Final = gateway.request("POST", "/v1/responses", body, key=key, headers=headers)
assert response.status_code == 200, response.text
frames: Final = tuple(
JSON_OBJECT.validate_json(line.removeprefix("data: "))
for line in response.text.splitlines()
if line.startswith("data: ") and line != "data: [DONE]"
)
assert frames and frames[-1]["type"] == "response.completed", response.text
advertised: Final = tuple(
dict.fromkeys(string_value(object_value(event["response"])["id"]) for event in frames if "response" in event)
)
assert len(advertised) == 1, advertised
return advertised[0]
def _created_response_id(
gateway: Gateway, key: str, body: dict[str, JsonValue], headers: Mapping[str, str] | None = None
) -> str:
if body.get("stream") is True:
return _terminal_response_id(gateway, key, body, headers)
response: Final = gateway.request("POST", "/v1/responses", body, key=key, headers=headers)
assert response.status_code == 200, response.text
return string_value(JSON_OBJECT.validate_json(response.content)["id"])
def _read_and_cancel(
gateway: Gateway, key: str, response_id: str, headers: Mapping[str, str] | None = None
) -> tuple[httpx.Response, ...]:
responses: Final = tuple(
gateway.request(method, path, key=key, headers=headers)
for method, path in (
("GET", f"/v1/responses/{response_id}"),
("GET", f"/v1/responses/{response_id}"),
("POST", f"/v1/responses/{response_id}/cancel"),
)
)
assert [response.status_code for response in responses] == [200, 200, 200], [r.text for r in responses]
return responses
def _spend_logs_lookup(gateway: Gateway, request_id: str) -> list[JsonValue]:
response: Final = gateway.request("GET", "/spend/logs", params={"request_id": request_id})
assert response.status_code == 200, response.text
return [row["request_id"] for row in _JSON_LIST.validate_json(response.content)]
def _request_ids_for_key(gateway: Gateway, key: str, count: int) -> list[str]:
rows: Final = eventually(
lambda: gateway.rows(
'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE api_key=%s ORDER BY "endTime"',
(sha256(key.encode()).hexdigest(),),
),
lambda found: len(found) >= count,
seconds=70,
)
return [string_value(row["request_id"]) for row in rows]
def _logs_page_search(gateway: Gateway, client_id: str) -> list[str]:
now: Final = datetime.now(timezone.utc)
page: Final = gateway.get(
"/spend/logs/ui",
params={
"search": client_id,
"start_date": (now - timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"),
"end_date": (now + timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"),
},
)
return [string_value(object_value(row)["request_id"]) for row in list_value(page["data"])]
@pytest.mark.timeout(120)
@pytest.mark.parametrize("litellm_model", [_NATIVE_MODEL, _BRIDGED_MODEL], ids=["native", "bridge"])
def test_streamed_responses_spend_row_is_filed_under_the_id_the_client_saw(
gateway: Gateway, litellm_model: str
) -> None:
with wire_server(_respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=litellm_model, api_base=f"{wire.url}/v1", api_key="synthetic-key")
key: Final = scenario.key(models=[model])
client_id: Final = _terminal_response_id(
gateway, key, {"model": model, "input": f"marker-{uuid4().hex}", "stream": True}
)
assert client_id.startswith("resp_"), client_id
assert _request_ids_for_key(gateway, key, 1) == [client_id]
assert _logs_page_search(gateway, client_id) == [client_id]
assert _spend_logs_lookup(gateway, client_id) == [client_id]
def _proxy_config(
directory: Path,
general_settings: Mapping[str, JsonValue],
litellm_settings: Mapping[str, JsonValue] = MappingProxyType({}),
) -> Path:
stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
path: Final = directory / "responses-spend.yaml"
path.write_text(
yaml.safe_dump(
{
**stock,
"general_settings": {**object_value(stock["general_settings"]), **general_settings},
"litellm_settings": {**object_value(stock["litellm_settings"]), **litellm_settings},
}
)
)
return path
def _prompt_storing_config(directory: Path) -> Path:
return _proxy_config(directory, {"store_prompts_in_spend_logs": True})
def test_a_follow_up_turn_on_a_streamed_bridged_response_replays_the_first_turn(
gateway: Gateway, tmp_path: Path
) -> None:
first_prompt: Final = f"first turn {uuid4().hex}"
with (
wire_server(_respond) as wire,
owned_proxy(gateway, tmp_path, {}, config=_prompt_storing_config(tmp_path)) as storing,
storing.scenario() as scenario,
):
model: Final = scenario.model(model=_BRIDGED_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key")
key: Final = scenario.key(models=[model])
first_id: Final = _terminal_response_id(storing, key, {"model": model, "input": first_prompt, "stream": True})
assert _request_ids_for_key(storing, key, 1) == [first_id]
wire.drain()
second_id: Final = _terminal_response_id(
storing,
key,
{"model": model, "input": "second turn", "previous_response_id": first_id, "stream": True},
)
replayed: Final = _JSON_LIST.validate_python(JSON_OBJECT.validate_json(wire.drain()[-1].body)["messages"])
assert [(message["role"], message["content"]) for message in replayed] == [
("user", first_prompt),
("assistant", _ANSWER),
("user", "second turn"),
], replayed
assert _request_ids_for_key(storing, key, 2) == [first_id, second_id]
@pytest.mark.timeout(180)
@pytest.mark.parametrize("stream_first_turn", [True, False], ids=["streamed-first-turn", "non-streamed-first-turn"])
def test_a_follow_up_through_the_id_a_retrieve_returned_replays_the_first_turn_once(
gateway: Gateway, tmp_path: Path, stream_first_turn: bool
) -> None:
first_prompt: Final = f"first turn {uuid4().hex}"
with (
wire_server(_respond) as wire,
owned_proxy(gateway, tmp_path, {}, config=_prompt_storing_config(tmp_path)) as storing,
storing.scenario() as scenario,
):
native: Final = scenario.model(model=_NATIVE_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key")
bridged: Final = scenario.model(model=_BRIDGED_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key")
key: Final = scenario.key(models=[native, bridged])
first_id: Final = _created_response_id(
storing, key, {"model": native, "input": first_prompt, "stream": stream_first_turn}
)
retrieved: Final = storing.request("GET", f"/v1/responses/{first_id}", key=key)
assert retrieved.status_code == 200, retrieved.text
retrieved_id: Final = string_value(JSON_OBJECT.validate_json(retrieved.content)["id"])
assert retrieved_id == first_id
stored: Final = _request_ids_for_key(storing, key, 2)
assert stored.count(first_id) == 1, stored
assert len(set(stored)) == 2, stored
assert _spend_logs_lookup(storing, first_id) == [first_id]
wire.drain()
_terminal_response_id(
storing,
key,
{"model": bridged, "input": "second turn", "previous_response_id": retrieved_id, "stream": True},
)
replayed: Final = _JSON_LIST.validate_python(JSON_OBJECT.validate_json(wire.drain()[-1].body)["messages"])
assert [(message["role"], message["content"]) for message in replayed] == [
("user", first_prompt),
("assistant", _ANSWER),
("user", "second turn"),
], replayed
def _replayed_turns(wire_body: bytes) -> list[tuple[JsonValue, JsonValue]]:
replayed: Final = _JSON_LIST.validate_python(JSON_OBJECT.validate_json(wire_body)["messages"])
return [(message["role"], message["content"]) for message in replayed]
@pytest.mark.timeout(180)
@pytest.mark.parametrize("reader", ["proxy-admin", "master-key"])
def test_a_reader_other_than_the_creator_continues_through_the_id_its_retrieve_returned(
gateway: Gateway, tmp_path: Path, reader: str
) -> None:
first_prompt: Final = f"first turn {uuid4().hex}"
with (
wire_server(_respond) as wire,
owned_proxy(gateway, tmp_path, {}, config=_prompt_storing_config(tmp_path)) as storing,
storing.scenario() as scenario,
):
native: Final = scenario.model(model=_NATIVE_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key")
bridged: Final = scenario.model(model=_BRIDGED_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key")
creator: Final = scenario.key(user_id=scenario.user(), models=[native, bridged])
reader_key: Final = (
scenario.key(user_id=scenario.user(user_role="proxy_admin"), models=[native, bridged])
if reader == "proxy-admin"
else storing.key
)
first_id: Final = _terminal_response_id(
storing, creator, {"model": native, "input": first_prompt, "stream": True}
)
assert _request_ids_for_key(storing, creator, 1) == [first_id]
retrieved: Final = storing.request("GET", f"/v1/responses/{first_id}", key=reader_key)
assert retrieved.status_code == 200, retrieved.text
retrieved_id: Final = string_value(JSON_OBJECT.validate_json(retrieved.content)["id"])
wire.drain()
_terminal_response_id(
storing,
reader_key,
{"model": bridged, "input": "second turn", "previous_response_id": retrieved_id, "stream": True},
)
assert _replayed_turns(wire.drain()[-1].body) == [
("user", first_prompt),
("assistant", _ANSWER),
("user", "second turn"),
]
assert retrieved_id == first_id
@pytest.mark.timeout(180)
def test_reads_and_cancels_sent_with_a_session_header_add_no_replayed_turns(gateway: Gateway, tmp_path: Path) -> None:
first_prompt: Final = f"first turn {uuid4().hex}"
session: Final = {"x-litellm-session-id": f"session-{uuid4().hex}"}
with (
wire_server(_respond) as wire,
owned_proxy(gateway, tmp_path, {}, config=_prompt_storing_config(tmp_path)) as storing,
storing.scenario() as scenario,
):
native: Final = scenario.model(model=_NATIVE_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key")
bridged: Final = scenario.model(model=_BRIDGED_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key")
key: Final = scenario.key(models=[native, bridged])
first_id: Final = _created_response_id(storing, key, {"model": native, "input": first_prompt}, session)
_read_and_cancel(storing, key, first_id, session)
assert len(_request_ids_for_key(storing, key, 4)) == 4
wire.drain()
_terminal_response_id(
storing,
key,
{"model": bridged, "input": "second turn", "previous_response_id": first_id, "stream": True},
session,
)
assert _replayed_turns(wire.drain()[-1].body) == [
("user", first_prompt),
("assistant", _ANSWER),
("user", "second turn"),
]
@pytest.mark.timeout(180)
def test_each_read_and_cancel_reaches_callbacks_and_spend_logs_under_an_id_of_its_own(
gateway: Gateway, tmp_path: Path
) -> None:
logged_call_types: Final = frozenset({"aresponses", "aget_responses", "acancel_responses"})
with (
wire_server(_respond) as wire,
wire_server(lambda _: Reply()) as sink,
owned_proxy(
gateway,
tmp_path,
{"GENERIC_LOGGER_ENDPOINT": sink.url},
config=_proxy_config(tmp_path, {}, {"callbacks": ["generic_api"], "DEFAULT_FLUSH_INTERVAL_SECONDS": 1}),
) as logging_proxy,
logging_proxy.scenario() as scenario,
):
model: Final = scenario.model(model=_NATIVE_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key")
key: Final = scenario.key(models=[model])
first_id: Final = _created_response_id(logging_proxy, key, {"model": model, "input": f"marker-{uuid4().hex}"})
reads: Final = _read_and_cancel(logging_proxy, key, first_id)
assert [string_value(JSON_OBJECT.validate_json(read.content)["id"]) for read in reads] == [first_id] * 3
batches: Final[list[bytes]] = [] # mutable-ok: drain() consumes, later polls keep earlier batches
def delivered() -> tuple[tuple[str, str], ...]:
batches.extend(batch.body for batch in sink.drain())
return tuple(
(string_value(event["call_type"]), string_value(event["id"]))
for event in (object_value(item) for batch in batches for item in _JSON_LIST.validate_json(batch))
if event["call_type"] in logged_call_types
)
events: Final = eventually(delivered, lambda found: len(found) >= 4, seconds=30)
read_ids: Final = [logged_id for call_type, logged_id in events if call_type != "aresponses"]
assert [logged_id for call_type, logged_id in events if call_type == "aresponses"] == [first_id], events
assert len(read_ids) == 3, events
assert first_id not in read_ids, events
assert len(set(read_ids)) == 3, events
assert sorted(_request_ids_for_key(logging_proxy, key, 4)) == sorted([first_id, *read_ids])
@pytest.mark.timeout(240)
def test_with_id_security_off_reads_and_cancels_add_no_rows_and_no_replayed_turns(
gateway: Gateway, tmp_path: Path
) -> None:
first_prompt: Final = f"first turn {uuid4().hex}"
session: Final = {"x-litellm-session-id": f"session-{uuid4().hex}"}
with (
wire_server(_respond) as wire,
owned_proxy(
gateway,
tmp_path,
{},
config=_proxy_config(
tmp_path, {"store_prompts_in_spend_logs": True, "disable_responses_id_security": True}
),
) as insecure,
insecure.scenario() as scenario,
):
native: Final = scenario.model(model=_NATIVE_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key")
bridged: Final = scenario.model(model=_BRIDGED_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key")
key: Final = scenario.key(models=[native, bridged])
first_id: Final = _created_response_id(insecure, key, {"model": native, "input": first_prompt}, session)
reads: Final = _read_and_cancel(insecure, key, first_id, session)
assert [string_value(JSON_OBJECT.validate_json(read.content)["id"]) for read in reads] == [first_id] * 3
later_id: Final = _created_response_id(insecure, key, {"model": native, "input": "a later request"})
assert _request_ids_for_key(insecure, key, 2) == [first_id, later_id]
wire.drain()
_terminal_response_id(
insecure,
key,
{"model": bridged, "input": "second turn", "previous_response_id": first_id, "stream": True},
session,
)
replayed: Final = _JSON_LIST.validate_python(JSON_OBJECT.validate_json(wire.drain()[-1].body)["messages"])
assert [(message["role"], message["content"]) for message in replayed] == [
("user", first_prompt),
("assistant", _ANSWER),
("user", "second turn"),
], replayed

View file

@ -5,19 +5,28 @@ import re
import sys
import threading
from datetime import datetime, timedelta, timezone
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from litellm.constants import _DEFAULT_TTL_FOR_HTTPX_CLIENTS
from litellm.constants import _DEFAULT_TTL_FOR_HTTPX_CLIENTS, INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.integrations.azure_storage import azure_storage as azure_storage_module
from litellm.integrations.azure_storage.azure_storage import (
AzureBlobStorageLogger,
_cached_credential_chain_token_provider,
adls_safe_file_name,
)
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
BackgroundResponseSettlement,
)
from litellm.types.secret_managers.get_azure_ad_token_provider import AzureCredentialType
from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload
from litellm.types.utils import (
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
StandardAuditLogPayload,
StandardLoggingPayload,
)
@pytest.fixture
@ -648,3 +657,171 @@ async def test_audit_log_is_uploaded_to_a_dated_audit_logs_path_with_an_account_
assert json.loads(uploaded) == audit_log
assert file_client.append_data.await_args.kwargs == {"data": uploaded, "offset": 0, "length": len(uploaded)}
file_client.flush_data.assert_awaited_once_with(position=len(uploaded), offset=0)
BACKGROUND_RESPONSE_ID: Final = "resp_YWJjZA=="
BACKGROUND_COST_POLL_KWARGS: Final = {
"call_type": "aget_responses",
"litellm_params": {
"metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN},
},
}
PASSTHROUGH_SETTLEMENT_KWARGS: Final = {
"call_type": "pass_through_endpoint",
"litellm_params": {"metadata": {}},
SETTLES_BACKGROUND_RESPONSE_KWARG: BackgroundResponseSettlement.SETTLES,
}
def _background_create_cost_and_other_events(
cost_kwargs: dict[str, object],
) -> tuple[tuple[dict[str, object], StandardLoggingPayload], ...]:
create: Final = StandardLoggingPayload(
id=BACKGROUND_RESPONSE_ID,
call_type="aresponses",
messages=[{"role": "user", "content": "background create input"}],
)
cost: Final = StandardLoggingPayload(id=BACKGROUND_RESPONSE_ID, call_type="aget_responses", response_cost=0.42)
ordinary: Final = StandardLoggingPayload(id="chatcmpl-ordinary", call_type="acompletion")
batch_retrieve: Final = StandardLoggingPayload(id="batch_abc123", call_type="aretrieve_batch", response_cost=0.07)
return (
({"call_type": "aresponses", "litellm_params": {"metadata": {}}, "standard_logging_object": create}, create),
({**cost_kwargs, "standard_logging_object": cost}, cost),
(
{"call_type": "acompletion", "litellm_params": {"metadata": {}}, "standard_logging_object": ordinary},
ordinary,
),
(
{
"call_type": "aretrieve_batch",
"litellm_params": {"metadata": {}},
"standard_logging_object": batch_retrieve,
},
batch_retrieve,
),
)
def _expected_files(
events: tuple[tuple[dict[str, object], StandardLoggingPayload], ...],
) -> dict[str, StandardLoggingPayload]:
create, cost, ordinary, batch_retrieve = (payload for _, payload in events)
return {
"resp_YWJjZA.json": create,
"resp_YWJjZA_background_cost.json": cost,
"chatcmpl-ordinary.json": ordinary,
"batch_abc123.json": batch_retrieve,
}
async def _log_and_flush(
logger: AzureBlobStorageLogger, events: tuple[tuple[dict[str, object], StandardLoggingPayload], ...]
) -> None:
for kwargs, payload in events:
await logger.async_log_success_event(kwargs, {"id": payload["id"]}, None, None)
await logger.async_send_batch()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"cost_kwargs",
(
pytest.param(BACKGROUND_COST_POLL_KWARGS, id="cost-poll"),
pytest.param(PASSTHROUGH_SETTLEMENT_KWARGS, id="passthrough-settlement"),
),
)
async def test_background_cost_event_keeps_its_own_file_beside_the_create_on_the_entra_path(
mock_env_vars, monkeypatch, cost_kwargs
):
monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True)
events: Final = _background_create_cost_and_other_events(cost_kwargs)
with patch( # test-quality-ok: REST client is created inside the method; assert emitted requests
"litellm.integrations.azure_storage.azure_storage.get_async_httpx_client"
) as mock_get_client:
mock_http_client = AsyncMock()
mock_http_client.put.return_value = MagicMock()
mock_http_client.patch.return_value = MagicMock()
mock_get_client.return_value = mock_http_client
logger = AzureBlobStorageLogger()
logger.azure_auth_token = "mock-azure-ad-token"
logger.token_expiry = None
await _log_and_flush(logger, events)
base_url: Final = "https://test-account.dfs.core.windows.net/test-container/"
appends: Final = [call for call in mock_http_client.patch.call_args_list if "action=append" in call.args[0]]
uploaded: Final = {
call.args[0].removeprefix(base_url).split("?")[0]: json.loads(call.kwargs["data"]) for call in appends
}
assert uploaded == _expected_files(events), (
"the create, its background cost event and every other event must each keep their own file"
)
@pytest.mark.asyncio
async def test_failure_event_is_uploaded_to_a_file_named_after_its_id(mock_env_vars, monkeypatch):
monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True)
failure: Final = StandardLoggingPayload(id="chatcmpl-failed", call_type="acompletion", error_str="provider failure")
with patch( # test-quality-ok: REST client is created inside the method; assert emitted requests
"litellm.integrations.azure_storage.azure_storage.get_async_httpx_client"
) as mock_get_client:
mock_http_client = AsyncMock()
mock_http_client.put.return_value = MagicMock()
mock_http_client.patch.return_value = MagicMock()
mock_get_client.return_value = mock_http_client
logger = AzureBlobStorageLogger()
logger.azure_auth_token = "mock-azure-ad-token"
logger.token_expiry = None
await logger.async_log_failure_event(
{"call_type": "acompletion", "litellm_params": {"metadata": {}}, "standard_logging_object": failure},
None,
None,
None,
)
await logger.async_send_batch()
base_url: Final = "https://test-account.dfs.core.windows.net/test-container/"
appends: Final = [call for call in mock_http_client.patch.call_args_list if "action=append" in call.args[0]]
assert {call.args[0].removeprefix(base_url).split("?")[0]: json.loads(call.kwargs["data"]) for call in appends} == {
"chatcmpl-failed.json": failure
}
@pytest.mark.asyncio
async def test_background_cost_event_keeps_its_own_file_beside_the_create_with_an_account_key(
workload_identity_env_vars, monkeypatch
):
monkeypatch.setenv("AZURE_STORAGE_ACCOUNT_KEY", "dGVzdC1rZXk=")
monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True)
monkeypatch.setattr(azure_storage_module, "datetime", _FrozenDatetime)
events: Final = _background_create_cost_and_other_events(BACKGROUND_COST_POLL_KWARGS)
uploaded: Final[dict[str, StandardLoggingPayload]] = {}
def file_client_for(file_path: str) -> MagicMock:
file_client = MagicMock()
file_client.create_file = AsyncMock()
file_client.flush_data = AsyncMock()
async def append_data(data: bytes, offset: int, length: int) -> None:
uploaded[file_path] = json.loads(data)
file_client.append_data = append_data
return file_client
file_system_client = MagicMock()
file_system_client.get_file_client = MagicMock(side_effect=file_client_for)
service_client = MagicMock()
service_client.get_file_system_client = MagicMock(return_value=file_system_client)
fake_aio_module = MagicMock()
fake_aio_module.DataLakeServiceClient = MagicMock(return_value=service_client)
with patch.dict(sys.modules, {"azure.storage.filedatalake.aio": fake_aio_module}):
logger = AzureBlobStorageLogger()
await _log_and_flush(logger, events)
today: Final = _FrozenDatetime.now().strftime("%Y-%m-%d")
assert uploaded == {f"{today}/{name}": payload for name, payload in _expected_files(events).items()}, (
"the create, its background cost event and every other event must each keep their own file"
)

View file

@ -20,10 +20,16 @@ from litellm.integrations.clickhouse.clickhouse_spend_logger import (
strip_cache_hit_suffix,
)
from litellm.integrations.clickhouse.context import lens_analysis
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.litellm_core_utils.internal_call_metadata import BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX
from litellm.litellm_core_utils import litellm_logging
from litellm.litellm_core_utils.secret_redaction import REDACTED
from litellm.tracing.types import SpendLogRecord
from litellm.types.utils import StandardLoggingPayload
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
BackgroundResponseSettlement,
)
from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, StandardLoggingPayload
_JSON_OBJECT_ADAPTER: Final = TypeAdapter(Mapping[str, JsonValue])
@ -349,6 +355,41 @@ def test_cache_hit_id_is_stripped_for_response_id():
assert strip_cache_hit_suffix("chatcmpl-xyz") == "chatcmpl-xyz"
@pytest.mark.parametrize(
"cost_kwargs",
(
pytest.param(
{
"call_type": "aget_responses",
"litellm_params": {
"metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN}
},
},
id="cost-poll",
),
pytest.param(
{
"call_type": "pass_through_endpoint",
"litellm_params": {"metadata": {}},
SETTLES_BACKGROUND_RESPONSE_KWARG: BackgroundResponseSettlement.SETTLES,
},
id="passthrough-settlement",
),
),
)
def test_background_cost_row_keeps_its_own_request_id_beside_its_create(cost_kwargs: Mapping[str, object]) -> None:
payload: Final = cast(StandardLoggingPayload, _payload(id="resp_YWJjZA", call_type="aresponses"))
create_kwargs: Final[Mapping[str, object]] = {"call_type": "aresponses", "litellm_params": {"metadata": {}}}
rows: Final = [spend_log_row_from_payload(payload, kwargs) for kwargs in (create_kwargs, cost_kwargs)]
assert [row["request_id"] for row in rows] == [
"resp_YWJjZA",
f"resp_YWJjZA{BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX}",
]
assert [row["response_id"] for row in rows] == ["resp_YWJjZA", "resp_YWJjZA"]
def test_parse_traceparent_valid_missing_malformed():
assert parse_traceparent(TRACEPARENT) == (TRACE_ID, SPAN_ID)
assert parse_traceparent(None) == ("", "")

View file

@ -1,14 +1,18 @@
import functools
import http.server
import importlib.abc
import ipaddress
import os
import queue
import ssl
import sys
import threading
from collections.abc import Iterator
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from importlib.machinery import ModuleSpec
from pathlib import Path
from types import ModuleType
from typing import Final
import pytest
@ -75,6 +79,36 @@ def write_self_signed_cert(directory: Path, stem: str) -> tuple[Path, Path]:
return certificate_path, key_path
PROXY_ONLY_PACKAGES: Final = frozenset({"fastapi", "starlette"})
class _ProxyOnlyPackageBlocker(importlib.abc.MetaPathFinder):
def find_spec(
self,
fullname: str,
path: Sequence[str] | None = None,
target: ModuleType | None = None,
) -> ModuleSpec | None:
if fullname.partition(".")[0] in PROXY_ONLY_PACKAGES:
raise ModuleNotFoundError(f"No module named {fullname!r}")
return None
def _needs_proxy_extras(module_name: str) -> bool:
return (
module_name.partition(".")[0] in PROXY_ONLY_PACKAGES
or module_name == "litellm.proxy"
or module_name.startswith("litellm.proxy.")
)
@pytest.fixture
def sdk_only_install(monkeypatch: pytest.MonkeyPatch) -> None:
for module_name in tuple(filter(_needs_proxy_extras, sys.modules)):
monkeypatch.delitem(sys.modules, module_name)
monkeypatch.setattr(sys, "meta_path", [_ProxyOnlyPackageBlocker(), *sys.meta_path])
@pytest.fixture(autouse=True)
def restore_process_environment() -> Iterator[None]:
original: Final = dict(os.environ)

View file

@ -13,9 +13,11 @@ import litellm
import pytest
import respx
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.integrations.gcs_bucket.gcs_bucket import GCSBucketLogger
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
mock_response_data: Final = {
@ -172,9 +174,7 @@ async def test_get_payload_not_found(monkeypatch):
assert payload is None
def _gcs_logger_with_upload_status(
monkeypatch: pytest.MonkeyPatch, status_code: int
) -> tuple[GCSBucketLogger, Mock]:
def _gcs_logger_with_upload_status(monkeypatch: pytest.MonkeyPatch, status_code: int) -> tuple[GCSBucketLogger, Mock]:
monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True)
monkeypatch.setattr(google.auth, "default", _google_default_credentials)
monkeypatch.setenv("GCS_FLUSH_INTERVAL", "3600")
@ -394,3 +394,98 @@ async def test_batch_line_items_are_serialized_off_the_event_loop(monkeypatch: p
(thread,) = threads
assert thread is not threading.main_thread()
@pytest.mark.asyncio
async def test_unbatched_background_cost_event_keeps_its_own_object_readable_by_its_spend_log_id(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("GCS_USE_BATCHED_LOGGING", "false")
logger, _ = _gcs_logger_with_upload_status(monkeypatch, 200)
object_prefix: Final = "/storage/v1/b/test-bucket/o/"
stored: Final[dict[str, object]] = {}
def bucket(request: httpx.Request) -> httpx.Response:
if request.method == "POST":
stored[request.url.params["name"]] = json.loads(request.content)
return httpx.Response(200, json={"name": request.url.params["name"]})
name: Final = request.url.path.removeprefix(object_prefix)
if name not in stored:
return httpx.Response(404, text="No such object")
return httpx.Response(200, content=json.dumps(stored[name]).encode("utf-8"))
logger.async_httpx_client = AsyncHTTPHandler(transport=httpx.MockTransport(bucket))
response_id: Final = "resp_YmFja2dyb3VuZA"
create: Final = {
"id": response_id,
"messages": [{"role": "user", "content": "background create input"}],
"error_str": None,
}
cost: Final = {"id": response_id, "response_cost": 0.42, "error_str": None}
ordinary: Final = {"id": "chatcmpl-ordinary", "error_str": None}
batch_retrieve: Final = {"id": "batch_abc123", "response_cost": 0.07, "error_str": None}
events: Final = (
({"call_type": "aresponses", "litellm_params": {"metadata": {}}}, create),
(
{
"call_type": "aget_responses",
"litellm_params": {
"metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN}
},
},
cost,
),
({"call_type": "acompletion", "litellm_params": {"metadata": {}}}, ordinary),
({"call_type": "aretrieve_batch", "litellm_params": {"metadata": {}}}, batch_retrieve),
)
for kwargs, payload in events:
await logger.async_log_success_event(
{**kwargs, "standard_logging_object": payload}, {"id": payload["id"]}, None, None
)
await logger.flush_queue()
assert {name.split("/", maxsplit=1)[1]: payload for name, payload in stored.items()} == {
response_id: create,
f"{response_id}_background_cost": cost,
"chatcmpl-ordinary": ordinary,
"batch_abc123": batch_retrieve,
}, "the create, its background cost event and every other event must each keep their own object"
now: Final = datetime.now(timezone.utc)
assert await logger.get_request_response_payload(f"{response_id}_background_cost", now, None) == cost
assert await logger.get_request_response_payload(response_id, now, None) == create
@pytest.mark.asyncio
async def test_unbatched_gcs_log_id_still_names_a_background_cost_event_object(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("GCS_USE_BATCHED_LOGGING", "false")
logger, _ = _gcs_logger_with_upload_status(monkeypatch, 200)
stored: Final[dict[str, object]] = {}
def bucket(request: httpx.Request) -> httpx.Response:
stored[request.url.params["name"]] = json.loads(request.content)
return httpx.Response(200, json={"name": request.url.params["name"]})
logger.async_httpx_client = AsyncHTTPHandler(transport=httpx.MockTransport(bucket))
cost: Final = {"id": "resp_YmFja2dyb3VuZA", "response_cost": 0.42, "error_str": None}
await logger.async_log_success_event(
{
"call_type": "aget_responses",
"litellm_params": {
"metadata": {
INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
"gcs_log_id": "qa-run-7",
}
},
"standard_logging_object": cost,
},
{"id": cost["id"]},
None,
None,
)
await logger.flush_queue()
((name, payload),) = stored.items()
object_name: Final = name.split("/", maxsplit=1)[1]
assert object_name.startswith("custom-") and object_name.endswith("-qa-run-7"), object_name
assert payload == cost

View file

@ -2,11 +2,15 @@ import os
import unittest
from litellm._uuid import uuid
from datetime import datetime, timezone
from typing import Final
from unittest.mock import MagicMock, patch
from litellm.integrations.deepeval.api import Endpoints, HttpMethods
from litellm.integrations.deepeval.deepeval import DeepEvalLogger
from litellm.integrations.deepeval.types import SpanApiType, TraceSpanApiStatus
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.litellm_core_utils.internal_call_metadata import BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX
from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
class TestDeepEvalLogger(unittest.TestCase):
@ -94,6 +98,37 @@ class TestDeepEvalLogger(unittest.TestCase):
self.assertEqual(llm_span["inputTokenCount"], 10)
self.assertEqual(llm_span["outputTokenCount"], 20)
def test_background_cost_span_keeps_its_own_uuid_beside_its_create(self) -> None:
standard_logging_object: Final = {
"id": "resp_YWJjZA",
"trace_id": self.trace_id,
"model": self.model,
"response": {"usage": {}, "choices": [{"message": {"content": "OK"}}]},
}
create_kwargs: Final = {
"input": self.input_str,
"call_type": "aresponses",
"litellm_params": {"metadata": {}},
"standard_logging_object": standard_logging_object,
}
cost_kwargs: Final = {
"input": self.input_str,
"call_type": "aget_responses",
"litellm_params": {
"metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN}
},
"standard_logging_object": standard_logging_object,
}
for kwargs in (create_kwargs, cost_kwargs):
self.logger.log_success_event(kwargs, self.mock_response_obj, self.start_time, self.end_time)
span_uuids: Final = [
call.kwargs["body"]["llmSpans"][0]["uuid"]
for call in self.mock_api_instance.send_request.call_args_list
]
self.assertEqual(span_uuids, ["resp_YWJjZA", f"resp_YWJjZA{BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX}"])
def test_log_failure_event(self):
error_message = "This is an error."
kwargs = {

View file

@ -0,0 +1,52 @@
from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Final
from unittest.mock import MagicMock
import pytest
import litellm
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.integrations.dynamodb import DyanmoDBLogger
from litellm.litellm_core_utils.internal_call_metadata import BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
BackgroundResponseSettlement,
)
from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
BACKGROUND_COST_POLL_KWARGS: Final[Mapping[str, object]] = {
"call_type": "aget_responses",
"litellm_params": {"metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN}},
}
PASSTHROUGH_SETTLEMENT_KWARGS: Final[Mapping[str, object]] = {
"call_type": "pass_through_endpoint",
"litellm_params": {"metadata": {}},
SETTLES_BACKGROUND_RESPONSE_KWARG: BackgroundResponseSettlement.SETTLES,
}
@pytest.mark.parametrize(
"cost_kwargs",
(
pytest.param(BACKGROUND_COST_POLL_KWARGS, id="cost-poll"),
pytest.param(PASSTHROUGH_SETTLEMENT_KWARGS, id="passthrough-settlement"),
),
)
@pytest.mark.usefixtures("sdk_only_install")
def test_background_cost_event_keeps_its_own_item_beside_its_create(
monkeypatch: pytest.MonkeyPatch, cost_kwargs: Mapping[str, object]
) -> None:
monkeypatch.setattr(litellm, "dynamodb_table_name", "litellm-logs")
dynamodb: Final = MagicMock()
logger: Final = DyanmoDBLogger(dynamodb=dynamodb)
response_obj: Final = {"id": "resp_YWJjZA", "usage": {"total_tokens": 3}}
now: Final = datetime.now(timezone.utc)
create_kwargs: Final[Mapping[str, object]] = {"call_type": "aresponses", "litellm_params": {"metadata": {}}}
for kwargs in (create_kwargs, cost_kwargs):
logger.log_event(kwargs, response_obj, now, now, print_verbose=print)
dynamodb.Table.assert_called_with("litellm-logs")
written_ids: Final = [call.kwargs["Item"]["id"] for call in dynamodb.Table.return_value.put_item.call_args_list]
assert written_ids == ["resp_YWJjZA", f"resp_YWJjZA{BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX}"]

View file

@ -1,11 +1,30 @@
import json
import os
from collections.abc import Mapping
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import litellm
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.integrations.openmeter import OpenMeterLogger
from litellm.litellm_core_utils.internal_call_metadata import BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
BackgroundResponseSettlement,
)
from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
BACKGROUND_COST_POLL_KWARGS: Final[Mapping[str, object]] = {
"call_type": "aget_responses",
"litellm_params": {"metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN}},
}
PASSTHROUGH_SETTLEMENT_KWARGS: Final[Mapping[str, object]] = {
"call_type": "pass_through_endpoint",
"litellm_params": {"metadata": {}},
SETTLES_BACKGROUND_RESPONSE_KWARG: BackgroundResponseSettlement.SETTLES,
}
class TestOpenMeterIntegration:
@ -36,6 +55,31 @@ class TestOpenMeterIntegration:
with pytest.raises(Exception, match=r"Missing keys.*OPENMETER_API_KEY"):
OpenMeterLogger()
@pytest.mark.parametrize(
"cost_kwargs",
(
pytest.param(BACKGROUND_COST_POLL_KWARGS, id="cost-poll"),
pytest.param(PASSTHROUGH_SETTLEMENT_KWARGS, id="passthrough-settlement"),
),
)
@pytest.mark.usefixtures("sdk_only_install")
def test_background_cost_event_gets_its_own_event_id_beside_its_create(
self, cost_kwargs: Mapping[str, object]
) -> None:
logger: Final = OpenMeterLogger()
response_obj: Final = {"id": "resp_YWJjZA"}
create_kwargs: Final[Mapping[str, object]] = {
"user": "user-1",
"call_type": "aresponses",
"litellm_params": {"metadata": {}},
}
create_event: Final = logger._common_logic(create_kwargs, response_obj)
cost_event: Final = logger._common_logic({**cost_kwargs, "user": "user-1", "response_cost": 0.42}, response_obj)
assert create_event["id"] == "resp_YWJjZA"
assert cost_event["id"] == f"resp_YWJjZA{BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX}"
def test_common_logic_with_string_user(self):
"""Test that _common_logic correctly handles string user parameter"""
logger = OpenMeterLogger()

View file

@ -11090,6 +11090,40 @@ def test_get_assembled_streaming_response_bills_a_provider_reported_usage_cost()
assert logging_obj.response_cost_calculator(result=assembled) == 0.0042
@pytest.mark.parametrize("continued_response_id", ["resp_continuedafteracutstream", None])
@pytest.mark.parametrize("client_facing_response_id", ["resp_encryptedidtheclientsaw", None])
def test_sync_success_handler_logs_a_stream_under_the_id_the_client_saw(
client_facing_response_id, continued_response_id
):
class Recorder(CustomLogger):
def __init__(self):
super().__init__()
self.logged = []
def log_success_event(self, kwargs, response_obj, start_time, end_time):
self.logged.append((response_obj.id, kwargs["standard_logging_object"]["id"]))
recorder: Final = Recorder()
logging_obj: Final = _responses_stream_logging_obj()
logging_obj.dynamic_success_callbacks = [recorder]
logging_obj.client_facing_response_id = client_facing_response_id
if continued_response_id is not None:
logging_obj.record_continued_partial(
Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2), 0.0, continued_response_id
)
now: Final = datetime.datetime.now()
logging_obj.success_handler(
result=_completed_responses_event(ResponseAPIUsage(input_tokens=3, output_tokens=2, total_tokens=5)),
start_time=now,
end_time=now,
cache_hit=False,
)
expected_id: Final = client_facing_response_id or continued_response_id or "resp-1"
assert recorder.logged == [(expected_id, expected_id)]
def test_response_cost_calculator_prices_terminal_responses_event_from_its_response():
logging_obj: Final = _responses_stream_logging_obj()
inner_response: Final = ResponsesAPIResponse(

View file

@ -285,6 +285,9 @@ NOVA_2_URL: Final = "wss://api.deepgram.com/v1/listen?model=nova-2"
[
pytest.param(NOVA_3_URL, (), True, id="streaming entry present"),
pytest.param(f"{NOVA_3_URL}&language=multi", (), True, id="multilingual entry present"),
pytest.param(
"wss://api.deepgram.com/v1/listen?model=nova-3-medical", (), True, id="medical streaming entry present"
),
pytest.param(NOVA_2_URL, (), False, id="only the pre-recorded entry"),
pytest.param(f"{NOVA_2_URL}&language=multi", ("deepgram/streaming/nova-2",), False, id="needs multilingual"),
pytest.param("wss://api.deepgram.com/v1/listen?model=nova-99-unmapped", (), False, id="nothing priced"),
@ -295,7 +298,8 @@ NOVA_2_URL: Final = "wss://api.deepgram.com/v1/listen?model=nova-2"
def test_deepgram_listen_is_priced(
monkeypatch: pytest.MonkeyPatch, upstream_url: str, extra_rows: tuple[str, ...], expected: bool
):
"""The bundled map prices only nova-3 for streaming; nova-2 has a pre-recorded row, which must never count."""
"""The bundled map prices nova-3 and nova-3-medical for streaming; nova-2 has only a pre-recorded row, which must
never count."""
monkeypatch.delitem(litellm.model_cost, "deepgram/streaming/nova-2", raising=False)
assert "deepgram/nova-2" in litellm.model_cost
for row in extra_rows:

View file

@ -3,6 +3,7 @@ import hashlib
import json
import time
from collections.abc import Iterable
from typing import Final
from unittest.mock import patch
import pytest
@ -19,7 +20,11 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
evict_and_broadcast,
publish_auth_cache_invalidation,
)
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache
from litellm.proxy.common_utils.user_api_key_cache import (
AUTH_OBJECTS_TARGET,
UserApiKeyCache,
budget_windows_absent_cache_key,
)
class _RecordingRedisClient(Redis):
@ -243,6 +248,28 @@ async def test_subscriber_deletes_key_object_partition_entry_on_message() -> Non
assert cache.get_cache(hashed_token, model_type=UserAPIKeyAuth) is None
@pytest.mark.asyncio
async def test_subscriber_evicts_a_user_whose_id_looks_like_a_no_window_marker_along_with_the_marker() -> None:
user_id: Final = budget_windows_absent_cache_key("key", "alice")
cache: Final = UserApiKeyCache()
cache.set_cache(user_id, {"user_id": user_id, "models": ["gpt-4o"]})
cache.budget_windows_absent_cache.set_cache(key=user_id, value=True)
pubsub: Final = _QueuePubSub(initial_messages=[_invalidation_message(user_id)])
subscriber: Final = AuthCacheInvalidationSubscriber(
redis_cache=_FakeRedisCache(client=_ScriptedPubSubRedisClient(pubsubs=[pubsub])),
user_api_key_cache=cache,
)
subscriber.start()
try:
assert await asyncio.wait_for(pubsub.drained.wait(), timeout=5)
finally:
await subscriber.stop()
assert cache.get_cache(user_id) is None
assert cache.budget_windows_absent_cache.get_cache(key=user_id) is None
@pytest.mark.asyncio
async def test_subscriber_deletes_additional_in_memory_cache_entry_on_message() -> None:
"""

View file

@ -1,6 +1,6 @@
import hashlib
import json
from typing import Any
from typing import Any, Final
import pytest
@ -11,6 +11,7 @@ from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.user_api_key_cache import (
UserApiKeyCache,
budget_windows_absent_cache_key,
end_user_cache_key,
get_management_object_ttl,
is_user_key_cache_key,
@ -32,6 +33,17 @@ class CapturingInMemoryCache(InMemoryCache):
super().set_cache(key, value, **kwargs)
class _ManualClock:
def __init__(self) -> None:
self.seconds = 1000.0
def __call__(self) -> float:
return self.seconds
def advance(self, seconds: float) -> None:
self.seconds += seconds
class FakeRedisCache(RedisCache):
"""
In-memory fake that enforces the UserApiKeyCache Redis payload contract.
@ -432,6 +444,29 @@ class TestUserKeyObjectPartition:
assert first_key.token == tokens[0]
assert cache.get_cache(end_user_cache_key("u0")) == {"user_id": "u0"}
def test_update_in_memory_max_size_applies_to_no_window_notes(self):
cache = UserApiKeyCache()
cache.update_in_memory_max_size(1)
cache.budget_windows_absent_cache.set_cache(key=budget_windows_absent_cache_key("key", "first"), value=True)
cache.budget_windows_absent_cache.set_cache(key=budget_windows_absent_cache_key("key", "second"), value=True)
assert cache.budget_windows_absent_cache.get_cache(key=budget_windows_absent_cache_key("key", "first")) is None
assert cache.budget_windows_absent_cache.get_cache(key=budget_windows_absent_cache_key("key", "second")) is True
def test_update_cache_ttl_applies_to_no_window_notes(self):
clock: Final = _ManualClock()
cache: Final = UserApiKeyCache(
default_in_memory_ttl=60, budget_windows_absent_in_memory_cache=InMemoryCache(clock=clock)
)
cache.update_cache_ttl(default_in_memory_ttl=5, default_redis_ttl=5)
note: Final = budget_windows_absent_cache_key("key", "no-windows")
cache.budget_windows_absent_cache.set_cache(key=note, value=True)
clock.advance(6)
assert cache.budget_windows_absent_cache.get_cache(key=note) is None
def test_update_in_memory_max_size_none_resets_key_object_partition_to_default(self):
cache = UserApiKeyCache(key_object_in_memory_cache=InMemoryCache(max_size_in_memory=1))
cache.update_in_memory_max_size(None)

View file

@ -471,3 +471,34 @@ def test_settings_store_truthiness_matches_emptiness() -> None:
assert bool(store) is True
del store["max_parallel_requests"]
assert bool(store) is False
def test_settings_store_ignores_stored_keys_outside_its_managed_db_keys_and_still_applies_them_from_yaml() -> None:
store: Final = SettingsStore("general_settings", managed_db_keys=frozenset({"max_parallel_requests"}))
stored_row: Final[dict[str, JsonValue]] = {"enforce_user_param": True, "max_parallel_requests": 5}
store.apply_db_row("general_settings", stored_row)
assert dict(store) == {"max_parallel_requests": 5}
assert store.source("enforce_user_param") == "unset"
assert store.unmanaged_db_keys() == ("enforce_user_param",)
store.load_yaml({"enforce_user_param": True})
store.apply_db_row("general_settings", stored_row)
assert dict(store) == {"enforce_user_param": True, "max_parallel_requests": 5}
assert store.source("enforce_user_param") == "config"
assert store.unmanaged_db_keys() == ("enforce_user_param",)
def test_settings_store_managed_db_keys_leave_other_rows_and_unrestricted_stores_whole() -> None:
restricted: Final = SettingsStore("general_settings", managed_db_keys=frozenset())
unrestricted: Final = SettingsStore("general_settings")
restricted.apply_db_row("ui_settings", {"disable_custom_api_keys": True})
unrestricted.apply_db_row("general_settings", {"enforce_user_param": True})
assert dict(restricted) == {"disable_custom_api_keys": True}
assert restricted.unmanaged_db_keys() == ()
assert dict(unrestricted) == {"enforce_user_param": True}
assert unrestricted.unmanaged_db_keys() == ()

View file

@ -715,3 +715,29 @@ def test_image_generation_uses_key_num_retries(monkeypatch: pytest.MonkeyPatch)
router: Final = proxy_server.llm_router
assert router is not None
assert get_deployment_failures_for_current_minute(router, "image-regression-failing") == 3
def test_failed_image_generation_spend_row_records_aimage_generation(monkeypatch: pytest.MonkeyPatch) -> None:
from unittest.mock import AsyncMock, MagicMock
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
spend_writer: Final = MagicMock(update_database=AsyncMock())
monkeypatch.setattr(litellm, "callbacks", [ProxyDBLogger(spend_writer=lambda: spend_writer)])
client, handler = _image_generation_client(monkeypatch, UserAPIKeyAuth(api_key="sk-test"))
try:
with client:
response: Final = client.post("/v1/images/generations", json={"model": "A", "prompt": "draw a tree"})
finally:
ProxyLogging._callback_capabilities_cache.clear()
assert (response.status_code, handler.calls) == (429, ["failing"]), response.text
db_call: Final = spend_writer.update_database.call_args.kwargs
payload: Final = get_logging_payload(
kwargs=db_call["kwargs"],
response_obj=db_call["completion_response"],
start_time=db_call["start_time"],
end_time=db_call["end_time"],
)
assert (payload["status"], payload["call_type"]) == ("failure", "aimage_generation")

View file

@ -6,7 +6,7 @@ from contextlib import ExitStack
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import AsyncMock, MagicMock, call, patch
from uuid import UUID, uuid4
import pytest
@ -51,7 +51,11 @@ from litellm.proxy.auth.auth_checks import (
)
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import AuthCacheInvalidationSubscriber
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, project_cache_key
from litellm.proxy.common_utils.user_api_key_cache import (
UserApiKeyCache,
budget_windows_absent_cache_key,
project_cache_key,
)
from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker
from litellm.proxy.management_endpoints.key_management_endpoints import (
KeyProjectBindingError,
@ -8911,7 +8915,10 @@ async def test_delete_cache_key_object_broadcasts_invalidation(monkeypatch):
# Real, observable state: the cache object itself no longer holds the entry.
assert real_user_api_key_cache.get_cache(key="hashed-broadcast-key") is None
mock_publish.assert_awaited_once_with(cache_key="hashed-broadcast-key")
assert mock_publish.await_args_list == [
call(cache_key="hashed-broadcast-key"),
call(cache_key=budget_windows_absent_cache_key("key", "hashed-broadcast-key")),
]
@pytest.mark.asyncio

View file

@ -2684,7 +2684,7 @@ async def test_team_block_unblock_reach_warmed_auth_on_every_worker(
from litellm.proxy._types import BlockTeamRequest
from litellm.proxy.auth.auth_checks import get_team_object
from litellm.proxy.auth.team_grants import team_model_aliases
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, budget_windows_absent_cache_key
from litellm.proxy.management_endpoints import team_endpoints
team_id: Final = "team-kill-switch"
@ -2721,7 +2721,11 @@ async def test_team_block_unblock_reach_warmed_auth_on_every_worker(
seen_by_auth: Final = await get_team_object(team_id=team_id, prisma_client=prisma_client, user_api_key_cache=cache)
assert seen_by_auth.blocked is blocked
assert team_model_aliases(seen_by_auth) == {"fast": "gpt-4o"}
assert redis.broadcast_keys == [f"team_id:{team_id}", "team_alias:kill-switch"]
assert redis.broadcast_keys == [
f"team_id:{team_id}",
"team_alias:kill-switch",
budget_windows_absent_cache_key("team", team_id),
]
assert (response.blocked, response.litellm_model_table, response.object_permission) == (blocked, None, None)

View file

@ -17,6 +17,8 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
if TYPE_CHECKING:
import respx
from litellm import Router
from tests.unit.proxy.router_entry_point_stubs import route_entry_points_to_stubs
@ -563,3 +565,72 @@ def openai_only_router(monkeypatch: pytest.MonkeyPatch) -> Router:
monkeypatch.setattr(proxy_server, "general_settings", {})
monkeypatch.setattr(proxy_server, "user_model", None)
return router
@pytest.fixture
def failed_spend_row(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
) -> Iterator[Callable[[], tuple[str, str]]]:
"""Route tts-1, whisper-1 and omni-moderation to an OpenAI deployment whose provider answers 400,
and return a reader for the (status, call_type) of the spend row the failure wrote."""
import litellm
from litellm import Router
from litellm.proxy import proxy_server
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.proxy_server import ProxyConfig
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.proxy.utils import ProxyLogging
api_base: Final = "https://failing-provider.example/v1"
respx_mock.post(url__startswith=api_base).respond(
400, json={"error": {"message": "rejected by the provider", "type": "invalid_request_error"}}
)
router: Final = Router(
model_list=[
{
"model_name": model_name,
"litellm_params": {
"model": f"openai/{provider_model}",
"api_base": api_base,
"api_key": "provider-key",
"max_retries": 0,
},
}
for model_name, provider_model in (
("tts-1", "tts-1"),
("whisper-1", "whisper-1"),
("omni-moderation", "omni-moderation-latest"),
)
],
num_retries=0,
)
cache: Final = UserApiKeyCache()
spend_writer: Final = MagicMock(update_database=AsyncMock())
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "callbacks", [ProxyDBLogger(spend_writer=lambda: spend_writer)])
for name, value in (
("llm_router", router),
("llm_model_list", router.model_list),
("prisma_client", None),
("general_settings", {}),
("user_model", None),
("proxy_config", ProxyConfig()),
("proxy_logging_obj", ProxyLogging(user_api_key_cache=cache)),
("user_api_key_cache", cache),
("version", "test-version"),
):
monkeypatch.setattr(proxy_server, name, value)
def last_row() -> tuple[str, str]:
db_call: Final = spend_writer.update_database.call_args.kwargs
payload: Final = get_logging_payload(
kwargs=db_call["kwargs"],
response_obj=db_call["completion_response"],
start_time=db_call["start_time"],
end_time=db_call["end_time"],
)
return payload["status"], payload["call_type"]
yield last_row
ProxyLogging._callback_capabilities_cache.clear()

View file

@ -32,11 +32,13 @@ import litellm
from litellm._logging import turn_on_json
from litellm.proxy._types import CommonProxyErrors, ConfigGeneralSettings
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
from litellm.proxy.config_resolvers.settings_store import SettingsStore
from litellm.router_api import ApplyResult, RouterSettings
from litellm.types.router import Deployment, LiteLLM_Params
from litellm.proxy.proxy_server import (
ProxyConfig,
_is_remote_module_url,
_reload_settings_store,
_scrub_db_overlay_remote_module_loads,
_scrub_guardrail_inner,
resolve_complexity_router_plugins,
@ -70,6 +72,15 @@ def _db_model(model_id: str, model_name: str = "gpt-3.5-turbo") -> SimpleNamespa
)
def test_yaml_settings_reload_keeps_db_general_settings_until_sync() -> None:
settings: Final = SettingsStore("general_settings")
settings.apply_db_row("general_settings", {"max_file_downloads_per_minute": 2})
_reload_settings_store("general_settings", settings, {})
assert settings["max_file_downloads_per_minute"] == 2
@pytest.mark.asyncio
async def test_proxy_config_adds_and_deletes_stale_deployment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.proxy import proxy_server
@ -102,7 +113,9 @@ def test_proxy_config_upserts_existing_deployment_without_duplicating_it(monkeyp
@pytest.mark.asyncio
async def test_proxy_config_preserves_models_when_config_read_fails(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
async def test_proxy_config_preserves_models_when_config_read_fails(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.proxy import proxy_server
router = litellm.Router(model_list=[_deployment("config-deployment").to_json(exclude_none=True)])
@ -135,6 +148,8 @@ async def test_proxy_config_keeps_database_models_and_deletes_stale_models(
assert desired == frozenset({"kept-deployment"})
assert router.get_model_ids() == ["kept-deployment"]
from tests._master_key import MASTER_KEY
@ -150,7 +165,6 @@ async def test_proxy_config_loads_lens_store_from_yaml(tmp_path, monkeypatch) ->
assert is_lens_tracing_enabled(settings["tracing"], {}) is True
@pytest.mark.asyncio
@pytest.mark.parametrize("shutdown_error", [False, True])
async def test_tracing_config_automatically_exports_spend_without_a_storage_dependency(
@ -176,7 +190,9 @@ async def test_tracing_config_automatically_exports_spend_without_a_storage_depe
outcome: Final = pytest.raises(RuntimeError, match="shutdown failure") if shutdown_error else nullcontext()
with outcome:
async with manage_tracing(enabled=True, client_factory=client):
logger: Final = next(callback for callback in litellm._async_success_callback if isinstance(callback, LensExporter))
logger: Final = next(
callback for callback in litellm._async_success_callback if isinstance(callback, LensExporter)
)
await logger.async_log_success_event(
{"standard_logging_object": {"id": "response-1", "response_cost": 0.25}}, None, None, None
)
@ -2693,7 +2709,6 @@ async def test_load_config_rejects_non_boolean_deny_by_default(
await ProxyConfig().load_config(router=None, config_file_path=str(config_file))
@pytest.mark.asyncio
@pytest.mark.parametrize(
"yaml_value", ["files-api-2025-04-14", "[files-api-2025-04-14, 7]"], ids=["bare-string", "non-string-entry"]
@ -2713,6 +2728,7 @@ async def test_load_config_rejects_a_malformed_vertex_drop_anthropic_betas_list(
with pytest.raises(ValidationError, match="vertex_ai_pass_through_drop_anthropic_betas"):
await ProxyConfig().load_config(router=None, config_file_path=str(config_file))
@pytest.mark.asyncio
async def test_ProxyConfig_load_config_resolves_router_settings_plugins(tmp_path, monkeypatch):
"""Regression: router_settings.plugins dotted-path strings must be resolved to
@ -4470,6 +4486,86 @@ async def test_ProxyConfig__update_general_settings_updates_health_check_retenti
reschedule.assert_awaited_once()
@pytest.mark.asyncio
async def test_ProxyConfig__update_general_settings_ignores_a_stored_key_no_settings_api_manages_and_warns_once(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {})
pc: Final = ProxyConfig()
stored_row: Final = {"enforce_user_param": True, "store_prompts_in_spend_logs": True}
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
await pc._update_general_settings(stored_row)
await pc._update_general_settings(stored_row)
from litellm.proxy import proxy_server
assert proxy_server.general_settings["store_prompts_in_spend_logs"] is True
assert "enforce_user_param" not in proxy_server.general_settings
warnings: Final = [record.getMessage() for record in caplog.records if "enforce_user_param" in record.getMessage()]
assert warnings == [
"general_settings.enforce_user_param is stored in the database, but the proxy does not apply that stored "
"value, so it is ignored. Set it in the config file to use it, or remove it with POST /config/field/delete."
]
@pytest.mark.asyncio
async def test_ProxyConfig__update_general_settings_warns_about_an_ignored_stored_key_the_config_file_also_sets(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
pc: Final = ProxyConfig()
pc._load_yaml_settings_stores({"general_settings": {"enforce_user_param": False}})
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", pc.settings)
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
await pc._update_general_settings({"enforce_user_param": True})
from litellm.proxy import proxy_server
assert proxy_server.general_settings["enforce_user_param"] is False
warnings: Final = [record.getMessage() for record in caplog.records if "enforce_user_param" in record.getMessage()]
assert warnings == [
"general_settings.enforce_user_param is stored in the database, but the proxy does not apply that stored "
"value, so it is ignored. The config file's value applies. Remove the stored copy with POST "
"/config/field/delete."
]
@pytest.mark.asyncio
async def test_ProxyConfig__update_general_settings_does_not_warn_about_a_stored_copy_of_an_admin_ui_field(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
pc: Final = ProxyConfig()
pc._load_yaml_settings_stores({"general_settings": {}})
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", pc.settings)
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
await pc._update_general_settings({"budget_rollover": True, "enforce_user_param": True})
warned: Final = [record.getMessage().split(" ")[0] for record in caplog.records if "is ignored" in record.getMessage()]
assert warned == ["general_settings.enforce_user_param"]
@pytest.mark.asyncio
async def test_ProxyConfig__update_general_settings_warns_once_each_time_the_stored_row_stops_being_an_object(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
pc: Final = ProxyConfig()
pc._load_yaml_settings_stores({"general_settings": {}})
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", pc.settings)
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
await pc._update_general_settings(None)
await pc._update_general_settings('{"store_prompts_in_spend_logs": true}')
await pc._update_general_settings('{"store_prompts_in_spend_logs": true}')
await pc._update_general_settings({"store_prompts_in_spend_logs": True})
await pc._update_general_settings('{"store_prompts_in_spend_logs": true}')
warnings: Final = [record.getMessage() for record in caplog.records if "JSON object" in record.getMessage()]
assert warnings == [
"general_settings is stored in the database as something other than a JSON object, so the proxy ignores "
"every stored general setting."
] * 2
def _paused_scheduler(monkeypatch):
from apscheduler.schedulers.asyncio import AsyncIOScheduler
@ -4918,9 +5014,6 @@ async def test_ProxyConfig__update_general_settings_none_input_noop():
# None input returns early.
result = await pc._update_general_settings(db_general_settings=None)
assert result is None
# Error-style: dict() will fail on non-mapping non-None input.
with pytest.raises(TypeError):
await pc._update_general_settings(db_general_settings=12345) # type: ignore[arg-type]
@pytest.mark.asyncio
@ -5080,7 +5173,9 @@ async def test_ProxyConfig__update_config_from_db_keeps_keys_the_config_file_omi
@pytest.mark.asyncio
@pytest.mark.parametrize("ui_settings_already_synced", [False, True])
async def test_ProxyConfig_add_deployment_continues_after_null_pass_through_endpoints(monkeypatch, ui_settings_already_synced):
async def test_ProxyConfig_add_deployment_continues_after_null_pass_through_endpoints(
monkeypatch: pytest.MonkeyPatch, ui_settings_already_synced: bool
) -> None:
from litellm.proxy import proxy_server
pc = ProxyConfig()
@ -5106,6 +5201,32 @@ async def test_ProxyConfig_add_deployment_continues_after_null_pass_through_endp
assert settings_refresh.await_count == (0 if ui_settings_already_synced else 1)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"stored_row, rebuilds_routes",
[({"pass_through_endpoints": []}, True), ('{"pass_through_endpoints": []}', False), ([], False)],
ids=["object", "json-string", "array"],
)
async def test_ProxyConfig_add_deployment_rebuilds_pass_through_routes_only_from_an_object_general_settings_row(
monkeypatch: pytest.MonkeyPatch, stored_row: object, rebuilds_routes: bool
) -> None:
from litellm.proxy import proxy_server
from litellm.proxy.pass_through_endpoints import pass_through_endpoints
pc: Final = ProxyConfig()
rebuild_routes: Final = AsyncMock()
monkeypatch.setattr(proxy_server, "general_settings", {})
monkeypatch.setattr(proxy_server, "prefetch_config_params", AsyncMock())
monkeypatch.setattr(proxy_server, "get_config_param", AsyncMock(return_value=SimpleNamespace(param_value=stored_row)))
monkeypatch.setattr(pass_through_endpoints, "initialize_pass_through_endpoints_in_db", rebuild_routes)
monkeypatch.setattr(pc, "_should_load_db_object", lambda *, object_type: object_type == "pass_through_endpoints")
monkeypatch.setattr(pc, "get_credentials", AsyncMock())
await pc.add_deployment(prisma_client=MagicMock(), proxy_logging_obj=MagicMock(), ui_settings_already_synced=True)
assert rebuild_routes.await_count == (1 if rebuilds_routes else 0)
# ---------------------------------------------------------------------------
# ProxyConfig._update_config_from_db
# ---------------------------------------------------------------------------
@ -5285,9 +5406,7 @@ def revision_test_environment(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv(key)
def _environment_variables_row(
values: Mapping[str, str | None] | None, revision: int | None
) -> SimpleNamespace | None:
def _environment_variables_row(values: Mapping[str, str | None] | None, revision: int | None) -> SimpleNamespace | None:
if values is None:
return None
return SimpleNamespace(
@ -5473,9 +5592,7 @@ async def test_environment_variables_row_read_before_an_overlapping_newer_sync_i
await newer_sync_finished.wait()
older_sync: Final = asyncio.create_task(
_sync_from(
pc, _config_database(read_older_row, AsyncMock(return_value=older_row), wait_for_the_newer_sync)
)
_sync_from(pc, _config_database(read_older_row, AsyncMock(return_value=older_row), wait_for_the_newer_sync))
)
await older_environment_row_read.wait()
await _sync_from(pc, _rows_database(newer_row, newer_row))

View file

@ -9,7 +9,9 @@ Pins (PR2):
from __future__ import annotations
import contextlib
import io
from typing import TYPE_CHECKING, Callable, Final
from unittest.mock import AsyncMock, MagicMock
import httpx
@ -22,6 +24,9 @@ from litellm.proxy._types import ProxyErrorTypes, ProxyException
from litellm.types.llms.openai import HttpxBinaryResponseContent
from litellm.types.router import RouterErrors
if TYPE_CHECKING:
from fastapi.testclient import TestClient
_TAG_ROUTING_DENIAL = f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=tts-1 and tags=['x']"
@ -359,3 +364,32 @@ def test_audio_transcription_error(client, auth_as, patched_transcription_error,
response = client.post(path, files=files, data=data)
assert response.status_code == 500
assert len(response.content) > 0
def _post_speech(client: TestClient) -> httpx.Response:
return client.post("/v1/audio/speech", json={"model": "tts-1", "input": "Hi", "voice": "alloy"})
def _post_transcription(client: TestClient) -> httpx.Response:
return client.post(
"/v1/audio/transcriptions",
files={"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")},
data={"model": "whisper-1"},
)
@pytest.mark.parametrize(
("send", "call_type"),
[(_post_speech, "aspeech"), (_post_transcription, "atranscription")],
)
def test_a_provider_rejection_logs_the_audio_endpoint_call_type(
client: TestClient,
auth_as: Callable[..., contextlib.AbstractContextManager[object]],
failed_spend_row: Callable[[], tuple[str, str]],
send: Callable[[TestClient], httpx.Response],
call_type: str,
) -> None:
with auth_as():
response: Final = send(client)
assert response.status_code == 400, response.text
assert failed_spend_row() == ("failure", call_type)

View file

@ -7,12 +7,17 @@ Pins (PR2):
from __future__ import annotations
import contextlib
from typing import TYPE_CHECKING, Callable, Final
from unittest.mock import AsyncMock, MagicMock
import pytest
from litellm.proxy import proxy_server
if TYPE_CHECKING:
from fastapi.testclient import TestClient
from .conftest import normalize # type: ignore[import-not-found]
HAPPY_RESPONSE = {
@ -109,3 +114,14 @@ def test_moderation_error(client, auth_as, moderation_pipeline_raises, path):
response = client.post(path, json=payload)
assert response.status_code == 500
assert len(response.content) > 0
def test_a_provider_rejection_logs_the_moderation_call_type(
client: TestClient,
auth_as: Callable[..., contextlib.AbstractContextManager[object]],
failed_spend_row: Callable[[], tuple[str, str]],
) -> None:
with auth_as():
response: Final = client.post("/v1/moderations", json={"model": "omni-moderation", "input": ["one", "two"]})
assert response.status_code == 400, response.text
assert failed_spend_row() == ("failure", "amoderation")

View file

@ -5,10 +5,13 @@ Tests for rerank_endpoints/endpoints.py response headers.
import json
import logging
from collections.abc import Iterator
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi import HTTPException, Request, Response
import respx
from fastapi import FastAPI, HTTPException, Request, Response
from fastapi.testclient import TestClient
import litellm.proxy.common_request_processing as common_request_processing_mod
import litellm.proxy.proxy_server as proxy_server_mod
@ -209,3 +212,68 @@ async def test_a_rejection_raised_before_routing_keeps_its_own_status(monkeypatc
error = await _rerank_failure(rejection, raised_before_routing=True, monkeypatch=monkeypatch)
assert (error.type, error.param, error.code) == ("bad_request_error", "session_id", "400")
def test_failed_rerank_spend_row_records_arerank(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
) -> None:
import litellm
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.proxy_server import ProxyConfig
from litellm.proxy.rerank_endpoints.endpoints import router as rerank_router
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.proxy.utils import ProxyLogging
provider: Final = respx_mock.post(url__startswith="https://rerank-provider.example").respond(
400, json={"message": "invalid documents"}
)
cache: Final = UserApiKeyCache()
spend_writer: Final = MagicMock(update_database=AsyncMock())
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "callbacks", [ProxyDBLogger(spend_writer=lambda: spend_writer)])
monkeypatch.setattr(proxy_server_mod, "general_settings", {})
monkeypatch.setattr(
proxy_server_mod,
"llm_router",
litellm.Router(
model_list=[
{
"model_name": "rerank-model",
"litellm_params": {
"model": "cohere/rerank-v3.5",
"api_base": "https://rerank-provider.example",
"api_key": "provider-key",
},
}
],
num_retries=0,
),
)
monkeypatch.setattr(proxy_server_mod, "proxy_config", ProxyConfig())
monkeypatch.setattr(proxy_server_mod, "proxy_logging_obj", ProxyLogging(user_api_key_cache=cache))
monkeypatch.setattr(proxy_server_mod, "user_api_key_cache", cache)
monkeypatch.setattr(proxy_server_mod, "user_model", None)
monkeypatch.setattr(proxy_server_mod, "version", "1.2.3")
app: Final = FastAPI()
app.include_router(rerank_router)
app.add_exception_handler(ProxyException, proxy_server_mod.openai_exception_handler)
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-test")
try:
with TestClient(app, raise_server_exceptions=False) as client:
response: Final = client.post(
"/v1/rerank", json={"model": "rerank-model", "query": "q", "documents": ["a", "b"]}
)
finally:
ProxyLogging._callback_capabilities_cache.clear()
assert (response.status_code, provider.call_count) == (400, 1), response.text
db_call: Final = spend_writer.update_database.call_args.kwargs
payload: Final = get_logging_payload(
kwargs=db_call["kwargs"],
response_obj=db_call["completion_response"],
start_time=db_call["start_time"],
end_time=db_call["end_time"],
)
assert (payload["status"], payload["call_type"]) == ("failure", "arerank")

View file

@ -5150,6 +5150,23 @@ def test_a_background_response_billing_event_writes_a_row_apart_from_the_read_th
assert request_ids == ("resp_shared", "resp_shared_background_cost")
@pytest.mark.parametrize("call_type", ["aget_responses", "acancel_responses"])
def test_a_read_or_cancel_row_carries_the_id_its_callbacks_logged(call_type: str):
row: Final = get_logging_payload(
kwargs={
"call_type": call_type,
"model": "gpt-5.6",
"litellm_call_id": "call-read",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
response_obj={"id": "resp_logged_by_callbacks", "object": "response", "status": "completed"},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert row["request_id"] == "resp_logged_by_callbacks"
@pytest.mark.parametrize("forged_marker", [True, BackgroundResponseSettlement.SETTLES.value])
def test_a_request_body_cannot_move_its_spend_row_onto_a_background_settlement_id(forged_marker):
"""A failed call logs its raw request body as kwargs, so a body field named like the settlement

View file

@ -66,7 +66,7 @@ def spend_counter_state():
original_prisma_client = ps.prisma_client
counter_cache = DualCache()
key_cache = DualCache()
key_cache = UserApiKeyCache()
ps.spend_counter_cache = counter_cache
ps.user_api_key_cache = key_cache
ps.prisma_client = None

View file

@ -1,5 +1,6 @@
import asyncio
import contextlib
import dataclasses
import importlib
import json
import logging
@ -10,9 +11,10 @@ import subprocess
import sys
import time
import types
from collections.abc import Callable
from collections.abc import Callable, Iterator, Mapping, Sequence
from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import MappingProxyType
from typing import Final
from unittest import mock
from unittest.mock import AsyncMock, MagicMock, call, create_autospec, mock_open, patch
@ -48,6 +50,7 @@ from litellm.proxy._types import (
)
from litellm.proxy.auth.login_throttle import LoginThrottle
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import WindowSpendUpdateQueue
from litellm.proxy.hooks.parallel_request_limiter_v3 import RequestRateLimiterStash
from litellm.proxy.proxy_server import app, aws_error_type_exception_handler, initialize, openai_exception_handler
from litellm.utils import _invalidate_model_cost_lowercase_map
@ -1804,9 +1807,7 @@ class _DatabaseWithOneStoredCredential:
@pytest.mark.asyncio
@pytest.mark.parametrize(
"encrypted_with, asks_to_migrate", [(PUBLICLY_KNOWN_KEY, True), ("sk-some-other-key", False)]
)
@pytest.mark.parametrize("encrypted_with, asks_to_migrate", [(PUBLICLY_KNOWN_KEY, True), ("sk-some-other-key", False)])
async def test_proxy_startup_asks_to_migrate_only_when_the_database_holds_values_under_the_unsafe_key(
monkeypatch, tmp_path, encrypted_with, asks_to_migrate
):
@ -1924,9 +1925,7 @@ async def test_proxy_startup_names_the_config_file_that_set_the_unsafe_key(monke
from litellm.proxy.auth.master_key_boot_check import UnsafeMasterKeyError
from litellm.proxy.proxy_server import proxy_startup_event
config_path, announced = _boot_with_general_settings(
monkeypatch, tmp_path, {"master_key": PUBLICLY_KNOWN_KEY}
)
config_path, announced = _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": PUBLICLY_KNOWN_KEY})
with pytest.raises(UnsafeMasterKeyError):
async with proxy_startup_event(FastAPI()):
@ -3807,6 +3806,24 @@ async def test_db_reload_keeps_the_resolved_value_of_a_config_owned_env_referenc
assert proxy_server_module.store_model_in_db is True
@pytest.mark.asyncio
async def test_db_reload_of_a_json_string_general_settings_row_reports_no_ignored_keys(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.proxy import proxy_server as proxy_server_module
from litellm.proxy.proxy_server import ProxyConfig
monkeypatch.setattr(proxy_server_module, "user_api_key_cache", MagicMock(), raising=False)
proxy_config: Final = ProxyConfig()
proxy_config.settings.load_yaml({})
monkeypatch.setattr(proxy_server_module, "general_settings", proxy_config.settings, raising=False)
await proxy_config._update_general_settings('{"budget_rollover": true, "master_key": "sk-stored"}')
assert proxy_config.settings.unmanaged_db_keys() == ()
assert proxy_config.settings.source("master_key") != "db"
def test_max_ui_session_budget_default_is_one_dollar():
"""LIT-4662: the dashboard session budget default is a product decision; the
old 0.25 default locked admins out of auto router Test Connection and the
@ -7295,7 +7312,7 @@ def test_deep_merge_dicts_skips_none_and_empty_lists(monkeypatch):
db_param_value = {
"max_parallel_requests": None,
"allowed_models": [],
"new_key": "new_value",
"store_prompts_in_spend_logs": True,
"nested": {
"key1": "updated_value1",
"key3": "value3",
@ -7308,7 +7325,7 @@ def test_deep_merge_dicts_skips_none_and_empty_lists(monkeypatch):
assert result["max_parallel_requests"] == 10
assert result["allowed_models"] == ["gpt-3.5-turbo", "gpt-4"]
assert result["new_key"] == "new_value"
assert result["store_prompts_in_spend_logs"] is True
assert result["nested"] == {"key1": "value1", "key2": "value2"}
@ -8394,10 +8411,18 @@ async def test_update_general_settings_db_pass_through_endpoint_overrides_yaml_e
request.headers = {}
request.query_params = {}
settings: Final = patch("litellm.proxy.proxy_server.general_settings", {"pass_through_endpoints": [yaml_endpoint]}) # test-quality-ok: the method reads this module global; no injection seam
yaml_endpoints: Final = patch("litellm.proxy.proxy_server.config_passthrough_endpoints", [yaml_endpoint]) # test-quality-ok: module global holding the YAML endpoints the fix merges in
initialize: Final = patch("litellm.proxy.proxy_server.initialize_pass_through_endpoints", AsyncMock()) # test-quality-ok: route registration needs the FastAPI app; auth is the observable here
master_key: Final = patch("litellm.proxy.proxy_server.master_key", MASTER_KEY) # test-quality-ok: a set master key is what makes a missing Authorization header a 401
settings: Final = patch(
"litellm.proxy.proxy_server.general_settings", {"pass_through_endpoints": [yaml_endpoint]}
) # test-quality-ok: the method reads this module global; no injection seam
yaml_endpoints: Final = patch(
"litellm.proxy.proxy_server.config_passthrough_endpoints", [yaml_endpoint]
) # test-quality-ok: module global holding the YAML endpoints the fix merges in
initialize: Final = patch(
"litellm.proxy.proxy_server.initialize_pass_through_endpoints", AsyncMock()
) # test-quality-ok: route registration needs the FastAPI app; auth is the observable here
master_key: Final = patch(
"litellm.proxy.proxy_server.master_key", MASTER_KEY
) # test-quality-ok: a set master key is what makes a missing Authorization header a 401
with settings, yaml_endpoints, initialize, master_key:
await ProxyConfig()._update_general_settings(db_general_settings={"pass_through_endpoints": [db_endpoint]})
@ -14075,17 +14100,18 @@ def test_startup_is_silent_when_mock_testing_params_disabled(caplog):
@contextlib.contextmanager
def _window_spend_enqueue_env(cached_objects: dict):
def _window_spend_enqueue_env(
cached_objects: Mapping[str, object], prisma_client: MagicMock | None = None
) -> Iterator[WindowSpendUpdateQueue]:
"""Point increment_spend_counters at throwaway caches and a real
WindowSpendUpdateQueue, and hand back the queue to inspect."""
import litellm.proxy.proxy_server as ps
from litellm.caching.dual_cache import DualCache
from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
WindowSpendUpdateQueue,
)
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
user_api_key_cache = MagicMock()
user_api_key_cache.async_get_cache = AsyncMock(side_effect=lambda key, **_: cached_objects.get(key))
user_api_key_cache: Final = UserApiKeyCache()
for key, value in cached_objects.items():
user_api_key_cache.set_cache(key=key, value=value)
queue = WindowSpendUpdateQueue()
proxy_logging_obj = MagicMock()
@ -14099,7 +14125,7 @@ def _window_spend_enqueue_env(cached_objects: dict):
)
ps.user_api_key_cache = user_api_key_cache
ps.spend_counter_cache = DualCache()
ps.prisma_client = None
ps.prisma_client = prisma_client
ps.proxy_logging_obj = proxy_logging_obj
try:
yield queue
@ -14287,6 +14313,293 @@ async def test_team_window_spend_row_carries_the_request_start_time():
assert enqueued[0]["started_at"] == "2026-08-10T12:00:00.500000"
_WINDOW_RESET_AT: Final = "2026-08-17T00:00:00+00:00"
def _window(budget_duration: str, max_budget: float) -> dict[str, object]:
return {"budget_duration": budget_duration, "max_budget": max_budget, "reset_at": _WINDOW_RESET_AT}
@dataclasses.dataclass(frozen=True, slots=True)
class _StoredRow:
budget_limits: list[dict[str, object]] | None
def _prisma_with_rows(
keys: Mapping[str, _StoredRow] = MappingProxyType({}),
teams: Mapping[str, _StoredRow] = MappingProxyType({}),
) -> MagicMock:
"""Key and team tables that answer find_unique from the given rows."""
client: Final = MagicMock()
client.db.litellm_verificationtoken.find_unique = AsyncMock(side_effect=lambda where: keys.get(where["token"]))
client.db.litellm_teamtable.find_unique = AsyncMock(side_effect=lambda where: teams.get(where["team_id"]))
return client
def _charged_windows(enqueued: Sequence[Mapping[str, object]]) -> list[tuple[object, object, object]]:
return [(item["entity_id"], item["window_duration"], item["spend"]) for item in enqueued]
@pytest.mark.asyncio
async def test_key_window_is_charged_when_the_cached_key_expired_before_the_request_ended():
"""A stream that outlives user_api_key_cache_ttl finds no cached key when its
cost lands. Its spend still belongs to the key's budget windows."""
import litellm.proxy.proxy_server as ps
from litellm.proxy.proxy_server import increment_spend_counters
prisma_client: Final = _prisma_with_rows(keys={"hashed-token": _StoredRow([_window("30d", 100.0)])})
with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue:
await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25)
enqueued: Final = await _drain(queue)
assert await ps.user_api_key_cache.async_get_cache(key="hashed-token") is None
assert [item["entity_type"] for item in enqueued] == ["key"]
assert _charged_windows(enqueued) == [("hashed-token", "30d", pytest.approx(0.25))]
@pytest.mark.asyncio
async def test_team_window_is_charged_when_the_cached_team_expired_before_the_request_ended():
import litellm.proxy.proxy_server as ps
from litellm.proxy.proxy_server import increment_spend_counters
prisma_client: Final = _prisma_with_rows(teams={"team-1": _StoredRow([_window("7d", 50.0)])})
with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue:
await increment_spend_counters(token=None, team_id="team-1", user_id=None, response_cost=1.5)
enqueued: Final = await _drain(queue)
assert await ps.user_api_key_cache.async_get_cache(key="team_id:team-1") is None
assert [item["entity_type"] for item in enqueued] == ["team"]
assert _charged_windows(enqueued) == [("team-1", "7d", pytest.approx(1.5))]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"token, keys",
[
pytest.param("litellm_proxy_master_key", {}, id="no-key-row"),
pytest.param("hashed-token", {"hashed-token": _StoredRow(budget_limits=None)}, id="key-row-without-windows"),
],
)
async def test_a_token_without_budget_windows_reads_the_database_once_across_requests(
token: str, keys: Mapping[str, _StoredRow]
):
from litellm.proxy.proxy_server import increment_spend_counters
prisma_client: Final = _prisma_with_rows(keys=keys)
find_unique: Final = prisma_client.db.litellm_verificationtoken.find_unique
with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue:
await increment_spend_counters(token=token, team_id=None, user_id=None, response_cost=0.25)
reads_after_first_request: Final = find_unique.await_count
for _ in range(3):
await increment_spend_counters(token=token, team_id=None, user_id=None, response_cost=0.25)
enqueued: Final = await _drain(queue)
assert enqueued == []
assert reads_after_first_request > 0
assert find_unique.await_count == reads_after_first_request
@pytest.mark.asyncio
async def test_every_cache_miss_charges_the_windows_the_key_row_holds_at_that_moment():
from litellm.proxy.proxy_server import increment_spend_counters
prisma_client: Final = _prisma_with_rows(keys={"hashed-token": _StoredRow([_window("30d", 100.0)])})
with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue:
await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25)
prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(
return_value=_StoredRow([_window("7d", 5.0)])
)
await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.5)
enqueued: Final = await _drain(queue)
assert sorted(_charged_windows(enqueued), key=lambda charge: str(charge[1])) == [
("hashed-token", "30d", pytest.approx(0.25)),
("hashed-token", "7d", pytest.approx(0.5)),
]
@pytest.mark.asyncio
async def test_a_failed_key_row_read_skips_that_request_and_the_next_one_reads_again():
from litellm.proxy.proxy_server import increment_spend_counters
prisma_client: Final = _prisma_with_rows()
prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(
side_effect=ConnectionError("database is unreachable")
)
with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue:
await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25)
after_failure: Final = await _drain(queue)
prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(
return_value=_StoredRow([_window("7d", 5.0)])
)
await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.5)
after_recovery: Final = await _drain(queue)
assert after_failure == []
assert _charged_windows(after_recovery) == [("hashed-token", "7d", pytest.approx(0.5))]
@pytest.mark.asyncio
async def test_a_late_request_after_a_key_update_charges_windows_the_key_did_not_have_before():
import litellm.proxy.proxy_server as ps
from litellm.proxy.auth.auth_checks import delete_cache_key_object
from litellm.proxy.proxy_server import increment_spend_counters
prisma_client: Final = _prisma_with_rows(keys={"hashed-token": _StoredRow(budget_limits=None)})
with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue:
await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25)
prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(
return_value=_StoredRow([_window("7d", 5.0)])
)
await delete_cache_key_object(
hashed_token="hashed-token", user_api_key_cache=ps.user_api_key_cache, proxy_logging_obj=None
)
await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.5)
enqueued: Final = await _drain(queue)
assert _charged_windows(enqueued) == [("hashed-token", "7d", pytest.approx(0.5))]
@pytest.mark.asyncio
async def test_a_key_update_that_lands_during_the_row_read_does_not_hide_the_windows_it_adds():
import litellm.proxy.proxy_server as ps
from litellm.proxy.auth.auth_checks import delete_cache_key_object
from litellm.proxy.proxy_server import increment_spend_counters
prisma_client: Final = _prisma_with_rows()
updated_row: Final = _StoredRow([_window("7d", 5.0)])
async def read_the_row_from_before_the_update(where: Mapping[str, str]) -> _StoredRow:
prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=updated_row)
await delete_cache_key_object(
hashed_token=where["token"], user_api_key_cache=ps.user_api_key_cache, proxy_logging_obj=None
)
return _StoredRow(budget_limits=None)
prisma_client.db.litellm_verificationtoken.find_unique = read_the_row_from_before_the_update
with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue:
ps.spend_counter_cache.set_cache(key="spend:key:hashed-token", value=0.0)
await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25)
_ = await _drain(queue)
await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.5)
after_the_update: Final = await _drain(queue)
assert _charged_windows(after_the_update) == [("hashed-token", "7d", pytest.approx(0.5))]
@pytest.mark.asyncio
async def test_late_requests_from_many_keys_with_windows_do_not_push_out_a_known_no_window_key():
import litellm.proxy.proxy_server as ps
from litellm.caching.in_memory_cache import DEFAULT_MAX_SIZE_IN_MEMORY
from litellm.proxy.proxy_server import increment_spend_counters
windowed_tokens: Final = tuple(f"windowed-{index}" for index in range(DEFAULT_MAX_SIZE_IN_MEMORY))
rows: Final = {
"no-windows": _StoredRow(budget_limits=None),
**{token: _StoredRow([_window("7d", 5.0)]) for token in windowed_tokens},
}
prisma_client: Final = _prisma_with_rows(keys=rows)
find_unique: Final = prisma_client.db.litellm_verificationtoken.find_unique
def no_window_key_reads() -> int:
return sum(1 for call in find_unique.await_args_list if call.kwargs["where"]["token"] == "no-windows")
with _window_spend_enqueue_env({}, prisma_client=prisma_client):
await increment_spend_counters(token="no-windows", team_id=None, user_id=None, response_cost=0.1)
reads_after_the_first_request: Final = no_window_key_reads()
for token in windowed_tokens:
await increment_spend_counters(token=token, team_id=None, user_id=None, response_cost=0.1)
ps.spend_counter_cache.set_cache(key="spend:key:no-windows", value=0.1)
await increment_spend_counters(token="no-windows", team_id=None, user_id=None, response_cost=0.1)
assert no_window_key_reads() == reads_after_the_first_request
@pytest.mark.asyncio
async def test_a_slow_window_row_read_does_not_hold_back_the_key_spend_counter():
import litellm.proxy.proxy_server as ps
from litellm.proxy.proxy_server import increment_spend_counters
prisma_client: Final = _prisma_with_rows()
read_started: Final = asyncio.Event()
finish_read: Final = asyncio.Event()
async def slow_row_read(where: Mapping[str, str]) -> _StoredRow:
read_started.set()
await finish_read.wait()
return _StoredRow([_window("7d", 5.0)])
prisma_client.db.litellm_verificationtoken.find_unique = slow_row_read
with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue:
ps.spend_counter_cache.set_cache(key="spend:key:hashed-token", value=0.0)
ps.spend_counter_cache.set_cache(key="spend:key:hashed-token:window:7d", value=1.0)
charge: Final = asyncio.create_task(
increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25)
)
assert await asyncio.wait_for(read_started.wait(), timeout=5)
key_spend_while_the_row_read_waits: Final = ps.spend_counter_cache.get_cache(key="spend:key:hashed-token")
finish_read.set()
await charge
window_spend: Final = ps.spend_counter_cache.get_cache(key="spend:key:hashed-token:window:7d")
charged: Final = await _drain(queue)
assert key_spend_while_the_row_read_waits == pytest.approx(0.25)
assert window_spend == pytest.approx(1.25)
assert _charged_windows(charged) == [("hashed-token", "7d", pytest.approx(0.25))]
@pytest.mark.asyncio
async def test_a_late_request_that_misses_the_updated_team_charges_windows_the_team_did_not_have_before():
"""Another worker's /team/update only evicts this worker's cached team, so its next late cost reads the row."""
import litellm.proxy.proxy_server as ps
from litellm.proxy._types import LiteLLM_TeamTable
from litellm.proxy.management_endpoints.team_endpoints import refresh_cached_team
from litellm.proxy.proxy_server import increment_spend_counters
prisma_client: Final = _prisma_with_rows(teams={"team-1": _StoredRow(budget_limits=None)})
updated_row: Final = _StoredRow([_window("7d", 50.0)])
with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue:
await increment_spend_counters(token=None, team_id="team-1", user_id=None, response_cost=1.5)
prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=updated_row)
await refresh_cached_team(
team_row=LiteLLM_TeamTable(team_id="team-1", budget_limits=updated_row.budget_limits),
user_api_key_cache=ps.user_api_key_cache,
proxy_logging_obj=ps.proxy_logging_obj,
)
ps.user_api_key_cache.delete_cache(key="team_id:team-1")
await increment_spend_counters(token=None, team_id="team-1", user_id=None, response_cost=2.0)
enqueued: Final = await _drain(queue)
assert _charged_windows(enqueued) == [("team-1", "7d", pytest.approx(2.0))]
@pytest.mark.asyncio
async def test_late_costs_for_many_tokens_without_windows_leave_the_cached_team_in_place():
from litellm.proxy.proxy_server import increment_spend_counters
team_obj: Final = MagicMock()
team_obj.budget_limits = [_window("7d", 50.0)]
with _window_spend_enqueue_env({"team_id:team-1": team_obj}, prisma_client=_prisma_with_rows()) as queue:
for token_number in range(250):
await increment_spend_counters(
token=f"token-{token_number}", team_id=None, user_id=None, response_cost=0.01
)
await increment_spend_counters(token=None, team_id="team-1", user_id=None, response_cost=1.5)
enqueued: Final = await _drain(queue)
assert _charged_windows(enqueued) == [("team-1", "7d", pytest.approx(1.5))]
def _mock_startup_prisma_client(health_check_error=None, connect_error=None):
client = MagicMock()
client.connect = AsyncMock(side_effect=connect_error)
@ -15844,6 +16157,166 @@ async def test_delete_config_general_settings_still_removes_a_key_the_database_o
assert "max_request_size_mb" not in pc.settings
@pytest.mark.asyncio
async def test_delete_config_general_settings_removes_a_stored_key_no_settings_api_manages(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.proxy._types import ConfigFieldDelete
from litellm.proxy.proxy_server import ProxyConfig, delete_config_general_settings
pc: Final = ProxyConfig()
pc._load_yaml_settings_stores({"general_settings": {}})
stored_row: Final = {"enforce_user_param": True, "store_prompts_in_spend_logs": True}
pc.settings.apply_db_row("general_settings", stored_row)
fake_prisma: Final = _fake_prisma_with_config(stored_row)
monkeypatch.setattr(proxy_server_module, "proxy_config", pc)
monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma)
admin: Final = UserAPIKeyAuth(api_key="hashed-admin", user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN)
await delete_config_general_settings(
data=ConfigFieldDelete(field_name="enforce_user_param", config_type="general_settings"),
user_api_key_dict=admin,
)
written: Final = json.loads(fake_prisma.db.litellm_config.upsert.await_args.kwargs["data"]["update"]["param_value"])
assert written == {"store_prompts_in_spend_logs": True}
assert pc.settings.unmanaged_db_keys() == ()
@pytest.mark.asyncio
async def test_delete_config_general_settings_stops_serving_the_removed_pass_through_routes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from fastapi import FastAPI
from starlette.routing import Route
from litellm.proxy._types import ConfigFieldDelete
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import _registered_pass_through_routes
from litellm.proxy.proxy_server import ProxyConfig, delete_config_general_settings
from litellm.types.passthrough_endpoints.pass_through_endpoints import LITELLM_PASS_THROUGH_ENDPOINT_MARKER
endpoint: Final = {"id": "stored-route", "path": "/stored-route", "target": "http://example.com"}
stored_row: Final = {"pass_through_endpoints": [endpoint], "store_prompts_in_spend_logs": True}
app: Final = FastAPI()
monkeypatch.setattr(proxy_server_module, "app", app)
monkeypatch.setattr(proxy_server_module, "premium_user", True)
monkeypatch.setattr(proxy_server_module, "config_passthrough_endpoints", None)
pc: Final = ProxyConfig()
pc._load_yaml_settings_stores({"general_settings": {}})
pc.settings.apply_db_row("general_settings", stored_row)
monkeypatch.setattr(proxy_server_module, "proxy_config", pc)
monkeypatch.setattr(proxy_server_module, "prisma_client", _fake_prisma_with_config(stored_row))
def served_paths() -> frozenset[str]:
return frozenset(
route.path
for route in app.router.routes
if isinstance(route, Route) and getattr(route.endpoint, LITELLM_PASS_THROUGH_ENDPOINT_MARKER, False) is True
)
saved_registry: Final = dict(_registered_pass_through_routes)
try:
_registered_pass_through_routes.clear()
await pc.serve_pass_through_endpoints([endpoint])
served_before: Final = served_paths()
await delete_config_general_settings(
data=ConfigFieldDelete(field_name="pass_through_endpoints", config_type="general_settings"),
user_api_key_dict=UserAPIKeyAuth(
api_key="hashed-admin", user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN
),
)
served_after: Final = served_paths()
finally:
_registered_pass_through_routes.clear()
_registered_pass_through_routes.update(saved_registry)
assert "/stored-route" in served_before
assert "/stored-route" not in served_after
@pytest.mark.asyncio
async def test_delete_config_general_settings_removes_an_ignored_stored_key_the_config_file_also_sets(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.proxy._types import ConfigFieldDelete
from litellm.proxy.proxy_server import ProxyConfig, delete_config_general_settings
pc: Final = ProxyConfig()
pc._load_yaml_settings_stores({"general_settings": {"enforce_user_param": False}})
stored_row: Final = {"enforce_user_param": True, "store_prompts_in_spend_logs": True}
pc.settings.apply_db_row("general_settings", stored_row)
fake_prisma: Final = _fake_prisma_with_config(stored_row)
monkeypatch.setattr(proxy_server_module, "proxy_config", pc)
monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma)
admin: Final = UserAPIKeyAuth(api_key="hashed-admin", user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN)
await delete_config_general_settings(
data=ConfigFieldDelete(field_name="enforce_user_param", config_type="general_settings"),
user_api_key_dict=admin,
)
written: Final = json.loads(fake_prisma.db.litellm_config.upsert.await_args.kwargs["data"]["update"]["param_value"])
assert written == {"store_prompts_in_spend_logs": True}
assert pc.settings["enforce_user_param"] is False
assert pc.settings.unmanaged_db_keys() == ()
@pytest.mark.asyncio
@pytest.mark.parametrize("stored_row", ['{"max_request_size_mb": 42, "disable_custom_api_keys": true}', "not json"])
@pytest.mark.parametrize("field_name", ["max_request_size_mb", "disable_custom_api_keys", "not_a_setting"])
async def test_delete_config_general_settings_leaves_a_string_row_untouched(
monkeypatch: pytest.MonkeyPatch, stored_row: str, field_name: str
) -> None:
from litellm.proxy._types import ConfigFieldDelete
from litellm.proxy.proxy_server import ProxyConfig, delete_config_general_settings
pc: Final = ProxyConfig()
pc._load_yaml_settings_stores({"general_settings": {}})
fake_prisma: Final = _fake_prisma_with_config(stored_row)
monkeypatch.setattr(proxy_server_module, "proxy_config", pc)
monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma)
admin: Final = UserAPIKeyAuth(api_key="hashed-admin", user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN)
with pytest.raises(HTTPException) as refused:
await delete_config_general_settings(
data=ConfigFieldDelete(field_name=field_name, config_type="general_settings"),
user_api_key_dict=admin,
)
assert (refused.value.status_code, refused.value.detail) == (
400,
{
"error": "general_settings is stored in the database as something other than a JSON object, so no field "
"can be removed from it"
},
)
fake_prisma.db.litellm_config.upsert.assert_not_awaited()
@pytest.mark.asyncio
async def test_delete_config_general_settings_refuses_a_key_neither_managed_nor_stored(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.proxy._types import ConfigFieldDelete
from litellm.proxy.proxy_server import ProxyConfig, delete_config_general_settings
pc: Final = ProxyConfig()
pc._load_yaml_settings_stores({"general_settings": {}})
fake_prisma: Final = _fake_prisma_with_config({"store_prompts_in_spend_logs": True})
monkeypatch.setattr(proxy_server_module, "proxy_config", pc)
monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma)
admin: Final = UserAPIKeyAuth(api_key="hashed-admin", user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN)
with pytest.raises(HTTPException) as refused:
await delete_config_general_settings(
data=ConfigFieldDelete(field_name="enforce_user_param", config_type="general_settings"),
user_api_key_dict=admin,
)
assert refused.value.status_code == 400
fake_prisma.db.litellm_config.upsert.assert_not_awaited()
@pytest.mark.asyncio
async def test_config_field_info_reports_the_declared_value_of_a_config_owned_secret(monkeypatch):
from litellm.proxy.proxy_server import ProxyConfig, get_config_general_settings

View file

@ -818,6 +818,165 @@ class TestPostCallFailureHookLiftsCallTypeAndStartTime:
assert payload["model"] == "MCP: deepwiki-ask_question"
assert payload["endTime"] - payload["startTime"] >= real_datetime.timedelta(seconds=2)
@pytest.mark.asyncio
async def test_failed_speech_spend_row_records_the_endpoint_call_type_over_the_body(
self, monkeypatch: pytest.MonkeyPatch
):
from types import SimpleNamespace
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
rejected: Final = litellm.BadRequestError(
message="Voice name nonexistent is not supported", model="gemini-tts", llm_provider="gemini"
)
request_data: Final = {
"model": "gemini-tts",
"input": "Hello there",
"voice": "nonexistent",
"call_type": "acompletion",
}
proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=DualCache())
proxy_logging_obj.alert_types = []
spend_writer: Final = SimpleNamespace(update_database=AsyncMock())
monkeypatch.setattr(litellm, "callbacks", [ProxyDBLogger(spend_writer=lambda: spend_writer)])
try:
with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()):
await proxy_logging_obj.post_call_failure_hook(
request_data=request_data,
original_exception=rejected,
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", request_route="/v1/audio/speech"),
dispatched_call_type="aspeech",
)
finally:
ProxyLogging._callback_capabilities_cache.clear()
db_call: Final = spend_writer.update_database.call_args.kwargs
payload: Final = get_logging_payload(
kwargs=db_call["kwargs"],
response_obj=db_call["completion_response"],
start_time=db_call["start_time"],
end_time=db_call["end_time"],
)
assert payload["call_type"] == "aspeech"
@pytest.mark.asyncio
async def test_failed_mcp_list_tools_row_inside_a_responses_request_keeps_list_mcp_tools(
self, monkeypatch: pytest.MonkeyPatch
):
from types import SimpleNamespace
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
request_data: Final = {
"model": "MCP: list_tools",
"call_type": "list_mcp_tools",
"input": [{"role": "system", "content": "list_tools"}],
}
proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=DualCache())
proxy_logging_obj.alert_types = []
spend_writer: Final = SimpleNamespace(update_database=AsyncMock())
monkeypatch.setattr(litellm, "callbacks", [ProxyDBLogger(spend_writer=lambda: spend_writer)])
try:
with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()):
await proxy_logging_obj.post_call_failure_hook(
request_data=request_data,
original_exception=RuntimeError("MCP server unreachable"),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", request_route="/v1/responses"),
route="/mcp/list_tools",
)
finally:
ProxyLogging._callback_capabilities_cache.clear()
db_call: Final = spend_writer.update_database.call_args.kwargs
payload: Final = get_logging_payload(
kwargs=db_call["kwargs"],
response_obj=db_call["completion_response"],
start_time=db_call["start_time"],
end_time=db_call["end_time"],
)
assert payload["call_type"] == "list_mcp_tools"
@pytest.mark.asyncio
async def test_failure_without_an_endpoint_call_type_does_not_guess_one_from_a_multi_method_path(
self, monkeypatch: pytest.MonkeyPatch
):
from types import SimpleNamespace
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=DualCache())
proxy_logging_obj.alert_types = []
spend_writer: Final = SimpleNamespace(update_database=AsyncMock())
monkeypatch.setattr(litellm, "callbacks", [ProxyDBLogger(spend_writer=lambda: spend_writer)])
try:
with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()):
await proxy_logging_obj.post_call_failure_hook(
request_data={},
original_exception=RuntimeError("vector store lookup failed"),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", request_route="/v1/vector_stores"),
)
finally:
ProxyLogging._callback_capabilities_cache.clear()
db_call: Final = spend_writer.update_database.call_args.kwargs
payload: Final = get_logging_payload(
kwargs=db_call["kwargs"],
response_obj=db_call["completion_response"],
start_time=db_call["start_time"],
end_time=db_call["end_time"],
)
assert payload["call_type"] == ""
@pytest.mark.asyncio
@pytest.mark.parametrize(
("request_data", "route", "endpoint_call_type"),
[
({"model": "gemini-tts", "input": "Hello there", "voice": "Kore"}, "/v1/audio/speech", "aspeech"),
({"model": "omni-moderation", "input": ["one", "two"]}, "/v1/moderations", "amoderation"),
({"model": "gpt-transcribe"}, "/v1/audio/transcriptions", "atranscription"),
],
)
async def test_request_rejected_by_the_proxy_records_the_endpoint_call_type(
self,
monkeypatch: pytest.MonkeyPatch,
request_data: dict[str, object],
route: str,
endpoint_call_type: str,
):
from types import SimpleNamespace
from fastapi import HTTPException
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=DualCache())
proxy_logging_obj.alert_types = []
spend_writer: Final = SimpleNamespace(update_database=AsyncMock())
monkeypatch.setattr(litellm, "callbacks", [ProxyDBLogger(spend_writer=lambda: spend_writer)])
try:
with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()):
await proxy_logging_obj.post_call_failure_hook(
request_data=dict(request_data),
original_exception=HTTPException(status_code=429, detail="Rate limit exceeded"),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", request_route=route),
dispatched_call_type=endpoint_call_type,
)
finally:
ProxyLogging._callback_capabilities_cache.clear()
db_call: Final = spend_writer.update_database.call_args.kwargs
payload: Final = get_logging_payload(
kwargs=db_call["kwargs"],
response_obj=db_call["completion_response"],
start_time=db_call["start_time"],
end_time=db_call["end_time"],
)
assert payload["call_type"] == endpoint_call_type
class TestPostCallFailureHookEstimatesDispatchedInputTokens:
"""A non-stream request that failed after dispatch (timeout, provider
@ -3576,6 +3735,44 @@ def test_an_earlier_timed_out_responses_attempt_counts_its_input_from_the_newest
assert sent == earlier_attempt_prompt_tokens([COMPACTION_INSTRUCTIONS, COMPACTION_ITEM, TURN_AFTER_COMPACTION])
@pytest.mark.asyncio
async def test_a_billed_responses_route_failure_captures_usage_from_the_newest_compaction_item() -> None:
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.litellm_logging import UsageCaptureObservation
from litellm.proxy.db.accounting_ledger import UsageWrite
from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
from litellm.proxy.utils import _failure_fields_to_lift
async def persist(_: tuple[UsageWrite, ...]) -> None:
return None
def captured_prompt_tokens(request_input: list[dict[str, str]]) -> int:
logging_obj: Final = _dispatched_failure_logging_obj(FAILURE_USAGE_MODEL, get_litellm_params())
logging_obj.model_call_details["call_type"] = "aresponses"
logging_obj.model_call_details["messages"] = request_input
lifted: Final = _failure_fields_to_lift(
{"litellm_logging_obj": logging_obj}, provider_billed=True, route_call_type="aresponses"
)
snapshot: Final = lifted["_accounting_usage_snapshot"]
assert isinstance(snapshot, UsageCaptureObservation) and snapshot.usage_json is not None
return json.loads(snapshot.usage_json)["prompt_tokens"]
runtime: Final = UsageCaptureRuntime()
runtime.start(persist, UsageCaptureSettings())
try:
sent: Final = captured_prompt_tokens(
[COMPACTION_INSTRUCTIONS, *COMPACTED_HISTORY, COMPACTION_ITEM, TURN_AFTER_COMPACTION]
)
since_compaction: Final = captured_prompt_tokens(
[COMPACTION_INSTRUCTIONS, COMPACTION_ITEM, TURN_AFTER_COMPACTION]
)
finally:
await runtime.stop(1)
assert sent == since_compaction
@pytest.mark.parametrize(
"current_attempt_api_call_start_time",
[None, 1700000001.0],

View file

@ -1083,6 +1083,128 @@ async def test_previous_response_history_starts_at_a_compaction_item_in_a_stored
]
def _stored_response_turn(
request_id: str, call_type: str, status: str, output: list[dict[str, object]], prompt: str | None = None
) -> dict[str, object]:
return {
"request_id": request_id,
"call_type": call_type,
"session_id": "session-with-reads",
"proxy_server_request": json.dumps({} if prompt is None else {"input": [{"role": "user", "content": prompt}]}),
"response": {"id": request_id, "object": "response", "status": status, "output": output},
}
def _identified_output_message(item_id: str, text: str) -> dict[str, object]:
return {**_output_message(text), "id": item_id}
def _replayed_roles_and_texts(session: ChatCompletionSession) -> list[tuple[object, object]]:
return [
(role, _text_of(content)) for role, content in (_role_and_content(message) for message in session["messages"])
]
@pytest.mark.asyncio
async def test_reads_and_cancels_of_a_response_the_session_already_holds_replay_no_turns():
answer: Final = [_identified_output_message("msg_answer", "OK")]
spend_logs: Final = (
_stored_response_turn("resp_create", "aresponses", "completed", answer, prompt="Reply with OK"),
_stored_response_turn("resp_read", "aget_responses", "completed", answer),
_stored_response_turn("resp_streamed_read", "aget_responses", "completed", answer),
_stored_response_turn("resp_cancel", "acancel_responses", "completed", answer),
)
session: Final = await _history_from_stored_turns(spend_logs, "resp_create")
assert _replayed_roles_and_texts(session) == [("user", "Reply with OK"), ("assistant", "OK")]
@pytest.mark.asyncio
async def test_a_polled_background_response_replays_its_finished_output_once():
answer: Final = [_identified_output_message("msg_answer", "One, two, three")]
spend_logs: Final = (
_stored_response_turn("resp_queued", "aresponses", "queued", [], prompt="Count to three"),
_stored_response_turn("resp_poll_running", "aget_responses", "in_progress", []),
_stored_response_turn("resp_poll_done", "aget_responses", "completed", answer),
_stored_response_turn("resp_poll_again", "aget_responses", "completed", answer),
)
session: Final = await _history_from_stored_turns(spend_logs, "resp_queued")
assert _replayed_roles_and_texts(session) == [("user", "Count to three"), ("assistant", "One, two, three")]
@pytest.mark.asyncio
async def test_a_background_response_that_grows_between_polls_replays_each_output_item_once():
first: Final = _identified_output_message("msg_first", "One")
second: Final = _identified_output_message("msg_second", "Two")
spend_logs: Final = (
_stored_response_turn("resp_queued", "aresponses", "queued", [], prompt="Count to two"),
_stored_response_turn("resp_poll_partial", "aget_responses", "in_progress", [first]),
_stored_response_turn("resp_poll_done", "aget_responses", "completed", [first, second]),
_stored_response_turn("resp_poll_again", "aget_responses", "completed", [first, second]),
)
session: Final = await _history_from_stored_turns(spend_logs, "resp_queued")
assert _replayed_roles_and_texts(session) == [("user", "Count to two"), ("assistant", "One"), ("assistant", "Two")]
@pytest.mark.asyncio
async def test_an_item_a_poll_caught_half_written_replays_its_finished_text_once():
spend_logs: Final = (
_stored_response_turn("resp_queued", "aresponses", "queued", [], prompt="Count to three"),
_stored_response_turn(
"resp_poll_partial", "aget_responses", "in_progress", [_identified_output_message("msg_answer", "One, t")]
),
_stored_response_turn(
"resp_poll_done",
"aget_responses",
"completed",
[_identified_output_message("msg_answer", "One, two, three"), _identified_output_message("msg_more", "Done")],
),
)
session: Final = await _history_from_stored_turns(spend_logs, "resp_queued")
assert _replayed_roles_and_texts(session) == [
("user", "Count to three"),
("assistant", "One, two, three"),
("assistant", "Done"),
]
@pytest.mark.asyncio
async def test_a_polled_output_item_without_an_id_still_replays_in_order():
spend_logs: Final = (
_stored_response_turn("resp_queued", "aresponses", "queued", [], prompt="Count to two"),
_stored_response_turn(
"resp_poll_done",
"aget_responses",
"completed",
[_output_message("One"), _identified_output_message("msg_two", "Two")],
),
)
session: Final = await _history_from_stored_turns(spend_logs, "resp_queued")
assert _replayed_roles_and_texts(session) == [("user", "Count to two"), ("assistant", "One"), ("assistant", "Two")]
@pytest.mark.asyncio
async def test_a_read_that_ends_before_its_streamed_create_replays_the_answer_once_after_the_prompt():
answer: Final = [_identified_output_message("msg_answer", "OK")]
spend_logs: Final = (
_stored_response_turn("resp_read_mid_stream", "aget_responses", "completed", answer),
_stored_response_turn("resp_create", "aresponses", "completed", answer, prompt="Say OK"),
)
session: Final = await _history_from_stored_turns(spend_logs, "resp_create")
assert _replayed_roles_and_texts(session) == [("user", "Say OK"), ("assistant", "OK")]
def _tool_call_ids(message: object) -> tuple[str, ...]:
tool_calls: Final = message.get("tool_calls") if isinstance(message, dict) else getattr(message, "tool_calls", None)
return tuple(call["id"] if isinstance(call, dict) else call.id for call in tool_calls or ())

View file

@ -4,23 +4,33 @@ Tests for ResponsesIDSecurity hook.
Tests the security hook that prevents user B from seeing response from user A.
"""
import datetime
from collections.abc import Callable, Mapping
from unittest.mock import MagicMock, patch
import pytest
from fastapi import HTTPException
from litellm.caching.dual_cache import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.hooks.responses_id_security import (
ResponsesIDSecurity,
_streams_response_ids,
)
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import (
GenericEvent,
ResponseAPIUsage,
ResponseCompletedEvent,
ResponseCreatedEvent,
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
)
from litellm.types.utils import SpecialEnums
from litellm.types.responses.main import DeleteResponseResult
from litellm.types.utils import CallTypesLiteral, Choices, Message, ModelResponse, SpecialEnums, Usage
@pytest.fixture
@ -55,12 +65,12 @@ class TestIsEncryptedResponseId:
import litellm.proxy.hooks.responses_id_security as responses_module
with patch.object(responses_module, "decrypt_value_helper") as mock_decrypt:
mock_decrypt.return_value = f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}response_id:resp_123;user_id:user-456"
result = responses_id_security._is_encrypted_response_id(
"resp_encrypted_value"
mock_decrypt.return_value = (
f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}response_id:resp_123;user_id:user-456"
)
result = responses_id_security._is_encrypted_response_id("resp_encrypted_value")
assert result is True
mock_decrypt.assert_called_once()
@ -88,9 +98,7 @@ class TestDecryptResponseId:
with patch.object(responses_module, "decrypt_value_helper") as mock_decrypt:
mock_decrypt.return_value = f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}response_id:resp_original_123;user_id:user-456;team_id:team-789"
original_id, user_id, team_id = responses_id_security._decrypt_response_id(
"resp_encrypted_value"
)
original_id, user_id, team_id = responses_id_security._decrypt_response_id("resp_encrypted_value")
assert original_id == "resp_original_123"
assert user_id == "user-456"
@ -104,9 +112,7 @@ class TestDecryptResponseId:
with patch.object(responses_module, "decrypt_value_helper") as mock_decrypt:
mock_decrypt.return_value = None
original_id, user_id, team_id = responses_id_security._decrypt_response_id(
"resp_plain_value"
)
original_id, user_id, team_id = responses_id_security._decrypt_response_id("resp_plain_value")
assert original_id == "resp_plain_value"
assert user_id is None
@ -116,9 +122,7 @@ class TestDecryptResponseId:
class TestCheckUserAccessToResponseId:
"""Test check_user_access_to_response_id function"""
def test_check_user_access_same_user(
self, responses_id_security, mock_user_api_key_dict
):
def test_check_user_access_same_user(self, responses_id_security, mock_user_api_key_dict):
"""Test that same user has access to their response ID"""
result = responses_id_security.check_user_access_to_response_id(
response_id_user_id="test-user-123",
@ -128,9 +132,7 @@ class TestCheckUserAccessToResponseId:
assert result is True
def test_check_user_access_different_user_raises_exception(
self, responses_id_security, mock_user_api_key_dict
):
def test_check_user_access_different_user_raises_exception(self, responses_id_security, mock_user_api_key_dict):
"""Test that different user is denied access to response ID"""
with patch("litellm.proxy.proxy_server.general_settings", {}):
with pytest.raises(HTTPException) as exc_info:
@ -143,9 +145,7 @@ class TestCheckUserAccessToResponseId:
assert exc_info.value.status_code == 403
assert "Forbidden" in exc_info.value.detail
def test_check_user_access_different_team_raises_exception(
self, responses_id_security, mock_user_api_key_dict
):
def test_check_user_access_different_team_raises_exception(self, responses_id_security, mock_user_api_key_dict):
"""Test that different team is denied access to response ID"""
with patch("litellm.proxy.proxy_server.general_settings", {}):
with pytest.raises(HTTPException) as exc_info:
@ -158,9 +158,7 @@ class TestCheckUserAccessToResponseId:
assert exc_info.value.status_code == 403
assert "Forbidden" in exc_info.value.detail
def test_check_user_access_team_a_to_team_b_without_user_id(
self, responses_id_security
):
def test_check_user_access_team_a_to_team_b_without_user_id(self, responses_id_security):
"""Test that key from team A (without user_id) cannot access response from team B (without user_id)"""
# Create a mock user from team A without user_id
mock_auth_team_a = MagicMock()
@ -179,9 +177,7 @@ class TestCheckUserAccessToResponseId:
assert exc_info.value.status_code == 403
assert "team" in exc_info.value.detail.lower()
def test_check_user_access_team_a_to_team_b_with_user_id(
self, responses_id_security
):
def test_check_user_access_team_a_to_team_b_with_user_id(self, responses_id_security):
"""Test that key from team A (without user_id) cannot access response from team B (with user_id)"""
# Create a mock user from team A without user_id
mock_auth_team_a = MagicMock()
@ -217,9 +213,7 @@ class TestCheckUserAccessToResponseId:
assert result is True
def test_check_user_access_admin_can_access_any_response(
self, responses_id_security
):
def test_check_user_access_admin_can_access_any_response(self, responses_id_security):
"""Test that proxy admin can access any response ID"""
from litellm.proxy._types import LitellmUserRoles
@ -238,9 +232,7 @@ class TestCheckUserAccessToResponseId:
assert result is True
def test_check_user_access_security_disabled(
self, responses_id_security, mock_user_api_key_dict
):
def test_check_user_access_security_disabled(self, responses_id_security, mock_user_api_key_dict):
"""Test that when security is disabled, any user can access any response"""
with patch(
"litellm.proxy.proxy_server.general_settings",
@ -266,9 +258,7 @@ class TestAsyncPreCallHook:
"""Test pre-call hook decrypts previous_response_id for aresponses call"""
data = {"previous_response_id": "resp_encrypted_value"}
with patch.object(
responses_id_security, "_is_encrypted_response_id", return_value=True
):
with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True):
with patch.object(
responses_id_security,
"_decrypt_response_id",
@ -284,15 +274,11 @@ class TestAsyncPreCallHook:
assert result["previous_response_id"] == "resp_original_123"
@pytest.mark.asyncio
async def test_async_pre_call_hook_aget_responses(
self, responses_id_security, mock_user_api_key_dict, mock_cache
):
async def test_async_pre_call_hook_aget_responses(self, responses_id_security, mock_user_api_key_dict, mock_cache):
"""Test pre-call hook decrypts response_id for aget_responses call"""
data = {"response_id": "resp_encrypted_456"}
with patch.object(
responses_id_security, "_is_encrypted_response_id", return_value=True
):
with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True):
with patch.object(
responses_id_security,
"_decrypt_response_id",
@ -308,9 +294,7 @@ class TestAsyncPreCallHook:
assert result["response_id"] == "resp_original_456"
@pytest.mark.asyncio
async def test_async_pre_call_hook_team_a_accessing_team_b_response(
self, responses_id_security, mock_cache
):
async def test_async_pre_call_hook_team_a_accessing_team_b_response(self, responses_id_security, mock_cache):
"""Test pre-call hook prevents team A from accessing team B response"""
# Create a mock user from team A
mock_auth_team_a = MagicMock()
@ -320,9 +304,7 @@ class TestAsyncPreCallHook:
data = {"response_id": "resp_encrypted_team_b"}
with patch.object(
responses_id_security, "_is_encrypted_response_id", return_value=True
):
with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True):
with patch.object(
responses_id_security,
"_decrypt_response_id",
@ -341,9 +323,7 @@ class TestAsyncPreCallHook:
assert "team" in exc_info.value.detail.lower()
@pytest.mark.asyncio
async def test_async_pre_call_hook_team_a_accessing_team_b_with_user(
self, responses_id_security, mock_cache
):
async def test_async_pre_call_hook_team_a_accessing_team_b_with_user(self, responses_id_security, mock_cache):
"""Test pre-call hook prevents team A (no user) from accessing team B response (with user)"""
# Create a mock user from team A without user_id
mock_auth_team_a = MagicMock()
@ -353,9 +333,7 @@ class TestAsyncPreCallHook:
data = {"response_id": "resp_encrypted_team_b_with_user"}
with patch.object(
responses_id_security, "_is_encrypted_response_id", return_value=True
):
with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True):
with patch.object(
responses_id_security,
"_decrypt_response_id",
@ -375,9 +353,7 @@ class TestAsyncPreCallHook:
assert "forbidden" in exc_info.value.detail.lower()
@pytest.mark.asyncio
async def test_async_pre_call_hook_same_team_access(
self, responses_id_security, mock_cache
):
async def test_async_pre_call_hook_same_team_access(self, responses_id_security, mock_cache):
"""Test pre-call hook allows team A to access their own team's response"""
# Create a mock user from team A
mock_auth_team_a = MagicMock()
@ -387,9 +363,7 @@ class TestAsyncPreCallHook:
data = {"response_id": "resp_encrypted_team_a"}
with patch.object(
responses_id_security, "_is_encrypted_response_id", return_value=True
):
with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True):
with patch.object(
responses_id_security,
"_decrypt_response_id",
@ -405,9 +379,7 @@ class TestAsyncPreCallHook:
assert result["response_id"] == "resp_original_team_a"
@pytest.mark.asyncio
async def test_async_pre_call_hook_adelete_responses_team_security(
self, responses_id_security, mock_cache
):
async def test_async_pre_call_hook_adelete_responses_team_security(self, responses_id_security, mock_cache):
"""Test pre-call hook prevents team A from deleting team B's response"""
# Create a mock user from team A
mock_auth_team_a = MagicMock()
@ -417,9 +389,7 @@ class TestAsyncPreCallHook:
data = {"response_id": "resp_encrypted_team_b"}
with patch.object(
responses_id_security, "_is_encrypted_response_id", return_value=True
):
with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True):
with patch.object(
responses_id_security,
"_decrypt_response_id",
@ -438,9 +408,7 @@ class TestAsyncPreCallHook:
assert "team" in exc_info.value.detail.lower()
@pytest.mark.asyncio
async def test_async_pre_call_hook_acancel_responses_team_security(
self, responses_id_security, mock_cache
):
async def test_async_pre_call_hook_acancel_responses_team_security(self, responses_id_security, mock_cache):
"""Test pre-call hook prevents team A from canceling team B's response"""
# Create a mock user from team A
mock_auth_team_a = MagicMock()
@ -450,9 +418,7 @@ class TestAsyncPreCallHook:
data = {"response_id": "resp_encrypted_team_b"}
with patch.object(
responses_id_security, "_is_encrypted_response_id", return_value=True
):
with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True):
with patch.object(
responses_id_security,
"_decrypt_response_id",
@ -470,16 +436,13 @@ class TestAsyncPreCallHook:
assert exc_info.value.status_code == 403
assert "team" in exc_info.value.detail.lower()
@pytest.mark.asyncio
async def test_async_pre_call_hook_alist_input_items_decrypts_response_id(
self, responses_id_security, mock_user_api_key_dict, mock_cache
):
data = {"response_id": "resp_encrypted_789"}
with patch.object(
responses_id_security, "_is_encrypted_response_id", return_value=True
):
with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True):
with patch.object(
responses_id_security,
"_decrypt_response_id",
@ -496,9 +459,7 @@ class TestAsyncPreCallHook:
assert result["response_id"] == "resp_original_789"
@pytest.mark.asyncio
async def test_async_pre_call_hook_alist_input_items_team_security(
self, responses_id_security, mock_cache
):
async def test_async_pre_call_hook_alist_input_items_team_security(self, responses_id_security, mock_cache):
mock_auth_team_a = MagicMock()
mock_auth_team_a.user_id = None
mock_auth_team_a.team_id = "team-a"
@ -506,9 +467,7 @@ class TestAsyncPreCallHook:
data = {"response_id": "resp_encrypted_team_b"}
with patch.object(
responses_id_security, "_is_encrypted_response_id", return_value=True
):
with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True):
with patch.object(
responses_id_security,
"_decrypt_response_id",
@ -620,9 +579,7 @@ class TestAsyncPostCallStreamingIteratorHook:
"/openai/v1/responses/resp_rawprovider123",
],
)
async def test_streamed_id_encrypted_on_all_responses_routes(
self, responses_id_security, route, monkeypatch
):
async def test_streamed_id_encrypted_on_all_responses_routes(self, responses_id_security, route, monkeypatch):
streamed_id = await self._drain_streamed_id(responses_id_security, route, monkeypatch)
assert streamed_id != "resp_rawprovider123"
@ -634,12 +591,8 @@ class TestAsyncPostCallStreamingIteratorHook:
)
@pytest.mark.asyncio
async def test_streamed_id_untouched_on_non_responses_route(
self, responses_id_security, monkeypatch
):
streamed_id = await self._drain_streamed_id(
responses_id_security, "/chat/completions", monkeypatch
)
async def test_streamed_id_untouched_on_non_responses_route(self, responses_id_security, monkeypatch):
streamed_id = await self._drain_streamed_id(responses_id_security, "/chat/completions", monkeypatch)
assert streamed_id == "resp_rawprovider123"
assert not responses_id_security._is_encrypted_response_id(streamed_id)
@ -698,12 +651,8 @@ class TestStreamedGenericEventIdEncryption:
def _advertised_ids(events):
nested = (getattr(event, "response", None) for event in events)
return [
payload["id"] if isinstance(payload, dict) else payload.id
for payload in nested
if payload is not None
] + [
event.id for event in events if isinstance(getattr(event, "id", None), str)
]
payload["id"] if isinstance(payload, dict) else payload.id for payload in nested if payload is not None
] + [event.id for event in events if isinstance(getattr(event, "id", None), str)]
async def _drain(self, responses_id_security, monkeypatch):
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-test-salt-key-abcdefghij")
@ -723,9 +672,7 @@ class TestStreamedGenericEventIdEncryption:
]
@pytest.mark.asyncio
async def test_every_event_advertises_the_same_encrypted_id(
self, responses_id_security, monkeypatch
):
async def test_every_event_advertises_the_same_encrypted_id(self, responses_id_security, monkeypatch):
events = await self._drain(responses_id_security, monkeypatch)
advertised = self._advertised_ids(events)
@ -742,22 +689,14 @@ class TestStreamedGenericEventIdEncryption:
)
@pytest.mark.asyncio
async def test_raw_provider_id_never_reaches_the_client(
self, responses_id_security, monkeypatch
):
async def test_raw_provider_id_never_reaches_the_client(self, responses_id_security, monkeypatch):
events = await self._drain(responses_id_security, monkeypatch)
assert [self.RAW_ID in event.model_dump_json() for event in events] == [
False
] * len(events)
assert [self.RAW_ID in event.model_dump_json() for event in events] == [False] * len(events)
@pytest.mark.asyncio
async def test_sibling_fields_survive_the_rewrite(
self, responses_id_security, monkeypatch
):
_, queued, keepalive, later, _ = await self._drain(
responses_id_security, monkeypatch
)
async def test_sibling_fields_survive_the_rewrite(self, responses_id_security, monkeypatch):
_, queued, keepalive, later, _ = await self._drain(responses_id_security, monkeypatch)
assert queued.response["status"] == "queued"
assert later.response["status"] == "in_progress"
@ -769,27 +708,19 @@ class TestAsyncPostCallSuccessHook:
"""Test async_post_call_success_hook function"""
@pytest.mark.asyncio
async def test_async_post_call_success_hook_encrypts_response(
self, responses_id_security, mock_user_api_key_dict
):
async def test_async_post_call_success_hook_encrypts_response(self, responses_id_security, mock_user_api_key_dict):
"""Test post-call hook encrypts ResponsesAPIResponse"""
mock_response = ResponsesAPIResponse(
id="resp_789", created_at=1234567890, output=[], status="completed"
)
mock_response = ResponsesAPIResponse(id="resp_789", created_at=1234567890, output=[], status="completed")
data = {}
with patch.object(
responses_id_security, "_encrypt_response_id", return_value=mock_response
) as mock_encrypt:
with patch.object(responses_id_security, "_encrypt_response_id", return_value=mock_response) as mock_encrypt:
result = await responses_id_security.async_post_call_success_hook(
data=data,
user_api_key_dict=mock_user_api_key_dict,
response=mock_response,
)
mock_encrypt.assert_called_once_with(
mock_response, mock_user_api_key_dict, request_cache=None
)
mock_encrypt.assert_called_once_with(mock_response, mock_user_api_key_dict, request_cache=None)
assert result == mock_response
@pytest.mark.asyncio
@ -846,9 +777,7 @@ def _auth(user_id="owner-user", team_id="owner-team", user_role=None):
def _issue_managed_id(hook, owner, provider_response_id=_FABRICATED_PROVIDER_RESPONSE_ID):
"""Mint an id exactly the way the proxy hands one to a client on create."""
issued = hook._encrypt_response_id(
ResponsesAPIResponse(
id=provider_response_id, created_at=1234567890, output=[], status="completed"
),
ResponsesAPIResponse(id=provider_response_id, created_at=1234567890, output=[], status="completed"),
owner,
)
return issued.id
@ -1080,9 +1009,7 @@ class TestClientSuppliedRetainedIdCannotBypassAuthorization:
assert exc_info.value.status_code == 403
@pytest.mark.asyncio
async def test_stranger_forging_their_own_id_never_reaches_someone_elses_response(
self, mock_cache, salt_key_env
):
async def test_stranger_forging_their_own_id_never_reaches_someone_elses_response(self, mock_cache, salt_key_env):
hook = _hook()
stranger = _auth(user_id="stranger-user", team_id="stranger-team")
stranger_id = _issue_managed_id(hook, stranger, provider_response_id="resp_strangerownprovideridcccccccc")
@ -1098,3 +1025,436 @@ class TestClientSuppliedRetainedIdCannotBypassAuthorization:
assert result["response_id"] == "resp_strangerownprovideridcccccccc"
assert result["response_id"] != victim_provider_id
def _admin():
from litellm.proxy._types import LitellmUserRoles
return _auth(user_id="admin-user", team_id="admin-team", user_role=LitellmUserRoles.PROXY_ADMIN)
def _provider_answer(response_id: str) -> ResponsesAPIResponse:
return ResponsesAPIResponse(id=response_id, created_at=1234567890, output=[], status="completed")
class TestReadsAnswerWithTheIdTheClientHolds:
"""A follow-up looks its history up by the id the client holds, so a read must not hand out a new one."""
@pytest.mark.asyncio
@pytest.mark.parametrize("call_type", ["aget_responses", "acancel_responses"])
@pytest.mark.parametrize(
"issuer, reader",
[
(_auth(), _auth()),
(_auth(team_id=None), _auth(team_id="another-team")),
(_auth(user_id=None), _auth(user_id=None)),
(_auth(user_id=None), _auth(user_id="teammate-user")),
(_auth(), _admin()),
(_auth(user_id=None), _auth(user_id=None, team_id="admin-team", user_role=_admin().user_role)),
],
ids=[
"same-key",
"same-user-new-team",
"same-team-no-user",
"teammate-reading-a-team-key-id",
"admin-user",
"admin-without-user",
],
)
async def test_a_caller_with_access_gets_back_the_exact_id_they_sent(
self, mock_cache, salt_key_env, call_type, issuer, reader
):
hook = _hook()
issued_id = _issue_managed_id(hook, issuer)
data = await hook.async_pre_call_hook(
user_api_key_dict=reader, cache=mock_cache, data={"response_id": issued_id}, call_type=call_type
)
result = await hook.async_post_call_success_hook(
data=data, user_api_key_dict=reader, response=_provider_answer(data["response_id"])
)
assert result.id == issued_id
@pytest.mark.asyncio
async def test_a_follow_up_gets_an_id_for_its_own_response_never_the_previous_one(self, mock_cache, salt_key_env):
hook = _hook()
owner = _auth()
previous_id = _issue_managed_id(hook, owner)
data = await hook.async_pre_call_hook(
user_api_key_dict=owner,
cache=mock_cache,
data={"model": "gpt-fake", "previous_response_id": previous_id},
call_type="aresponses",
)
result = await hook.async_post_call_success_hook(
data=data, user_api_key_dict=owner, response=_provider_answer("resp_followupprovideridzzzzzzzzzzzzzzz")
)
assert result.id != previous_id
assert hook._decrypt_response_id(result.id) == (
"resp_followupprovideridzzzzzzzzzzzzzzz",
"owner-user",
"owner-team",
)
@pytest.mark.asyncio
async def test_an_admin_read_by_a_raw_provider_id_gets_a_managed_id_back(self, mock_cache, salt_key_env):
hook = _hook()
admin = _admin()
data = await hook.async_pre_call_hook(
user_api_key_dict=admin,
cache=mock_cache,
data={"response_id": _FABRICATED_UNMANAGED_ID},
call_type="aget_responses",
)
result = await hook.async_post_call_success_hook(
data=data, user_api_key_dict=admin, response=_provider_answer(_FABRICATED_UNMANAGED_ID)
)
assert hook._is_encrypted_response_id(result.id)
assert hook._decrypt_response_id(result.id) == (_FABRICATED_UNMANAGED_ID, "admin-user", "admin-team")
_LOGGED_MANAGED_ID = "resp_managedidlitellmbuiltbeforethehookran"
def _streamed_response(response_id):
return ResponsesAPIResponse(
id=response_id,
created_at=0,
model="gpt-5.1",
object="response",
output=[],
parallel_tool_calls=False,
tool_choice="auto",
tools=[],
usage=ResponseAPIUsage(input_tokens=3, output_tokens=2, total_tokens=5),
)
def _upstream_frames():
return (
ResponseCreatedEvent(
type=ResponsesAPIStreamEvents.RESPONSE_CREATED, response=_streamed_response(_LOGGED_MANAGED_ID)
),
ResponseCompletedEvent(
type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=_streamed_response(_LOGGED_MANAGED_ID)
),
)
def _native_logged_result():
return ResponseCompletedEvent(
type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=_streamed_response(_LOGGED_MANAGED_ID)
)
def _bridged_logged_result():
return ModelResponse(
id="chatcmpl-bridgedchatcompletionid",
model="gpt-5.1",
choices=[Choices(message=Message(role="assistant", content="hi"), finish_reason="stop", index=0)],
usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5),
)
class _SpendRowRecorder(CustomLogger):
def __init__(self):
super().__init__()
self.spend_row_ids = []
self.standard_logging_ids = []
self.callback_response_ids = []
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
self.spend_row_ids.append(get_logging_payload(kwargs, response_obj, start_time, end_time)["request_id"])
self.callback_response_ids.append(response_obj.id)
if "standard_logging_object" in kwargs:
self.standard_logging_ids.append(kwargs["standard_logging_object"]["id"])
async def _stream_logged_mid_stream(
hook: ResponsesIDSecurity,
route: str,
logged_result: ResponseCompletedEvent | ModelResponse,
caller: UserAPIKeyAuth | None = None,
request_data: Mapping[str, object] | None = None,
) -> tuple[str, _SpendRowRecorder]:
streaming_caller = caller if caller is not None else _auth()
recorder = _SpendRowRecorder()
logging_obj = LiteLLMLoggingObj(
model="gpt-5.1",
messages=[],
stream=True,
call_type="aresponses",
start_time=datetime.datetime.now(),
litellm_call_id="streamed-responses-call",
function_id="streamed-responses-fn",
dynamic_async_success_callbacks=[recorder],
)
logging_obj.update_environment_variables(
model="gpt-5.1", user="", optional_params={}, litellm_params={"api_base": ""}
)
upstream = _upstream_frames()
async def frames():
for frame in upstream:
yield frame
client_stream = hook.async_post_call_streaming_iterator_hook(
user_api_key_dict=streaming_caller.model_copy(update={"request_route": route}),
response=frames(),
request_data={**(request_data or {}), "litellm_logging_obj": logging_obj},
)
first_frame = await anext(client_stream)
now = datetime.datetime.now()
await logging_obj.async_success_handler(result=logged_result, start_time=now, end_time=now, cache_hit=False)
client_frames = [first_frame, *[frame async for frame in client_stream]]
return client_frames[-1].response.id, recorder
class TestStreamedSpendRowCarriesTheIdTheClientSaw:
@pytest.mark.asyncio
@pytest.mark.parametrize("logged_result", [_native_logged_result, _bridged_logged_result], ids=["native", "bridge"])
@pytest.mark.parametrize("route", ["/v1/responses", "/responses", "/openai/v1/responses"])
async def test_spend_row_and_callbacks_log_the_encrypted_id_of_the_terminal_frame(
self, salt_key_env, route, logged_result
):
hook = _hook()
client_id, recorder = await _stream_logged_mid_stream(hook, route, logged_result())
assert hook._decrypt_response_id(client_id) == (_LOGGED_MANAGED_ID, "owner-user", "owner-team")
assert recorder.spend_row_ids == [client_id]
assert recorder.standard_logging_ids == [client_id]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"hook, route",
[
(_hook(), "/chat/completions"),
(_hook(general_settings={"disable_responses_id_security": True}), "/v1/responses"),
(_hook(signing_key=None), "/v1/responses"),
],
ids=["not-a-create-route", "security-disabled", "no-signing-key"],
)
async def test_stream_the_hook_leaves_alone_keeps_its_logged_id(self, salt_key_env, hook, route):
client_id, recorder = await _stream_logged_mid_stream(hook, route, _native_logged_result())
assert client_id == _LOGGED_MANAGED_ID
assert recorder.spend_row_ids == [_LOGGED_MANAGED_ID]
assert recorder.standard_logging_ids == [_LOGGED_MANAGED_ID]
class TestStreamedReadAnswersWithTheIdTheClientHolds:
@pytest.mark.asyncio
@pytest.mark.parametrize("route_prefix", ["/v1/responses", "/responses", "/openai/v1/responses"])
async def test_the_owner_sees_their_id_while_callbacks_log_one_of_the_reads_own(
self, mock_cache, salt_key_env, route_prefix
):
hook = _hook()
owner = _auth()
issued_id = _issue_managed_id(hook, owner, provider_response_id=_LOGGED_MANAGED_ID)
data = await hook.async_pre_call_hook(
user_api_key_dict=owner, cache=mock_cache, data={"response_id": issued_id}, call_type="aget_responses"
)
client_id, recorder = await _stream_logged_mid_stream(
hook, f"{route_prefix}/{issued_id}", _native_logged_result(), caller=owner, request_data=data
)
assert client_id == issued_id
assert recorder.spend_row_ids == recorder.standard_logging_ids
assert len(recorder.standard_logging_ids) == 1
assert recorder.standard_logging_ids[0] != issued_id
assert hook._decrypt_response_id(recorder.standard_logging_ids[0]) == (
_LOGGED_MANAGED_ID,
"owner-user",
"owner-team",
)
@pytest.mark.asyncio
async def test_an_admin_streamed_read_of_someone_elses_response_sees_the_id_sent_and_logs_one_of_its_own(
self, mock_cache, salt_key_env
):
hook = _hook()
admin = _admin()
issued_id = _issue_managed_id(hook, _auth(), provider_response_id=_LOGGED_MANAGED_ID)
data = await hook.async_pre_call_hook(
user_api_key_dict=admin, cache=mock_cache, data={"response_id": issued_id}, call_type="aget_responses"
)
client_id, recorder = await _stream_logged_mid_stream(
hook, f"/v1/responses/{issued_id}", _native_logged_result(), caller=admin, request_data=data
)
assert client_id == issued_id
assert recorder.spend_row_ids == recorder.standard_logging_ids
assert len(recorder.standard_logging_ids) == 1
assert hook._decrypt_response_id(recorder.standard_logging_ids[0]) == (
_LOGGED_MANAGED_ID,
"admin-user",
"admin-team",
)
@pytest.mark.asyncio
async def test_a_frame_naming_the_held_response_under_another_wrapping_still_gets_the_id_sent(
self, mock_cache, salt_key_env
):
hook = _hook()
owner = _auth()
issued_id = _issue_managed_id(hook, owner, provider_response_id=_routed_response_id(_LOGGED_MANAGED_ID))
data = await hook.async_pre_call_hook(
user_api_key_dict=owner, cache=mock_cache, data={"response_id": issued_id}, call_type="aget_responses"
)
client_id, _ = await _stream_logged_mid_stream(
hook, f"/v1/responses/{issued_id}", _native_logged_result(), caller=owner, request_data=data
)
assert client_id == issued_id
async def _read_logged_by_callbacks(
hook: ResponsesIDSecurity,
caller: UserAPIKeyAuth,
issued_id: str,
call_type: CallTypesLiteral,
cache: DualCache,
answer: Callable[[str], ResponsesAPIResponse | DeleteResponseResult] = _provider_answer,
) -> tuple[str, _SpendRowRecorder]:
recorder = _SpendRowRecorder()
now = datetime.datetime.now()
logging_obj = LiteLLMLoggingObj(
model="gpt-5.1",
messages=[],
stream=False,
call_type=call_type,
start_time=now,
litellm_call_id=f"{call_type}-call",
function_id=f"{call_type}-fn",
dynamic_async_success_callbacks=[recorder],
)
logging_obj.update_environment_variables(
model="gpt-5.1", user="", optional_params={}, litellm_params={"api_base": ""}
)
data = await hook.async_pre_call_hook(
user_api_key_dict=caller, cache=cache, data={"response_id": issued_id}, call_type=call_type
)
provider_answer = answer(data["response_id"])
client_response = await hook.async_post_call_success_hook(
data=data, user_api_key_dict=caller, response=provider_answer
)
await logging_obj.async_success_handler(result=provider_answer, start_time=now, end_time=now, cache_hit=False)
return client_response.id, recorder
class TestEveryReadLogsAnIdOfItsOwn:
@pytest.mark.asyncio
@pytest.mark.parametrize("call_type", ["aget_responses", "acancel_responses"])
@pytest.mark.parametrize("reader", [_auth(), _admin()], ids=["owner", "admin"])
async def test_callbacks_and_the_spend_row_log_a_new_id_per_read_never_the_one_the_client_holds(
self, mock_cache, salt_key_env, call_type, reader
):
hook = _hook()
issued_id = _issue_managed_id(hook, _auth())
reads = [await _read_logged_by_callbacks(hook, reader, issued_id, call_type, mock_cache) for _ in range(2)]
assert [client_id for client_id, _ in reads] == [issued_id, issued_id]
logged_ids = [id_ for _, recorder in reads for id_ in recorder.standard_logging_ids]
assert [recorder.spend_row_ids for _, recorder in reads] == [[logged_id] for logged_id in logged_ids]
assert len(set(logged_ids)) == 2
assert issued_id not in logged_ids
assert {hook._decrypt_response_id(logged_id) for logged_id in logged_ids} == {
(_FABRICATED_PROVIDER_RESPONSE_ID, reader.user_id, reader.team_id)
}
_RAW_UPSTREAM_RESPONSE_ID = "resp_06de59f3upstreamiddddddddddddddddd"
def _routed_response_id(upstream_response_id: str) -> str:
return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id(
_provider_answer(upstream_response_id), "openai", {"model_info": {"id": "deployment-1"}}
).id
def _deletion_of(response_id: str) -> DeleteResponseResult:
return DeleteResponseResult(
id=ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(response_id),
object="response.deleted",
deleted=True,
)
class TestDeleteAnswersWithTheIdTheClientHolds:
@pytest.mark.asyncio
@pytest.mark.parametrize("reader", [_auth(), _admin()], ids=["owner", "admin"])
@pytest.mark.parametrize(
"stored_id",
[_routed_response_id(_RAW_UPSTREAM_RESPONSE_ID), _RAW_UPSTREAM_RESPONSE_ID],
ids=["routed-deployment", "plain-id"],
)
async def test_a_caller_with_access_gets_back_the_exact_id_they_sent(
self, mock_cache, salt_key_env, reader, stored_id
):
hook = _hook()
issued_id = _issue_managed_id(hook, _auth(), provider_response_id=stored_id)
data = await hook.async_pre_call_hook(
user_api_key_dict=reader, cache=mock_cache, data={"response_id": issued_id}, call_type="adelete_responses"
)
result = await hook.async_post_call_success_hook(
data=data, user_api_key_dict=reader, response=_deletion_of(data["response_id"])
)
assert (result.id, result.object, result.deleted) == (issued_id, "response.deleted", True)
@pytest.mark.asyncio
@pytest.mark.parametrize("reader", [_auth(), _admin()], ids=["owner", "admin"])
async def test_callbacks_and_the_spend_row_log_an_id_of_its_own_never_the_one_the_client_holds(
self, mock_cache, salt_key_env, reader
):
hook = _hook()
issued_id = _issue_managed_id(
hook, _auth(), provider_response_id=_routed_response_id(_RAW_UPSTREAM_RESPONSE_ID)
)
client_id, recorder = await _read_logged_by_callbacks(
hook, reader, issued_id, "adelete_responses", mock_cache, answer=_deletion_of
)
assert client_id == issued_id
assert recorder.spend_row_ids == recorder.callback_response_ids
assert len(recorder.callback_response_ids) == 1
assert recorder.callback_response_ids[0] != issued_id
assert hook._decrypt_response_id(recorder.callback_response_ids[0]) == (
_RAW_UPSTREAM_RESPONSE_ID,
reader.user_id,
reader.team_id,
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"sent_id",
[_routed_response_id(_RAW_UPSTREAM_RESPONSE_ID), _RAW_UPSTREAM_RESPONSE_ID],
ids=["routed-deployment", "plain-id"],
)
async def test_an_admin_delete_by_a_raw_provider_id_gets_the_provider_id_back_and_logs_an_id_of_its_own(
self, mock_cache, salt_key_env, sent_id
):
hook = _hook()
client_id, recorder = await _read_logged_by_callbacks(
hook, _admin(), sent_id, "adelete_responses", mock_cache, answer=_deletion_of
)
assert client_id == _RAW_UPSTREAM_RESPONSE_ID
assert [hook._decrypt_response_id(logged_id) for logged_id in recorder.callback_response_ids] == [
(_RAW_UPSTREAM_RESPONSE_ID, "admin-user", "admin-team")
]

View file

@ -215,9 +215,6 @@
},
"max-lines": {
"count": 1
},
"react/no-unescaped-entities": {
"count": 2
}
},
"src/app/(dashboard)/guardrails/_components/content_filter/CompetitorIntentConfiguration.tsx": {
@ -247,6 +244,9 @@
}
},
"src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeModal.tsx": {
"max-lines": {
"count": 1
},
"no-nested-ternary": {
"count": 6
},
@ -281,6 +281,9 @@
"local/no-complex-jsx-arrow": {
"count": 1
},
"max-lines": {
"count": 1
},
"max-params": {
"count": 1
},

View file

@ -1,4 +1,5 @@
import { useAccessGroupDetails } from "@/app/(dashboard)/hooks/accessGroups/useAccessGroupDetails";
import type { TFunction } from "i18next";
import {
ArrowLeftIcon,
BotIcon,
@ -10,6 +11,7 @@ import {
UsersIcon,
} from "lucide-react";
import { useState } from "react";
import { useTranslation } from "react-i18next";
import DefaultProxyAdminTag from "@/components/common_components/DefaultProxyAdminTag";
import { BadgeLink } from "@/components/shared/BadgeLink";
import CopyButton from "@/components/shared/CopyButton";
@ -34,11 +36,11 @@ const MAX_PREVIEW = 5;
const shortId = (id: string) => (id.length > 20 ? `${id.slice(0, 10)}...${id.slice(-6)}` : id);
const describeAllowedTools = (tools: readonly string[] | undefined): string => {
const describeAllowedTools = (tools: readonly string[] | undefined, t: TFunction): string => {
if (tools === undefined) {
return "All tools";
return t("accessGroups.detail.toolPermissions.allTools");
}
return tools.length === 0 ? "No tools" : tools.join(", ");
return tools.length === 0 ? t("accessGroups.detail.toolPermissions.noTools") : tools.join(", ");
};
function MCPToolPermissionList({
@ -48,16 +50,17 @@ function MCPToolPermissionList({
servers: readonly AccessGroupResource[];
toolPermissions: Readonly<Record<string, string[]>> | null;
}) {
const { t } = useTranslation();
if (toolPermissions === null || servers.length === 0) {
return null;
}
return (
<section className="mt-6 space-y-2" aria-label="MCP tool permissions">
<section className="mt-6 space-y-2" aria-label={t("accessGroups.detail.toolPermissions.label")}>
{servers.map(({ id, name }) => {
return (
<div key={id} className="flex flex-wrap items-baseline gap-2 text-sm">
<span className="font-medium">{name ?? id}</span>
<span className="text-muted-foreground">{describeAllowedTools(toolPermissions[id])}</span>
<span className="text-muted-foreground">{describeAllowedTools(toolPermissions[id], t)}</span>
</div>
);
})}
@ -106,6 +109,7 @@ function ResourceBadge({
}
export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailProps) {
const { t } = useTranslation();
const { data: accessGroup, isLoading } = useAccessGroupDetails(accessGroupId);
const [isEditModalVisible, setIsEditModalVisible] = useState(false);
const [showAllKeys, setShowAllKeys] = useState(false);
@ -124,10 +128,16 @@ export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailPr
if (!accessGroup) {
return (
<div className="p-6 px-12">
<Button variant="ghost" size="icon" aria-label="Back" onClick={onBack} className="mb-4">
<Button
variant="ghost"
size="icon"
aria-label={t("accessGroups.detail.back")}
onClick={onBack}
className="mb-4"
>
<ArrowLeftIcon className="size-4" />
</Button>
<p className="py-8 text-center text-sm text-muted-foreground">Access group not found</p>
<p className="py-8 text-center text-sm text-muted-foreground">{t("accessGroups.detail.notFound")}</p>
</div>
);
}
@ -146,47 +156,50 @@ export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailPr
<div className="p-6 px-12">
<div className="mb-6 flex items-center justify-between">
<div className="flex items-center gap-4">
<Button variant="ghost" size="icon" aria-label="Back" onClick={onBack}>
<Button variant="ghost" size="icon" aria-label={t("accessGroups.detail.back")} onClick={onBack}>
<ArrowLeftIcon className="size-4" />
</Button>
<div>
<h1 className="text-xl font-semibold tracking-tight text-foreground">{accessGroup.access_group_name}</h1>
<div className="flex items-center gap-1 text-sm text-muted-foreground">
<span>ID: {accessGroup.access_group_id}</span>
<CopyButton value={accessGroup.access_group_id} label="Copy access group ID" />
<span>
{t("accessGroups.detail.idLabel")}
{accessGroup.access_group_id}
</span>
<CopyButton value={accessGroup.access_group_id} label={t("accessGroups.detail.copyId")} />
</div>
</div>
</div>
<Button onClick={() => setIsEditModalVisible(true)}>
<EditIcon className="size-4" />
Edit Access Group
{t("accessGroups.detail.edit")}
</Button>
</div>
<Card className="mb-6">
<CardHeader>
<CardTitle>Group Details</CardTitle>
<CardTitle>{t("accessGroups.detail.groupDetails")}</CardTitle>
</CardHeader>
<CardContent>
<dl className="grid grid-cols-[max-content_1fr] gap-x-4 gap-y-2 text-sm">
<dt className="text-muted-foreground">Description</dt>
<dt className="text-muted-foreground">{t("accessGroups.detail.description")}</dt>
<dd className="text-foreground">{accessGroup.description || "—"}</dd>
<dt className="text-muted-foreground">Created</dt>
<dt className="text-muted-foreground">{t("accessGroups.detail.created")}</dt>
<dd className="flex items-center gap-1 text-foreground">
{new Date(accessGroup.created_at).toLocaleString()}
{accessGroup.created_by && (
<>
<span>by</span>
<span>{t("accessGroups.detail.by")}</span>
<DefaultProxyAdminTag userId={accessGroup.created_by} />
</>
)}
</dd>
<dt className="text-muted-foreground">Last Updated</dt>
<dt className="text-muted-foreground">{t("accessGroups.detail.lastUpdated")}</dt>
<dd className="flex items-center gap-1 text-foreground">
{new Date(accessGroup.updated_at).toLocaleString()}
{accessGroup.updated_by && (
<>
<span>by</span>
<span>{t("accessGroups.detail.by")}</span>
<DefaultProxyAdminTag userId={accessGroup.updated_by} />
</>
)}
@ -200,13 +213,15 @@ export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailPr
<CardHeader>
<CardTitle className="flex items-center gap-2">
<KeyIcon className="size-4" />
Attached Keys
{t("accessGroups.detail.attachedKeys")}
<Badge variant="secondary">{keys.length}</Badge>
</CardTitle>
{keys.length > MAX_PREVIEW && (
<CardAction>
<Button variant="link" size="sm" onClick={() => setShowAllKeys(!showAllKeys)}>
{showAllKeys ? "Show Less" : `View All (${keys.length})`}
{showAllKeys
? t("accessGroups.detail.showLess")
: t("accessGroups.detail.viewAll", { count: keys.length })}
</Button>
</CardAction>
)}
@ -219,7 +234,7 @@ export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailPr
))}
</div>
) : (
<p className="text-sm text-muted-foreground">No keys attached</p>
<p className="text-sm text-muted-foreground">{t("accessGroups.detail.noKeys")}</p>
)}
</CardContent>
</Card>
@ -228,13 +243,15 @@ export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailPr
<CardHeader>
<CardTitle className="flex items-center gap-2">
<UsersIcon className="size-4" />
Attached Teams
{t("accessGroups.detail.attachedTeams")}
<Badge variant="secondary">{teams.length}</Badge>
</CardTitle>
{teams.length > MAX_PREVIEW && (
<CardAction>
<Button variant="link" size="sm" onClick={() => setShowAllTeams(!showAllTeams)}>
{showAllTeams ? "Show Less" : `View All (${teams.length})`}
{showAllTeams
? t("accessGroups.detail.showLess")
: t("accessGroups.detail.viewAll", { count: teams.length })}
</Button>
</CardAction>
)}
@ -247,7 +264,7 @@ export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailPr
))}
</div>
) : (
<p className="text-sm text-muted-foreground">No teams attached</p>
<p className="text-sm text-muted-foreground">{t("accessGroups.detail.noTeams")}</p>
)}
</CardContent>
</Card>
@ -259,37 +276,37 @@ export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailPr
<TabsList variant="line" className="h-auto w-full justify-start rounded-none border-b p-0">
<TabsTrigger value="models" className="flex-none gap-2 rounded-none px-4 py-2">
<LayersIcon className="size-4" />
Models
{t("accessGroups.detail.tabs.models")}
<Badge variant="secondary">{models.length}</Badge>
</TabsTrigger>
<TabsTrigger value="mcp" className="flex-none gap-2 rounded-none px-4 py-2">
<ServerIcon className="size-4" />
MCP Servers
{t("accessGroups.detail.tabs.mcpServers")}
<Badge variant="secondary">{mcpServers.length}</Badge>
</TabsTrigger>
<TabsTrigger value="agents" className="flex-none gap-2 rounded-none px-4 py-2">
<BotIcon className="size-4" />
Agents
{t("accessGroups.detail.tabs.agents")}
<Badge variant="secondary">{agents.length}</Badge>
</TabsTrigger>
<TabsTrigger value="skills" className="flex-none gap-2 rounded-none px-4 py-2">
<SparklesIcon className="size-4" />
Skills
{t("accessGroups.detail.tabs.skills")}
<Badge variant="secondary">{skills.length}</Badge>
</TabsTrigger>
</TabsList>
<TabsContent value="models" className="pt-4">
<ResourceList items={models} emptyMessage="No models assigned to this group" />
<ResourceList items={models} emptyMessage={t("accessGroups.detail.empty.models")} />
</TabsContent>
<TabsContent value="mcp" className="pt-4">
<ResourceList items={mcpServers} emptyMessage="No MCP servers assigned to this group" />
<ResourceList items={mcpServers} emptyMessage={t("accessGroups.detail.empty.mcpServers")} />
<MCPToolPermissionList servers={mcpServers} toolPermissions={accessGroup.mcp_tool_permissions ?? null} />
</TabsContent>
<TabsContent value="agents" className="pt-4">
<ResourceList items={agents} emptyMessage="No agents assigned to this group" />
<ResourceList items={agents} emptyMessage={t("accessGroups.detail.empty.agents")} />
</TabsContent>
<TabsContent value="skills" className="pt-4">
<ResourceList items={skills} emptyMessage="No skills assigned to this group" />
<ResourceList items={skills} emptyMessage={t("accessGroups.detail.empty.skills")} />
</TabsContent>
</Tabs>
</CardContent>

View file

@ -2,6 +2,7 @@
import { BotIcon, InfoIcon, LayersIcon, ServerIcon, SparklesIcon } from "lucide-react";
import type { UseFormReturn } from "react-hook-form";
import { useTranslation } from "react-i18next";
import { z } from "zod";
import { useAgents } from "@/app/(dashboard)/hooks/agents/useAgents";
@ -16,9 +17,10 @@ import SkillSelector from "@/components/skills/SkillSelector";
import { Input } from "@/components/ui/input";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { Textarea } from "@/components/ui/textarea";
import i18n from "@/i18n/i18n";
export const accessGroupFormSchema = z.object({
name: z.string().min(1, "Please enter the access group name"),
name: z.string().min(1, { error: () => i18n.t("accessGroups.form.name.required") }),
description: z.string(),
modelIds: z.array(z.string()),
mcpServerIds: z.array(z.string()),
@ -80,6 +82,7 @@ export function AccessGroupBaseForm({
onTabChange,
initialServerIds,
}: AccessGroupBaseFormProps) {
const { t } = useTranslation();
const { accessToken } = useAuthorized();
const { data: agentsData } = useAgents();
const { data: mcpServersData } = useMCPServers();
@ -98,56 +101,61 @@ export function AccessGroupBaseForm({
<TabsList className="w-full">
<TabsTrigger value={GENERAL_TAB}>
<InfoIcon size={16} />
General Info
{t("accessGroups.form.tabs.general")}
</TabsTrigger>
<TabsTrigger value={MODELS_TAB}>
<LayersIcon size={16} />
Models
{t("accessGroups.form.tabs.models")}
</TabsTrigger>
<TabsTrigger value={MCP_SERVERS_TAB}>
<ServerIcon size={16} />
MCP Servers
{t("accessGroups.form.tabs.mcpServers")}
</TabsTrigger>
<TabsTrigger value={AGENTS_TAB}>
<BotIcon size={16} />
Agents
{t("accessGroups.form.tabs.agents")}
</TabsTrigger>
<TabsTrigger value={SKILLS_TAB}>
<SparklesIcon size={16} />
Skills
{t("accessGroups.form.tabs.skills")}
</TabsTrigger>
</TabsList>
<TabsContent value={GENERAL_TAB} className="pt-4">
<FieldGroup>
<FormField control={form.control} name="name" label="Group Name">
<FormField control={form.control} name="name" label={t("accessGroups.form.name.label")}>
{({ ref, ...field }) => (
<Input {...field} ref={ref} placeholder="e.g. Engineering Team" disabled={isNameDisabled} />
<Input
{...field}
ref={ref}
placeholder={t("accessGroups.form.name.placeholder")}
disabled={isNameDisabled}
/>
)}
</FormField>
<FormField control={form.control} name="description" label="Description">
<FormField control={form.control} name="description" label={t("accessGroups.form.description.label")}>
{({ ref, ...field }) => (
<Textarea {...field} ref={ref} rows={4} placeholder="Describe the purpose of this access group..." />
<Textarea {...field} ref={ref} rows={4} placeholder={t("accessGroups.form.description.placeholder")} />
)}
</FormField>
</FieldGroup>
</TabsContent>
<TabsContent value={MODELS_TAB} className="pt-4">
<FormField control={form.control} name="modelIds" label="Allowed Models">
<FormField control={form.control} name="modelIds" label={t("accessGroups.form.models.label")}>
{(field) => <ModelSelect context="global" value={field.value} onChange={field.onChange} />}
</FormField>
</TabsContent>
<TabsContent value={MCP_SERVERS_TAB} className="pt-4">
<FormField control={form.control} name="mcpServerIds" label="Allowed MCP Servers">
<FormField control={form.control} name="mcpServerIds" label={t("accessGroups.form.mcpServers.label")}>
{({ id, value, onChange }) => (
<MultiSelect
id={id}
value={value}
onValueChange={onChange}
options={mcpServerOptions}
placeholder="Select MCP servers"
placeholder={t("accessGroups.form.mcpServers.placeholder")}
/>
)}
</FormField>
@ -160,14 +168,14 @@ export function AccessGroupBaseForm({
</TabsContent>
<TabsContent value={AGENTS_TAB} className="pt-4">
<FormField control={form.control} name="agentIds" label="Allowed Agents">
<FormField control={form.control} name="agentIds" label={t("accessGroups.form.agents.label")}>
{({ id, value, onChange }) => (
<MultiSelect
id={id}
value={value}
onValueChange={onChange}
options={agentOptions}
placeholder="Select agents"
placeholder={t("accessGroups.form.agents.placeholder")}
/>
)}
</FormField>
@ -177,15 +185,15 @@ export function AccessGroupBaseForm({
<FormField
control={form.control}
name="skillNames"
label="Allowed Skills"
description="Enabled skills are visible to everyone. Grant disabled (private) Claude Code plugins to the teams and keys in this group here."
label={t("accessGroups.form.skills.label")}
description={t("accessGroups.form.skills.description")}
>
{({ value, onChange }) => (
<SkillSelector
onChange={onChange}
value={value}
accessToken={accessToken || ""}
placeholder="Select skills (optional)"
placeholder={t("accessGroups.form.skills.placeholder")}
/>
)}
</FormField>

View file

@ -2,6 +2,7 @@
import React, { useState } from "react";
import { useFormState } from "react-hook-form";
import { useTranslation } from "react-i18next";
import { toast } from "@/lib/toast";
import { useZodForm } from "@/lib/forms/useZodForm";
@ -39,6 +40,7 @@ const toFormValues = (accessGroup: AccessGroupResponse): AccessGroupFormValues =
});
function AccessGroupEditForm({ accessGroup, onCancel, onSuccess }: Omit<AccessGroupEditModalProps, "visible">) {
const { t } = useTranslation();
const form = useZodForm(accessGroupFormSchema, { defaultValues: toFormValues(accessGroup) });
const { dirtyFields } = useFormState({ control: form.control });
const editMutation = useEditAccessGroup();
@ -66,7 +68,7 @@ function AccessGroupEditForm({ accessGroup, onCancel, onSuccess }: Omit<AccessGr
{ accessGroupId: accessGroup.access_group_id, params },
{
onSuccess: () => {
toast.success("Access group updated successfully");
toast.success(t("accessGroups.edit.success"));
onSuccess?.();
onCancel();
},
@ -87,10 +89,10 @@ function AccessGroupEditForm({ accessGroup, onCancel, onSuccess }: Omit<AccessGr
<div className="mt-6 flex justify-end gap-2">
<Button type="button" variant="outline" onClick={onCancel} disabled={editMutation.isPending}>
Cancel
{t("accessGroups.edit.cancel")}
</Button>
<Button type="button" onClick={() => void handleOk()} disabled={editMutation.isPending}>
Save Changes
{t("accessGroups.edit.save")}
</Button>
</div>
</form>
@ -98,11 +100,12 @@ function AccessGroupEditForm({ accessGroup, onCancel, onSuccess }: Omit<AccessGr
}
export function AccessGroupEditModal({ visible, accessGroup, onCancel, onSuccess }: AccessGroupEditModalProps) {
const { t } = useTranslation();
return (
<Dialog open={visible} onOpenChange={(open) => !open && onCancel()}>
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[700px]">
<DialogHeader>
<DialogTitle>Edit Access Group</DialogTitle>
<DialogTitle>{t("accessGroups.edit.title")}</DialogTitle>
</DialogHeader>
<AccessGroupEditForm
key={accessGroup.access_group_id}

View file

@ -3,6 +3,7 @@ import { AccessGroupResponse, useAccessGroups } from "@/app/(dashboard)/hooks/ac
import { useDeleteAccessGroup } from "@/app/(dashboard)/hooks/accessGroups/useDeleteAccessGroup";
import { Boxes, Plus, SearchIcon, X } from "lucide-react";
import { useMemo, useState } from "react";
import { useTranslation } from "react-i18next";
import DeleteResourceModal from "@/components/common_components/DeleteResourceModal";
import { PageHeader, PageHeaderControls, PageHeaderDescription, PageHeaderTitle } from "@/components/shared/PageHeader";
import { Button } from "@/components/ui/button";
@ -33,6 +34,7 @@ function mapResponseToAccessGroup(r: AccessGroupResponse): AccessGroup {
}
export function AccessGroupsPage() {
const { t } = useTranslation();
const { userRole } = useAuthorized();
// Admin Viewer follows the read-parity rule: see access groups, no writes.
const canModify = isProxyAdminRole(userRole ?? "");
@ -65,14 +67,14 @@ export function AccessGroupsPage() {
<PageHeader>
<PageHeaderTitle>
<Boxes />
Access Groups
{t("accessGroups.page.title")}
</PageHeaderTitle>
<PageHeaderDescription>Manage resource permissions for your organization</PageHeaderDescription>
<PageHeaderDescription>{t("accessGroups.page.description")}</PageHeaderDescription>
{canModify && (
<PageHeaderControls>
<Button onClick={() => setIsCreateModalVisible(true)}>
<Plus className="size-4" />
Create Access Group
{t("accessGroups.page.create")}
</Button>
</PageHeaderControls>
)}
@ -85,13 +87,17 @@ export function AccessGroupsPage() {
<SearchIcon className="size-4 text-muted-foreground" />
</InputGroupAddon>
<InputGroupInput
placeholder="Search groups by name, ID, or description..."
placeholder={t("accessGroups.page.searchPlaceholder")}
value={searchText}
onChange={(e) => setSearchText(e.target.value)}
/>
{searchText && (
<InputGroupAddon align="inline-end">
<InputGroupButton size="icon-xs" aria-label="Clear search" onClick={() => setSearchText("")}>
<InputGroupButton
size="icon-xs"
aria-label={t("accessGroups.page.clearSearch")}
onClick={() => setSearchText("")}
>
<X />
</InputGroupButton>
</InputGroupAddon>
@ -113,13 +119,13 @@ export function AccessGroupsPage() {
<DeleteResourceModal
isOpen={!!groupToDelete}
title="Delete Access Group"
message="Are you sure you want to delete this access group? This action cannot be undone."
resourceInformationTitle="Access Group Information"
title={t("accessGroups.page.deleteModal.title")}
message={t("accessGroups.page.deleteModal.message")}
resourceInformationTitle={t("accessGroups.page.deleteModal.informationTitle")}
resourceInformation={[
{ label: "ID", value: groupToDelete?.id, code: true },
{ label: "Name", value: groupToDelete?.name },
{ label: "Description", value: groupToDelete?.description || "—" },
{ label: t("accessGroups.page.deleteModal.id"), value: groupToDelete?.id, code: true },
{ label: t("accessGroups.page.deleteModal.name"), value: groupToDelete?.name },
{ label: t("accessGroups.page.deleteModal.description"), value: groupToDelete?.description || "—" },
]}
onCancel={() => setGroupToDelete(null)}
onOk={() => {

View file

@ -3,6 +3,7 @@
import { SortingState } from "@tanstack/react-table";
import { Layers } from "lucide-react";
import { useMemo, useState } from "react";
import { useTranslation } from "react-i18next";
import { DataTable } from "@/components/shared/DataTable";
@ -21,18 +22,17 @@ interface AccessGroupsTableProps {
const PAGE_SIZE_OPTIONS = [10, 25, 50];
function EmptyState({ isFiltered }: { isFiltered: boolean }) {
const { t } = useTranslation();
return (
<div className="flex flex-col items-center gap-1 py-6">
<div className="mb-1 flex size-10 items-center justify-center rounded-lg bg-muted">
<Layers className="size-5 text-muted-foreground" />
</div>
<div className="text-sm font-medium text-foreground">
{isFiltered ? "No matching access groups" : "No access groups yet"}
{isFiltered ? t("accessGroups.table.empty.filteredTitle") : t("accessGroups.table.empty.title")}
</div>
<div className="text-sm text-muted-foreground">
{isFiltered
? "Try a different search term."
: "Create an access group to manage resource permissions for your organization."}
{isFiltered ? t("accessGroups.table.empty.filteredDescription") : t("accessGroups.table.empty.description")}
</div>
</div>
);
@ -46,12 +46,13 @@ export function AccessGroupsTable({
onGroupClick,
onDeleteClick,
}: AccessGroupsTableProps) {
const { t } = useTranslation();
const [sorting, setSorting] = useState<SortingState>([]);
const columns = useMemo(() => {
const deps = { canModify, onGroupClick, onDeleteClick };
const deps = { canModify, onGroupClick, onDeleteClick, t };
return getAccessGroupsTableColumns(deps);
}, [canModify, onGroupClick, onDeleteClick]);
}, [canModify, onGroupClick, onDeleteClick, t]);
return (
<DataTable
@ -64,7 +65,7 @@ export function AccessGroupsTable({
paginationMode="client"
pageSizeOptions={PAGE_SIZE_OPTIONS}
isLoading={isLoading}
loadingMessage="Loading access groups…"
loadingMessage={t("accessGroups.table.loading")}
noDataMessage={<EmptyState isFiltered={isFiltered} />}
size="compact"
/>

View file

@ -1,7 +1,9 @@
"use client";
import { ColumnDef } from "@tanstack/react-table";
import type { TFunction } from "i18next";
import { Bot, Layers, MoreHorizontal, Server, Sparkles, Trash2 } from "lucide-react";
import { useTranslation } from "react-i18next";
import { DataTableSortHeader } from "@/components/shared/DataTable";
import { DateCell, IdentityCell } from "@/components/shared/table_cells";
@ -37,12 +39,13 @@ const RESOURCE_TONES: Record<"models" | "mcpServers" | "agents" | "skills", Reso
};
function ResourcesCell({ group }: { group: AccessGroup }) {
const { t } = useTranslation();
const items = [
{ key: "models" as const, label: "Models", count: group.modelIds.length },
{ key: "mcpServers" as const, label: "MCP Servers", count: group.mcpServerIds.length },
{ key: "agents" as const, label: "Agents", count: group.agentIds.length },
{ key: "skills" as const, label: "Skills", count: group.skillNames.length },
];
{ key: "models" as const, label: "accessGroups.table.resources.models", count: group.modelIds.length },
{ key: "mcpServers" as const, label: "accessGroups.table.resources.mcpServers", count: group.mcpServerIds.length },
{ key: "agents" as const, label: "accessGroups.table.resources.agents", count: group.agentIds.length },
{ key: "skills" as const, label: "accessGroups.table.resources.skills", count: group.skillNames.length },
] as const;
return (
<div className="flex items-center gap-1.5">
@ -52,7 +55,7 @@ function ResourcesCell({ group }: { group: AccessGroup }) {
return (
<span
key={item.key}
title={`${item.count} ${item.label}`}
title={t(item.label, { count: item.count })}
className={cn(
"inline-flex items-center gap-1.5 rounded-md px-2 py-1 text-xs font-medium ring-1 ring-inset [&_svg]:size-3.5",
tone.className,
@ -74,10 +77,11 @@ function AccessGroupRowActions({
group: AccessGroup;
onDeleteClick: (group: AccessGroup) => void;
}) {
const { t } = useTranslation();
return (
<DropdownMenu>
<DropdownMenuTrigger
aria-label="Open access group actions"
aria-label={t("accessGroups.table.rowActions.open")}
data-testid={`access-group-actions-${group.id}`}
className={cn(buttonVariants({ variant: "ghost", size: "icon-sm" }), "text-muted-foreground")}
>
@ -90,7 +94,7 @@ function AccessGroupRowActions({
onClick={() => onDeleteClick(group)}
>
<Trash2 />
Delete access group
{t("accessGroups.table.rowActions.delete")}
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
@ -101,19 +105,21 @@ interface AccessGroupsTableColumnsDeps {
canModify: boolean;
onGroupClick: (id: string) => void;
onDeleteClick: (group: AccessGroup) => void;
t: TFunction;
}
export const getAccessGroupsTableColumns = ({
canModify,
onGroupClick,
onDeleteClick,
t,
}: AccessGroupsTableColumnsDeps): ColumnDef<AccessGroup>[] => {
const columns: ColumnDef<AccessGroup>[] = [
{
id: "id",
accessorKey: "id",
meta: { title: "ID" },
header: "ID",
meta: { title: t("accessGroups.table.columns.id") },
header: t("accessGroups.table.columns.id"),
size: 200,
enableSorting: false,
cell: ({ row }) => (
@ -127,8 +133,8 @@ export const getAccessGroupsTableColumns = ({
{
id: "name",
accessorKey: "name",
meta: { title: "Name" },
header: ({ column }) => <DataTableSortHeader column={column} title="Name" />,
meta: { title: t("accessGroups.table.columns.name") },
header: ({ column }) => <DataTableSortHeader column={column} title={t("accessGroups.table.columns.name")} />,
size: 220,
enableSorting: true,
cell: ({ row }) => {
@ -142,8 +148,8 @@ export const getAccessGroupsTableColumns = ({
},
{
id: "resources",
meta: { title: "Resources" },
header: "Resources",
meta: { title: t("accessGroups.table.columns.resources") },
header: t("accessGroups.table.columns.resources"),
size: 220,
enableSorting: false,
cell: ({ row }) => <ResourcesCell group={row.original} />,
@ -151,8 +157,8 @@ export const getAccessGroupsTableColumns = ({
{
id: "createdAt",
accessorKey: "createdAt",
meta: { title: "Created" },
header: ({ column }) => <DataTableSortHeader column={column} title="Created" />,
meta: { title: t("accessGroups.table.columns.created") },
header: ({ column }) => <DataTableSortHeader column={column} title={t("accessGroups.table.columns.created")} />,
size: 150,
enableSorting: true,
sortingFn: "datetime",
@ -161,8 +167,8 @@ export const getAccessGroupsTableColumns = ({
{
id: "updatedAt",
accessorKey: "updatedAt",
meta: { title: "Updated" },
header: "Updated",
meta: { title: t("accessGroups.table.columns.updated") },
header: t("accessGroups.table.columns.updated"),
size: 150,
enableSorting: false,
cell: ({ row }) => <DateCell value={row.original.updatedAt} precision="date" />,
@ -178,7 +184,7 @@ export const getAccessGroupsTableColumns = ({
{
id: "actions",
meta: { className: "text-right", headerClassName: "text-right" },
header: () => <span className="sr-only">Actions</span>,
header: () => <span className="sr-only">{t("accessGroups.table.columns.actions")}</span>,
size: 64,
enableSorting: false,
enableHiding: false,

View file

@ -3,6 +3,7 @@
import { useMutation, useQueryClient } from "@tanstack/react-query";
import { BotIcon, InfoIcon, LayersIcon, ServerIcon, SparklesIcon } from "lucide-react";
import * as React from "react";
import { useTranslation } from "react-i18next";
import { accessGroupKeys } from "@/app/(dashboard)/hooks/accessGroups/useAccessGroups";
import { useAgents } from "@/app/(dashboard)/hooks/agents/useAgents";
@ -44,6 +45,7 @@ export const AccessGroupCreateDialog = ({
onOpenChange,
createAccessGroup = defaultCreateAccessGroup,
}: AccessGroupCreateDialogProps) => {
const { t } = useTranslation();
const queryClient = useQueryClient();
const form = useZodForm(accessGroupCreateSchema, { defaultValues: emptyAccessGroupFormValues });
const [activeTab, setActiveTab] = React.useState(GENERAL_TAB);
@ -70,12 +72,12 @@ export const AccessGroupCreateDialog = ({
const mutation = useMutation({
mutationFn: (body: AccessGroupCreateBody) => createAccessGroup(body),
onSuccess: () => {
toast.success("Access group created successfully");
toast.success(t("accessGroups.create.success"));
queryClient.invalidateQueries({ queryKey: accessGroupKeys.all });
closeAndReset();
},
onError: (error: unknown) =>
toast.fromError(error instanceof Error ? error.message : "Failed to create access group"),
toast.fromError(error instanceof Error ? error.message : t("accessGroups.create.failed")),
});
const handleOpenChange = (nextOpen: boolean) => {
@ -100,7 +102,7 @@ export const AccessGroupCreateDialog = ({
<Dialog open={open} onOpenChange={handleOpenChange}>
<DialogContent className="sm:max-w-2xl max-h-[90vh] overflow-y-auto">
<DialogHeader>
<DialogTitle>Create Access Group</DialogTitle>
<DialogTitle>{t("accessGroups.create.title")}</DialogTitle>
</DialogHeader>
<form onSubmit={onSubmit} noValidate>
@ -108,38 +110,40 @@ export const AccessGroupCreateDialog = ({
<TabsList className="w-full">
<TabsTrigger value={GENERAL_TAB}>
<InfoIcon />
General Info
{t("accessGroups.form.tabs.general")}
</TabsTrigger>
<TabsTrigger value="models">
<LayersIcon />
Models
{t("accessGroups.form.tabs.models")}
</TabsTrigger>
<TabsTrigger value="mcp-servers">
<ServerIcon />
MCP Servers
{t("accessGroups.form.tabs.mcpServers")}
</TabsTrigger>
<TabsTrigger value="agents">
<BotIcon />
Agents
{t("accessGroups.form.tabs.agents")}
</TabsTrigger>
<TabsTrigger value="skills">
<SparklesIcon />
Skills
{t("accessGroups.form.tabs.skills")}
</TabsTrigger>
</TabsList>
<TabsContent value={GENERAL_TAB} className="pt-4">
<FieldGroup>
<FormField control={form.control} name="name" label="Group Name">
{({ ref, ...field }) => <Input {...field} ref={ref} placeholder="e.g. Engineering Team" />}
<FormField control={form.control} name="name" label={t("accessGroups.form.name.label")}>
{({ ref, ...field }) => (
<Input {...field} ref={ref} placeholder={t("accessGroups.form.name.placeholder")} />
)}
</FormField>
<FormField control={form.control} name="description" label="Description">
<FormField control={form.control} name="description" label={t("accessGroups.form.description.label")}>
{({ ref, ...field }) => (
<Textarea
{...field}
ref={ref}
rows={4}
placeholder="Describe the purpose of this access group..."
placeholder={t("accessGroups.form.description.placeholder")}
/>
)}
</FormField>
@ -147,20 +151,20 @@ export const AccessGroupCreateDialog = ({
</TabsContent>
<TabsContent value="models" className="pt-4">
<FormField control={form.control} name="modelIds" label="Allowed Models">
<FormField control={form.control} name="modelIds" label={t("accessGroups.form.models.label")}>
{(field) => <ModelSelect context="global" value={field.value} onChange={field.onChange} />}
</FormField>
</TabsContent>
<TabsContent value="mcp-servers" className="pt-4">
<FormField control={form.control} name="mcpServerIds" label="Allowed MCP Servers">
<FormField control={form.control} name="mcpServerIds" label={t("accessGroups.form.mcpServers.label")}>
{({ id, value, onChange }) => (
<MultiSelect
id={id}
value={value}
onValueChange={onChange}
options={mcpServerOptions}
placeholder="Select MCP servers"
placeholder={t("accessGroups.form.mcpServers.placeholder")}
/>
)}
</FormField>
@ -172,14 +176,14 @@ export const AccessGroupCreateDialog = ({
</TabsContent>
<TabsContent value="agents" className="pt-4">
<FormField control={form.control} name="agentIds" label="Allowed Agents">
<FormField control={form.control} name="agentIds" label={t("accessGroups.form.agents.label")}>
{({ id, value, onChange }) => (
<MultiSelect
id={id}
value={value}
onValueChange={onChange}
options={agentOptions}
placeholder="Select agents"
placeholder={t("accessGroups.form.agents.placeholder")}
/>
)}
</FormField>
@ -189,15 +193,15 @@ export const AccessGroupCreateDialog = ({
<FormField
control={form.control}
name="skillNames"
label="Allowed Skills"
description="Enabled skills are visible to everyone. Grant disabled (private) Claude Code plugins to the teams and keys in this group here."
label={t("accessGroups.form.skills.label")}
description={t("accessGroups.form.skills.description")}
>
{({ value, onChange }) => (
<SkillSelector
onChange={onChange}
value={value}
accessToken={accessToken || ""}
placeholder="Select skills (optional)"
placeholder={t("accessGroups.form.skills.placeholder")}
/>
)}
</FormField>
@ -211,10 +215,10 @@ export const AccessGroupCreateDialog = ({
onClick={() => handleOpenChange(false)}
disabled={mutation.isPending}
>
Cancel
{t("accessGroups.create.cancel")}
</Button>
<Button type="submit" disabled={mutation.isPending}>
{mutation.isPending ? "Creating..." : "Create Group"}
{mutation.isPending ? t("accessGroups.create.submitting") : t("accessGroups.create.submit")}
</Button>
</DialogFooter>
</form>

View file

@ -10,7 +10,9 @@ import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { Info, TriangleAlert } from "lucide-react";
import React, { useEffect, useState } from "react";
import { Trans, useTranslation } from "react-i18next";
import { useBaseUrl } from "@/components/constants";
import i18n from "@/i18n/i18n";
import { toast } from "@/lib/toast";
import { addAllowedIP, deleteAllowedIP, getAllowedIPs, getSSOSettings } from "@/components/networking";
import SCIMConfig from "@/components/SCIM";
@ -38,22 +40,23 @@ import { useZodForm } from "@/lib/forms/useZodForm";
import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
const allowedIPSchema = z.object({
ip: z.string().min(1, "Please enter an IP address"),
ip: z.string().min(1, { error: () => i18n.t("adminPanel.page.addIp.required") }),
});
type AllowedIPFormValues = z.infer<typeof allowedIPSchema>;
const AddAllowedIPForm = ({ onSubmit }: { onSubmit: (values: AllowedIPFormValues) => Promise<void> }) => {
const { t } = useTranslation();
const form = useZodForm(allowedIPSchema, { defaultValues: { ip: "" } });
return (
<form onSubmit={form.handleSubmit(onSubmit)}>
<FieldGroup>
<FormField control={form.control} name="ip">
{({ ref, ...field }) => <Input ref={ref} placeholder="Enter IP address" {...field} />}
{({ ref, ...field }) => <Input ref={ref} placeholder={t("adminPanel.page.addIp.placeholder")} {...field} />}
</FormField>
<div>
<Button type="submit">Add IP Address</Button>
<Button type="submit">{t("adminPanel.page.addIp.submit")}</Button>
</div>
</FieldGroup>
</form>
@ -65,6 +68,7 @@ interface AdminPanelProps {
}
const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
const { t } = useTranslation();
const { premiumUser, accessToken, userId: userID } = useAuthorized();
const form = useSSOSettingsForm("admin-panel");
const [isAddSSOModalVisible, setIsAddSSOModalVisible] = useState(false);
@ -107,7 +111,7 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
const handleShowAllowedIPs = async () => {
try {
if (premiumUser !== true) {
toast.fromError("This feature is only available for premium users. Please upgrade your account.");
toast.fromError(t("adminPanel.page.toasts.premiumRequired"));
return;
}
if (accessToken) {
@ -118,7 +122,7 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
}
} catch (error) {
console.error("Error fetching allowed IPs:", error);
toast.fromError(`Failed to fetch allowed IPs ${error}`);
toast.fromError(t("adminPanel.page.toasts.fetchIpsFailed", { error: String(error) }));
setAllowedIPs([all_ip_address_allowed]);
} finally {
if (premiumUser === true) {
@ -134,11 +138,11 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
// Fetch the updated list of IPs
const updatedIPs = await getAllowedIPs(accessToken);
setAllowedIPs(updatedIPs);
toast.success("IP address added successfully");
toast.success(t("adminPanel.page.toasts.addIpSuccess"));
}
} catch (error) {
console.error("Error adding IP:", error);
toast.fromError(`Failed to add IP address ${error}`);
toast.fromError(t("adminPanel.page.toasts.addIpFailed", { error: String(error) }));
} finally {
setIsAddIPModalVisible(false);
}
@ -156,10 +160,10 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
// Fetch the updated list of IPs
const updatedIPs = await getAllowedIPs(accessToken);
setAllowedIPs(updatedIPs.length > 0 ? updatedIPs : [all_ip_address_allowed]);
toast.success("IP address deleted successfully");
toast.success(t("adminPanel.page.toasts.deleteIpSuccess"));
} catch (error) {
console.error("Error deleting IP:", error);
toast.fromError(`Failed to delete IP address ${error}`);
toast.fromError(t("adminPanel.page.toasts.deleteIpFailed", { error: String(error) }));
} finally {
setIsDeleteIPModalVisible(false);
setIPToDelete(null);
@ -214,23 +218,20 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
const tabItems = [
{
key: "sso-settings",
label: "SSO Settings",
label: t("adminPanel.page.tabs.ssoSettings"),
children: <SSOSettings />,
},
{
key: "security-settings",
label: "Security Settings",
label: t("adminPanel.page.tabs.securitySettings"),
children: (
<>
<Card className="block p-6">
<h3 className="mb-2 text-base font-semibold text-foreground">✨ Security Settings</h3>
<h3 className="mb-2 text-base font-semibold text-foreground">{t("adminPanel.page.security.title")}</h3>
<Alert variant="warning">
<TriangleAlert />
<AlertTitle>SSO Configuration Deprecated</AlertTitle>
<AlertDescription>
Editing SSO Settings on this page is deprecated and will be removed in a future version. Please use the
SSO Settings tab for SSO configuration.
</AlertDescription>
<AlertTitle>{t("adminPanel.page.security.deprecatedTitle")}</AlertTitle>
<AlertDescription>{t("adminPanel.page.security.deprecatedDescription")}</AlertDescription>
</Alert>
<div
style={{
@ -243,12 +244,12 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
>
<div>
<Button style={{ width: "150px" }} onClick={() => setIsAddSSOModalVisible(true)}>
{ssoConfigured ? "Edit SSO Settings" : "Add SSO"}
{ssoConfigured ? t("adminPanel.page.security.editSso") : t("adminPanel.page.security.addSso")}
</Button>
</div>
<div>
<Button style={{ width: "150px" }} onClick={handleShowAllowedIPs}>
Allowed IPs
{t("adminPanel.page.security.allowedIps")}
</Button>
</div>
<div>
@ -257,10 +258,10 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
onClick={() =>
premiumUser === true
? setIsUIAccessControlModalVisible(true)
: toast.fromError("Only premium users can configure UI access control")
: toast.fromError(t("adminPanel.page.toasts.uiAccessControlPremiumRequired"))
}
>
UI Access Control
{t("adminPanel.page.security.uiAccessControl")}
</Button>
</div>
</div>
@ -282,23 +283,25 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
<Dialog open={isAllowedIPModalVisible} onOpenChange={(open) => !open && setIsAllowedIPModalVisible(false)}>
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[800px]">
<DialogHeader>
<DialogTitle>Manage Allowed IP Addresses</DialogTitle>
<DialogTitle>{t("adminPanel.page.allowedIps.title")}</DialogTitle>
</DialogHeader>
<Table>
<TableHeader>
<TableRow>
<TableHead>IP Address</TableHead>
<TableHead className="text-right">Action</TableHead>
<TableHead>{t("adminPanel.page.allowedIps.columns.ipAddress")}</TableHead>
<TableHead className="text-right">{t("adminPanel.page.allowedIps.columns.action")}</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{allowedIPs.map((ip, index) => (
<TableRow key={index}>
<TableCell>{ip}</TableCell>
<TableCell>
{ip === all_ip_address_allowed ? t("adminPanel.page.allowedIps.allAllowed") : ip}
</TableCell>
<TableCell className="text-right">
{ip !== all_ip_address_allowed && (
<Button onClick={() => handleDeleteIP(ip)} variant="destructive" size="sm">
Delete
{t("adminPanel.page.allowedIps.delete")}
</Button>
)}
</TableCell>
@ -308,9 +311,11 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
</Table>
<DialogFooter>
<Button className="mx-1" onClick={() => setIsAddIPModalVisible(true)}>
Add IP Address
{t("adminPanel.page.allowedIps.addIp")}
</Button>
<Button onClick={() => setIsAllowedIPModalVisible(false)}>
{t("adminPanel.page.allowedIps.close")}
</Button>
<Button onClick={() => setIsAllowedIPModalVisible(false)}>Close</Button>
</DialogFooter>
</DialogContent>
</Dialog>
@ -318,7 +323,7 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
<Dialog open={isAddIPModalVisible} onOpenChange={(open) => !open && setIsAddIPModalVisible(false)}>
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto">
<DialogHeader>
<DialogTitle>Add Allowed IP Address</DialogTitle>
<DialogTitle>{t("adminPanel.page.addIp.title")}</DialogTitle>
</DialogHeader>
<AddAllowedIPForm onSubmit={handleAddIP} />
</DialogContent>
@ -327,16 +332,18 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
<Dialog open={isDeleteIPModalVisible} onOpenChange={(open) => !open && setIsDeleteIPModalVisible(false)}>
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto">
<DialogHeader>
<DialogTitle>Confirm Delete</DialogTitle>
<DialogTitle>{t("adminPanel.page.deleteIp.title")}</DialogTitle>
</DialogHeader>
<span className="text-sm text-foreground">
Are you sure you want to delete the IP address: {ipToDelete}?
<Trans i18nKey="adminPanel.page.deleteIp.message" components={{ ip: <>{ipToDelete}</> }} />
</span>
<DialogFooter>
<Button className="mx-1" onClick={() => confirmDeleteIP()}>
Yes
{t("adminPanel.page.deleteIp.confirm")}
</Button>
<Button onClick={() => setIsDeleteIPModalVisible(false)}>
{t("adminPanel.page.deleteIp.close")}
</Button>
<Button onClick={() => setIsDeleteIPModalVisible(false)}>Close</Button>
</DialogFooter>
</DialogContent>
</Dialog>
@ -348,13 +355,13 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
>
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[600px]">
<DialogHeader>
<DialogTitle>UI Access Control Settings</DialogTitle>
<DialogTitle>{t("adminPanel.page.uiAccessControlModal.title")}</DialogTitle>
</DialogHeader>
<UIAccessControlForm
accessToken={accessToken}
onSuccess={() => {
handleUIAccessControlOk();
toast.success("UI Access Control settings updated successfully");
toast.success(t("adminPanel.page.toasts.uiAccessControlUpdated"));
}}
/>
</DialogContent>
@ -362,12 +369,18 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
</div>
<Alert variant="info">
<Info />
<AlertTitle>Login without SSO</AlertTitle>
<AlertTitle>{t("adminPanel.page.security.loginWithoutSsoTitle")}</AlertTitle>
<AlertDescription>
If you need to login without sso, you can access{" "}
<a href={nonSssoUrl} target="_blank" rel="noopener noreferrer">
<b>{nonSssoUrl}</b>{" "}
</a>
<Trans
i18nKey="adminPanel.page.security.loginWithoutSso"
components={{
url: (
<a href={nonSssoUrl} target="_blank" rel="noopener noreferrer">
<b>{nonSssoUrl}</b>{" "}
</a>
),
}}
/>
</AlertDescription>
</Alert>
</>
@ -380,7 +393,7 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
},
{
key: "ui-settings",
label: "UI Settings",
label: t("adminPanel.page.tabs.uiSettings"),
children: (
<div className="flex flex-col gap-4">
<UISettings />
@ -391,7 +404,7 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
},
{
key: "logging-settings",
label: "Logging Settings",
label: t("adminPanel.page.tabs.loggingSettings"),
children: <LoggingSettings />,
},
{
@ -406,20 +419,20 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
},
{
key: "plugins",
label: "Plugins",
label: t("adminPanel.page.tabs.plugins"),
children: <PluginSettings />,
},
{
key: "web-search-interception",
label: "Web Search Interception",
label: t("adminPanel.page.tabs.webSearchInterception"),
children: <WebSearchInterceptionSettings />,
},
];
return (
<div className="w-full m-2 mt-2 p-8">
<h2 className="mb-2 text-base font-semibold text-foreground">Admin Access</h2>
<p className="mb-4 text-sm text-foreground">Go to &apos;Internal Users&apos; page to add other admins.</p>
<h2 className="mb-2 text-base font-semibold text-foreground">{t("adminPanel.page.title")}</h2>
<p className="mb-4 text-sm text-foreground">{t("adminPanel.page.subtitle")}</p>
<Tabs defaultValue={tabItems[0].key}>
<TabsList variant="line" className="mb-4 h-auto flex-wrap">
{tabItems.map((item) => (

View file

@ -27,6 +27,7 @@ import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/component
import { Input } from "@/components/ui/input";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import { Field, FieldDescription, FieldError, FieldGroup, FieldLabel } from "@/components/ui/field";
import i18n from "@/i18n/i18n";
import type { KeyValueFormValue, KillSwitchConfig, KillSwitchFormValue } from "./kill_switch_config";
export interface AgentSkillFormValue {
@ -282,7 +283,7 @@ export const AgentTagsInput = ({
value,
onValueChange,
placeholder,
emptyText = "No matching options",
emptyText = i18n.t("agents.formKit.noMatchingOptions"),
...props
}: AgentTagsInputProps) => {
const anchor = useComboboxAnchor();
@ -396,7 +397,7 @@ export const AgentMultiSelect = ({
value,
onValueChange,
placeholder,
emptyText = "No matching options",
emptyText = i18n.t("agents.formKit.noMatchingOptions"),
...props
}: AgentMultiSelectProps) => {
const anchor = useComboboxAnchor();

View file

@ -1,14 +1,17 @@
import React from "react";
import type { TFunction } from "i18next";
import { useTranslation } from "react-i18next";
import type { components } from "@/lib/http/schema";
import { useQuery } from "@tanstack/react-query";
import { apiClient } from "@/components/networking";
import { Button } from "@/components/ui/button";
import { readAgentIdentity } from "./agent_identity";
const authenticationMessage = (error: boolean, lastAuthenticated?: string | null): string => {
if (error) return "Could not load authentication evidence";
if (lastAuthenticated) return `Last authenticated identity match: ${new Date(lastAuthenticated).toLocaleString()}`;
return "Configured, awaiting an authenticated request";
const authenticationMessage = (t: TFunction, error: boolean, lastAuthenticated?: string | null): string => {
if (error) return t("agents.identityDetails.evidence.loadFailed");
if (lastAuthenticated)
return t("agents.identityDetails.evidence.lastMatch", { date: new Date(lastAuthenticated).toLocaleString() });
return t("agents.identityDetails.evidence.awaiting");
};
export const AgentIdentityDetails = ({
@ -22,6 +25,7 @@ export const AgentIdentityDetails = ({
accessToken: string | null;
isAdmin: boolean;
}) => {
const { t } = useTranslation();
const identity = readAgentIdentity(value);
const { data, isError, isFetching, refetch } = useQuery({
queryKey: ["agent-identity", agentId, identity],
@ -36,31 +40,39 @@ export const AgentIdentityDetails = ({
});
if (!identity || !isAdmin) return null;
const executionLabel = data?.enabled ? "Enabled" : "Disabled";
const executionLabel = data?.enabled
? t("agents.identityDetails.execution.enabled")
: t("agents.identityDetails.execution.disabled");
return (
<section aria-label="Agent Identity" className="mb-6 space-y-2 rounded-lg border border-border p-4">
<h3 className="font-medium">Agent Identity: Microsoft Entra ID</h3>
<section
aria-label={t("agents.identityDetails.regionLabel")}
className="mb-6 space-y-2 rounded-lg border border-border p-4"
>
<h3 className="font-medium">{t("agents.identityDetails.title")}</h3>
<p className="text-sm">
Tenant: <span className="font-mono">{identity.tenant_id}</span>
{t("agents.identityDetails.tenant")} <span className="font-mono">{identity.tenant_id}</span>
</p>
<>
<p className="text-sm">
Application (Client) ID: <span className="font-mono">{identity.client_id}</span>
{t("agents.identityDetails.clientId")} <span className="font-mono">{identity.client_id}</span>
</p>
<p className="text-sm">
{t("agents.identityDetails.servicePrincipal")}{" "}
{identity.service_principal_id || t("agents.identityDetails.notConfigured")}
</p>
<p className="text-sm">Enterprise application Object ID: {identity.service_principal_id || "Not configured"}</p>
</>
<p className="text-sm">
Execution: {data ? executionLabel : "Loading"} · Mode: {data?.execution_mode ?? "Loading"}
{t("agents.identityDetails.executionSummary", {
execution: data ? executionLabel : t("agents.identityDetails.loading"),
mode: data?.execution_mode ?? t("agents.identityDetails.loading"),
})}
</p>
<p className="text-sm">
{data?.identity?.active === false
? "Identity unbound; execution is disabled"
: authenticationMessage(isError, data?.last_authenticated_at)}
</p>
<p className="text-xs text-muted-foreground">
Recent evidence comes from a validated Entra token matching this binding. It is persisted across restarts and
cleared when the binding changes. Tool and model permissions are checked separately.
? t("agents.identityDetails.evidence.unbound")
: authenticationMessage(t, isError, data?.last_authenticated_at)}
</p>
<p className="text-xs text-muted-foreground">{t("agents.identityDetails.evidenceNote")}</p>
<div className="flex items-center gap-4">
<Button
variant="outline"
@ -70,10 +82,10 @@ export const AgentIdentityDetails = ({
void refetch();
}}
>
Refresh authentication evidence
{t("agents.identityDetails.refresh")}
</Button>
<a className="text-sm underline" href="/ui/logs/">
View request logs
{t("agents.identityDetails.viewLogs")}
</a>
</div>
</section>

View file

@ -1,26 +1,30 @@
import React, { useEffect, useState } from "react";
import { useWatch } from "react-hook-form";
import type { ParseKeys } from "i18next";
import { Trans, useTranslation } from "react-i18next";
import i18n from "@/i18n/i18n";
import { apiClient } from "@/components/networking";
import { Input } from "@/components/ui/input";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { AgentFormField, type AgentFormValues } from "./AgentFormKit";
import { entraTenantFromIssuer, IDENTITY_UUID_PATTERN } from "./agent_identity";
const PROVIDER_OPTIONS = [
{ value: "none", label: "No explicit identity binding" },
{ value: "microsoft_entra", label: "Microsoft Entra ID" },
const PROVIDER_OPTIONS: { value: string; labelKey: ParseKeys }[] = [
{ value: "none", labelKey: "agents.identityFields.providers.none" },
{ value: "microsoft_entra", labelKey: "agents.identityFields.providers.microsoftEntra" },
];
const EXECUTION_MODE_OPTIONS = [
{ value: "autonomous", label: "Autonomous" },
{ value: "delegated", label: "On behalf of a user" },
{ value: "both", label: "Both" },
const EXECUTION_MODE_OPTIONS: { value: string; labelKey: ParseKeys }[] = [
{ value: "autonomous", labelKey: "agents.identityFields.executionModes.autonomous" },
{ value: "delegated", labelKey: "agents.identityFields.executionModes.delegated" },
{ value: "both", labelKey: "agents.identityFields.executionModes.both" },
];
const EXECUTION_OPTIONS = [
{ value: "enabled", label: "Enabled" },
{ value: "disabled", label: "Disabled" },
const EXECUTION_OPTIONS: { value: string; labelKey: ParseKeys }[] = [
{ value: "enabled", labelKey: "agents.identityFields.execution.enabled" },
{ value: "disabled", labelKey: "agents.identityFields.execution.disabled" },
];
export const AgentIdentityFields = ({ accessToken }: { accessToken: string | null }) => {
const { t } = useTranslation();
const provider = useWatch<AgentFormValues>({ name: "identity_provider" });
const mode = useWatch<AgentFormValues>({ name: "execution_mode" });
const showScopes = mode !== "autonomous" && mode !== undefined;
@ -44,7 +48,7 @@ export const AgentIdentityFields = ({ accessToken }: { accessToken: string | nul
}
})
.catch(() => {
if (active) setError("Could not load the gateway's trusted identity providers");
if (active) setError(i18n.t("agents.identityFields.errors.providersLoadFailed"));
});
return () => {
active = false;
@ -53,18 +57,18 @@ export const AgentIdentityFields = ({ accessToken }: { accessToken: string | nul
return (
<>
<section aria-label="Agent Identity" className="my-6 space-y-4 rounded-lg border border-border p-4">
<section
aria-label={t("agents.identityFields.regionLabel")}
className="my-6 space-y-4 rounded-lg border border-border p-4"
>
<div>
<h3 className="font-medium">Agent Identity</h3>
<p className="mt-1 text-sm text-muted-foreground">
Connect an existing identity provider application to this agent. Its name and runtime address can change
independently.
</p>
<h3 className="font-medium">{t("agents.identityFields.title")}</h3>
<p className="mt-1 text-sm text-muted-foreground">{t("agents.identityFields.description")}</p>
</div>
<AgentFormField name="identity_provider" label="Identity Provider" defaultValue="none">
<AgentFormField name="identity_provider" label={t("agents.identityFields.provider")} defaultValue="none">
{({ value, onChange, id }) => (
<Select
items={PROVIDER_OPTIONS}
items={PROVIDER_OPTIONS.map((option) => ({ value: option.value, label: t(option.labelKey) }))}
value={typeof value === "string" ? value : "none"}
onValueChange={onChange}
>
@ -74,7 +78,7 @@ export const AgentIdentityFields = ({ accessToken }: { accessToken: string | nul
<SelectContent>
{PROVIDER_OPTIONS.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
{t(option.labelKey)}
</SelectItem>
))}
</SelectContent>
@ -85,13 +89,13 @@ export const AgentIdentityFields = ({ accessToken }: { accessToken: string | nul
<>
<AgentFormField
name="identity_tenant_id"
label="Trusted Entra Tenant"
rules={{ required: "Select a trusted tenant" }}
label={t("agents.identityFields.tenant.label")}
rules={{ required: t("agents.identityFields.tenant.required") }}
>
{({ value, onChange, id }) => (
<Select value={typeof value === "string" ? value : ""} onValueChange={onChange}>
<SelectTrigger id={id}>
<SelectValue placeholder="Select the gateway's trusted tenant" />
<SelectValue placeholder={t("agents.identityFields.tenant.placeholder")} />
</SelectTrigger>
<SelectContent>
{tenants.map((tenant) => (
@ -109,26 +113,24 @@ export const AgentIdentityFields = ({ accessToken }: { accessToken: string | nul
</p>
)}
{!error && tenants.length === 0 && (
<p className="text-sm text-muted-foreground">
No trusted Entra tenant is available. Configure JWT issuer and audience validation on the gateway first.
Dashboard Microsoft SSO is configured separately.
</p>
<p className="text-sm text-muted-foreground">{t("agents.identityFields.tenant.noneAvailable")}</p>
)}
<AgentFormField
name="identity_client_id"
label="Application (Client) ID"
label={t("agents.identityFields.clientId.label")}
rules={{
required: "Enter the Entra application client ID",
pattern: { value: IDENTITY_UUID_PATTERN, message: "Enter a valid application client UUID" },
required: t("agents.identityFields.clientId.required"),
pattern: { value: IDENTITY_UUID_PATTERN, message: t("agents.identityFields.clientId.invalid") },
}}
description={
<>
Find this under{" "}
<a className="underline" href="https://entra.microsoft.com/" target="_blank" rel="noreferrer">
Entra App registrations
</a>
, select your agent application, then Overview. No client secret is required here.
</>
<Trans
i18nKey="agents.identityFields.clientId.description"
components={{
registrations: (
<a className="underline" href="https://entra.microsoft.com/" target="_blank" rel="noreferrer" />
),
}}
/>
}
>
{({ value, onChange, ref, ...control }) => (
@ -141,10 +143,14 @@ export const AgentIdentityFields = ({ accessToken }: { accessToken: string | nul
/>
)}
</AgentFormField>
<AgentFormField name="execution_mode" label="Execution Mode" defaultValue="autonomous">
<AgentFormField
name="execution_mode"
label={t("agents.identityFields.executionMode")}
defaultValue="autonomous"
>
{({ value, onChange, id }) => (
<Select
items={EXECUTION_MODE_OPTIONS}
items={EXECUTION_MODE_OPTIONS.map((option) => ({ value: option.value, label: t(option.labelKey) }))}
value={typeof value === "string" ? value : "autonomous"}
onValueChange={onChange}
>
@ -154,7 +160,7 @@ export const AgentIdentityFields = ({ accessToken }: { accessToken: string | nul
<SelectContent>
{EXECUTION_MODE_OPTIONS.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
{t(option.labelKey)}
</SelectItem>
))}
</SelectContent>
@ -163,25 +169,25 @@ export const AgentIdentityFields = ({ accessToken }: { accessToken: string | nul
</AgentFormField>
<AgentFormField
name="identity_service_principal_id"
label="Enterprise Application Object ID"
label={t("agents.identityFields.servicePrincipal.label")}
rules={{
required: mode !== "delegated" ? "Enter the service principal Object ID" : false,
pattern: { value: IDENTITY_UUID_PATTERN, message: "Enter a valid service principal UUID" },
required: mode !== "delegated" ? t("agents.identityFields.servicePrincipal.required") : false,
pattern: { value: IDENTITY_UUID_PATTERN, message: t("agents.identityFields.servicePrincipal.invalid") },
}}
description={
<>
Open{" "}
<a
className="underline"
href="https://entra.microsoft.com/#view/Microsoft_AAD_IAM/StartboardApplicationsMenuBlade/~/AppAppsPreview"
target="_blank"
rel="noreferrer"
>
Entra Enterprise applications
</a>
, select this application, and copy its Object ID. The App registrations Object ID is a different
value.
</>
<Trans
i18nKey="agents.identityFields.servicePrincipal.description"
components={{
enterprise: (
<a
className="underline"
href="https://entra.microsoft.com/#view/Microsoft_AAD_IAM/StartboardApplicationsMenuBlade/~/AppAppsPreview"
target="_blank"
rel="noreferrer"
/>
),
}}
/>
}
>
{({ value, onChange, ref, ...control }) => (
@ -191,8 +197,8 @@ export const AgentIdentityFields = ({ accessToken }: { accessToken: string | nul
<AgentFormField
name="identity_required_roles"
label="Required Application Roles"
description="Comma-separated role values required on autonomous application tokens"
label={t("agents.identityFields.requiredRoles.label")}
description={t("agents.identityFields.requiredRoles.description")}
>
{({ value, onChange, ref, ...control }) => (
<Input
@ -209,9 +215,9 @@ export const AgentIdentityFields = ({ accessToken }: { accessToken: string | nul
<>
<AgentFormField
name="identity_required_scopes"
label="Required Delegated Scopes"
label={t("agents.identityFields.requiredScopes.label")}
defaultValue="user_impersonation"
rules={{ required: "Enter a delegated scope" }}
rules={{ required: t("agents.identityFields.requiredScopes.required") }}
>
{({ value, onChange, ref, ...control }) => (
<Input
@ -222,16 +228,13 @@ export const AgentIdentityFields = ({ accessToken }: { accessToken: string | nul
/>
)}
</AgentFormField>
<p className="text-sm text-muted-foreground">
Users must first sign in through this gateway&apos;s Microsoft SSO. Subsequent delegated calls must
satisfy both user and agent permissions.
</p>
<p className="text-sm text-muted-foreground">{t("agents.identityFields.requiredScopes.note")}</p>
</>
)}
<AgentFormField name="enabled" label="Execution" defaultValue={true}>
<AgentFormField name="enabled" label={t("agents.identityFields.executionLabel")} defaultValue={true}>
{({ value, onChange, id }) => (
<Select
items={EXECUTION_OPTIONS}
items={EXECUTION_OPTIONS.map((option) => ({ value: option.value, label: t(option.labelKey) }))}
value={value === false ? "disabled" : "enabled"}
onValueChange={(next) => onChange(next === "enabled")}
>
@ -241,18 +244,14 @@ export const AgentIdentityFields = ({ accessToken }: { accessToken: string | nul
<SelectContent>
{EXECUTION_OPTIONS.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
{t(option.labelKey)}
</SelectItem>
))}
</SelectContent>
</Select>
)}
</AgentFormField>
<p className="text-sm text-muted-foreground">
LiteLLM verifies the agent&apos;s Entra token before matching this identity. Saving these fields
configures the binding; an authenticated request provides verification. Runtime authentication headers are
configured separately.
</p>
<p className="text-sm text-muted-foreground">{t("agents.identityFields.verificationNote")}</p>
</>
)}
</section>

View file

@ -1,5 +1,6 @@
import { CircleAlert } from "lucide-react";
import React, { useState } from "react";
import { Trans, useTranslation } from "react-i18next";
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { Button } from "@/components/ui/button";
import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
@ -24,6 +25,7 @@ const AgentKillSwitchDangerZone: React.FC<AgentKillSwitchDangerZoneProps> = ({
accessToken,
isAdmin,
}) => {
const { t } = useTranslation();
const [isConfirmOpen, setIsConfirmOpen] = useState(false);
const [confirmationInput, setConfirmationInput] = useState("");
const [isFiring, setIsFiring] = useState(false);
@ -44,9 +46,9 @@ const AgentKillSwitchDangerZone: React.FC<AgentKillSwitchDangerZoneProps> = ({
const result = await triggerAgentKillSwitchCall(accessToken, agentId);
setLastResult(result);
setIsConfirmOpen(false);
toast.success(`Kill switch fired (HTTP ${result.status_code})`);
toast.success(t("agents.dangerZone.toasts.fired", { status: result.status_code }));
} catch (error) {
toast.error(error instanceof Error ? error.message : "Failed to fire kill switch");
toast.error(error instanceof Error ? error.message : t("agents.dangerZone.toasts.fireFailed"));
} finally {
setIsFiring(false);
}
@ -55,38 +57,33 @@ const AgentKillSwitchDangerZone: React.FC<AgentKillSwitchDangerZoneProps> = ({
return (
<section aria-labelledby="agent-danger-zone-heading" className="mt-6">
<h3 id="agent-danger-zone-heading" className="text-lg font-medium text-destructive">
Danger Zone
{t("agents.dangerZone.title")}
</h3>
<div className="mt-4 rounded-lg border border-destructive/40 bg-destructive/5 p-4">
<div className="flex flex-wrap items-start justify-between gap-4">
<div className="min-w-0 space-y-1 text-sm">
<p className="font-medium text-foreground">Kill switch</p>
<p className="font-medium text-foreground">{t("agents.dangerZone.killSwitch")}</p>
{killSwitch ? (
<>
<p className="text-muted-foreground">
Calls the configured webhook to stop this agent&apos;s upstream runtime. This can cause an outage for
everyone using the agent and cannot be undone from LiteLLM
</p>
<p className="text-muted-foreground">{t("agents.dangerZone.configuredDescription")}</p>
<p className="font-mono break-all text-foreground">
{killSwitch.method ?? "POST"} {killSwitch.url}
</p>
</>
) : (
<p className="text-muted-foreground">
Not configured. Add a kill switch webhook under Settings to enable this action
</p>
<p className="text-muted-foreground">{t("agents.dangerZone.notConfigured")}</p>
)}
</div>
{killSwitch && (
<Button type="button" variant="destructive" onClick={openConfirm} disabled={isFiring} aria-busy={isFiring}>
{isFiring && <UiLoadingSpinner className="size-4" />}
Fire Kill Switch
{t("agents.dangerZone.fire")}
</Button>
)}
</div>
{lastResult && (
<p className="mt-3 text-sm text-muted-foreground" role="status">
Last result: HTTP {lastResult.status_code}
{t("agents.dangerZone.lastResult", { status: lastResult.status_code })}
{lastResult.response_body ? ` ${lastResult.response_body}` : ""}
</p>
)}
@ -95,27 +92,32 @@ const AgentKillSwitchDangerZone: React.FC<AgentKillSwitchDangerZoneProps> = ({
<Dialog open={isConfirmOpen} onOpenChange={(open) => !open && !isFiring && setIsConfirmOpen(false)}>
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto">
<DialogHeader>
<DialogTitle>Fire kill switch for {agentName}?</DialogTitle>
<DialogTitle>{t("agents.dangerZone.confirm.title", { name: agentName })}</DialogTitle>
</DialogHeader>
<div className="space-y-4">
<Alert variant="error">
<CircleAlert />
<AlertTitle>This can cause an outage</AlertTitle>
<AlertTitle>{t("agents.dangerZone.confirm.outageTitle")}</AlertTitle>
<AlertDescription>
LiteLLM will call {killSwitch?.method ?? "POST"} {killSwitch?.url} immediately. Whatever that webhook
does to the agent is outside LiteLLM&apos;s control and cannot be reverted here
{t("agents.dangerZone.confirm.outageDescription", {
method: killSwitch?.method ?? "POST",
url: killSwitch?.url,
})}
</AlertDescription>
</Alert>
<div>
<p className="mb-2 text-base font-medium text-foreground">
Type <span className="font-semibold text-destructive">{agentName}</span> to confirm:
<Trans
i18nKey="agents.dangerZone.confirm.typeToConfirm"
components={{ name: <span className="font-semibold text-destructive">{agentName}</span> }}
/>
</p>
<InputGroup className="rounded-md">
<InputGroupAddon>
<CircleAlert className="size-3.5 text-destructive" />
</InputGroupAddon>
<InputGroupInput
aria-label="Confirm agent name"
aria-label={t("agents.dangerZone.confirm.inputLabel")}
value={confirmationInput}
onChange={(e) => setConfirmationInput(e.target.value)}
placeholder={agentName}
@ -126,7 +128,7 @@ const AgentKillSwitchDangerZone: React.FC<AgentKillSwitchDangerZoneProps> = ({
</div>
<DialogFooter>
<Button variant="outline" onClick={() => setIsConfirmOpen(false)} disabled={isFiring}>
Cancel
{t("agents.dangerZone.confirm.cancel")}
</Button>
<Button
variant="destructive"
@ -135,7 +137,7 @@ const AgentKillSwitchDangerZone: React.FC<AgentKillSwitchDangerZoneProps> = ({
aria-busy={isFiring}
>
{isFiring && <UiLoadingSpinner className="size-4" />}
{isFiring ? "Firing..." : "Fire Kill Switch"}
{isFiring ? t("agents.dangerZone.confirm.firing") : t("agents.dangerZone.fire")}
</Button>
</DialogFooter>
</DialogContent>

View file

@ -1,6 +1,7 @@
import { useTerminology } from "@/contexts/TerminologyContext";
import React, { useState, useEffect } from "react";
import { Info, Plus } from "lucide-react";
import { useTranslation } from "react-i18next";
import { getAgentsList, deleteAgentCall } from "@/components/networking";
import AddAgentForm from "./add_agent_form";
import { isAdminRole } from "@/utils/roles";
@ -33,6 +34,7 @@ interface AgentsResponse {
const AgentsPanel: React.FC<AgentsPanelProps> = ({ accessToken, userRole, teams }) => {
const term = useTerminology();
const { t } = useTranslation();
const [agentsList, setAgentsList] = useState<Agent[]>([]);
const [isAddModalVisible, setIsAddModalVisible] = useState(false);
const [isLoading, setIsLoading] = useState(true);
@ -122,11 +124,11 @@ const AgentsPanel: React.FC<AgentsPanelProps> = ({ accessToken, userRole, teams
setIsDeleting(true);
try {
await deleteAgentCall(accessToken, agentToDelete.id);
toast.success(`Agent "${agentToDelete.name}" deleted successfully`);
toast.success(t("agents.panel.toasts.deleted", { name: agentToDelete.name }));
await refetchAgents(healthCheckEnabled);
} catch (error) {
console.error("Error deleting agent:", error);
toast.fromError("Failed to delete agent");
toast.fromError(t("agents.panel.toasts.deleteFailed"));
} finally {
setIsDeleting(false);
setAgentToDelete(null);
@ -140,25 +142,18 @@ const AgentsPanel: React.FC<AgentsPanelProps> = ({ accessToken, userRole, teams
return (
<div className="w-full mx-auto flex-auto overflow-y-auto m-8 p-2">
<div className="flex flex-col gap-2 mb-4">
<h1 className="text-2xl font-bold">Agents</h1>
<p className="text-sm text-muted-foreground">
{term(
"List of A2A-spec agents that are available to be used in your organization. Go to AI Hub, to make agents public.",
)}
</p>
<h1 className="text-2xl font-bold">{t("agents.panel.title")}</h1>
<p className="text-sm text-muted-foreground">{term(t("agents.panel.description"))}</p>
<Alert className="mb-3">
<Info />
<AlertTitle>How do agents authenticate?</AlertTitle>
<AlertDescription>
Agents can authenticate with a virtual key or a trusted identity provider using JWT. Configure an identity
binding when adding or editing an agent. JWT authentication does not require a virtual key.
</AlertDescription>
<AlertTitle>{t("agents.panel.authAlert.title")}</AlertTitle>
<AlertDescription>{t("agents.panel.authAlert.description")}</AlertDescription>
</Alert>
{isAdmin && (
<div className="mt-2 flex items-center gap-4">
<Button onClick={handleAddAgent} disabled={!accessToken}>
<Plus />
Add New Agent
{t("agents.panel.addAgent")}
</Button>
</div>
)}
@ -201,15 +196,15 @@ const AgentsPanel: React.FC<AgentsPanelProps> = ({ accessToken, userRole, teams
>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>Delete Agent</AlertDialogTitle>
<AlertDialogTitle>{t("agents.panel.deleteDialog.title")}</AlertDialogTitle>
<AlertDialogDescription>
Are you sure you want to delete agent: {agentToDelete.name}? This action cannot be undone.
{t("agents.panel.deleteDialog.description", { name: agentToDelete.name })}
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>Cancel</AlertDialogCancel>
<AlertDialogCancel>{t("agents.panel.deleteDialog.cancel")}</AlertDialogCancel>
<Button variant="destructive" onClick={handleDeleteConfirm} disabled={isDeleting}>
Delete
{t("agents.panel.deleteDialog.confirm")}
</Button>
</AlertDialogFooter>
</AlertDialogContent>

View file

@ -3,6 +3,7 @@
import { SortingState } from "@tanstack/react-table";
import { Bot, CircleCheck, Search as SearchIcon, X } from "lucide-react";
import React, { useMemo, useState } from "react";
import { useTranslation } from "react-i18next";
import { Agent } from "@/components/agents/types";
import { DataTable } from "@/components/shared/DataTable";
@ -27,16 +28,17 @@ interface AgentsTableProps {
const DEFAULT_SORTING: SortingState = [{ id: "created_at", desc: true }];
function EmptyState({ isFiltered }: { isFiltered: boolean }) {
const { t } = useTranslation();
return (
<div className="flex flex-col items-center gap-1 py-6">
<div className="mb-1 flex size-10 items-center justify-center rounded-lg bg-muted">
<Bot className="size-5 text-muted-foreground" />
</div>
<div className="text-sm font-medium text-foreground">{isFiltered ? "No matching agents" : "No agents yet"}</div>
<div className="text-sm font-medium text-foreground">
{isFiltered ? t("agents.table.empty.filteredTitle") : t("agents.table.empty.title")}
</div>
<div className="text-sm text-muted-foreground">
{isFiltered
? "Adjust the search to see more agents."
: "Add an agent to make it available in your organization."}
{isFiltered ? t("agents.table.empty.filteredDescription") : t("agents.table.empty.description")}
</div>
</div>
);
@ -52,6 +54,7 @@ const AgentsTable: React.FC<AgentsTableProps> = ({
onAgentClick,
onDeleteClick,
}) => {
const { t } = useTranslation();
const [sorting, setSorting] = useState<SortingState>(DEFAULT_SORTING);
const [searchTerm, setSearchTerm] = useState("");
const filteredAgents = useMemo(
@ -79,7 +82,7 @@ const AgentsTable: React.FC<AgentsTableProps> = ({
sorting={sorting}
onSortingChange={setSorting}
isLoading={isLoading}
loadingMessage="Loading agents…"
loadingMessage={t("agents.table.loading")}
noDataMessage={<EmptyState isFiltered={agents.length > 0} />}
size="compact"
toolbar={() => (
@ -89,13 +92,17 @@ const AgentsTable: React.FC<AgentsTableProps> = ({
<SearchIcon className="size-4 text-muted-foreground" />
</InputGroupAddon>
<InputGroupInput
placeholder="Search agents by name, ID, or description..."
placeholder={t("agents.table.searchPlaceholder")}
value={searchTerm}
onChange={(e) => setSearchTerm(e.target.value)}
/>
{searchTerm && (
<InputGroupAddon align="inline-end">
<InputGroupButton size="icon-xs" aria-label="Clear search" onClick={() => setSearchTerm("")}>
<InputGroupButton
size="icon-xs"
aria-label={t("agents.table.clearSearch")}
onClick={() => setSearchTerm("")}
>
<X />
</InputGroupButton>
</InputGroupAddon>
@ -109,7 +116,7 @@ const AgentsTable: React.FC<AgentsTableProps> = ({
<CircleCheck
className={healthCheckEnabled ? "size-4 text-success" : "size-4 text-muted-foreground"}
/>
<span className="text-sm text-muted-foreground">Health Check</span>
<span className="text-sm text-muted-foreground">{t("agents.table.healthCheck.label")}</span>
<Switch
size="sm"
checked={healthCheckEnabled}
@ -119,7 +126,7 @@ const AgentsTable: React.FC<AgentsTableProps> = ({
</div>
}
/>
<TooltipContent>When enabled, only agents with reachable URLs are shown</TooltipContent>
<TooltipContent>{t("agents.table.healthCheck.hint")}</TooltipContent>
</Tooltip>
</TooltipProvider>
</div>

View file

@ -1,6 +1,7 @@
"use client";
import { ColumnDef } from "@tanstack/react-table";
import { useTranslation } from "react-i18next";
import { MoreHorizontal, Trash2 } from "lucide-react";
import { Agent } from "@/components/agents/types";
@ -14,6 +15,7 @@ import {
DropdownMenuItem,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import i18n from "@/i18n/i18n";
import { cn } from "@/lib/cva.config";
interface AgentRowActionsProps {
@ -22,10 +24,11 @@ interface AgentRowActionsProps {
}
function AgentRowActions({ agent, onDeleteClick }: AgentRowActionsProps) {
const { t } = useTranslation();
return (
<DropdownMenu>
<DropdownMenuTrigger
aria-label="Open agent actions"
aria-label={t("agents.table.actions.open")}
data-testid={`agent-actions-${agent.agent_id}`}
className={cn(buttonVariants({ variant: "ghost", size: "icon-sm" }), "text-muted-foreground")}
>
@ -38,7 +41,7 @@ function AgentRowActions({ agent, onDeleteClick }: AgentRowActionsProps) {
onClick={() => onDeleteClick(agent.agent_id, agent.agent_name)}
>
<Trash2 />
Delete
{t("agents.table.actions.delete")}
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
@ -59,8 +62,8 @@ export const getAgentsTableColumns = ({
{
id: "agent_name",
accessorKey: "agent_name",
meta: { title: "Agent Name" },
header: ({ column }) => <DataTableSortHeader column={column} title="Agent Name" />,
meta: { title: i18n.t("agents.table.columns.name") },
header: ({ column }) => <DataTableSortHeader column={column} title={i18n.t("agents.table.columns.name")} />,
size: 200,
enableSorting: true,
cell: ({ row }) => {
@ -75,8 +78,8 @@ export const getAgentsTableColumns = ({
{
id: "agent_id",
accessorKey: "agent_id",
meta: { title: "Agent ID" },
header: ({ column }) => <DataTableSortHeader column={column} title="Agent ID" />,
meta: { title: i18n.t("agents.table.columns.id") },
header: ({ column }) => <DataTableSortHeader column={column} title={i18n.t("agents.table.columns.id")} />,
size: 200,
enableSorting: true,
cell: ({ row }) => (
@ -90,22 +93,22 @@ export const getAgentsTableColumns = ({
{
id: "spend",
accessorKey: "spend",
meta: { title: "Spend (USD)", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="Spend (USD)" />,
meta: { title: i18n.t("agents.table.columns.spend"), numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title={i18n.t("agents.table.columns.spend")} />,
size: 130,
enableSorting: true,
cell: ({ row }) => <MoneyCell value={row.original.spend} decimals={4} />,
},
{
id: "model",
meta: { title: "Model" },
header: "Model",
meta: { title: i18n.t("agents.table.columns.model") },
header: i18n.t("agents.table.columns.model"),
size: 170,
enableSorting: false,
cell: ({ row }) => {
const model = row.original.litellm_params?.model;
if (!model) {
return <span className="text-muted-foreground">N/A</span>;
return <span className="text-muted-foreground">{i18n.t("agents.table.noModel")}</span>;
}
return (
<Badge variant="outline" className="max-w-40 font-normal">
@ -122,25 +125,26 @@ export const getAgentsTableColumns = ({
const timestamp = agent.created_at ? new Date(agent.created_at).getTime() : 0;
return Number.isNaN(timestamp) ? 0 : timestamp;
},
meta: { title: "Created" },
header: ({ column }) => <DataTableSortHeader column={column} title="Created" />,
meta: { title: i18n.t("agents.table.columns.created") },
header: ({ column }) => <DataTableSortHeader column={column} title={i18n.t("agents.table.columns.created")} />,
size: 150,
enableSorting: true,
cell: ({ row }) => <DateCell value={row.original.created_at} precision="date" />,
},
{
id: "status",
meta: { title: "Status" },
header: "Status",
meta: { title: i18n.t("agents.table.columns.status") },
header: i18n.t("agents.table.columns.status"),
size: 130,
enableSorting: false,
cell: ({ row }) => {
const hasKeys = (row.original.keys?.length ?? 0) > 0;
if (row.original.jwt_auth_configured) return <StatusBadge tone="success" label="JWT configured" />;
if (row.original.jwt_auth_configured)
return <StatusBadge tone="success" label={i18n.t("agents.table.status.jwtConfigured")} />;
return hasKeys ? (
<StatusBadge tone="success" label="Active" />
<StatusBadge tone="success" label={i18n.t("agents.table.status.active")} />
) : (
<StatusBadge tone="warning" label="Needs Setup" />
<StatusBadge tone="warning" label={i18n.t("agents.table.status.needsSetup")} />
);
},
},
@ -149,7 +153,7 @@ export const getAgentsTableColumns = ({
{
id: "actions",
meta: { className: "text-right", headerClassName: "text-right" },
header: () => <span className="sr-only">Actions</span>,
header: () => <span className="sr-only">{i18n.t("agents.table.columns.actions")}</span>,
size: 64,
enableSorting: false,
enableHiding: false,

View file

@ -1,5 +1,6 @@
import React from "react";
import { useFieldArray, useFormContext, useWatch } from "react-hook-form";
import { useTranslation } from "react-i18next";
import { Plus, Trash2 } from "lucide-react";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
@ -7,6 +8,7 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@
import { Textarea } from "@/components/ui/textarea";
import { Field, FieldTitle } from "@/components/ui/field";
import { PasswordInput } from "@/components/shared/PasswordInput";
import i18n from "@/i18n/i18n";
import { AgentFormField, AgentFormValues, labelWithHint } from "./AgentFormKit";
import { KILL_SWITCH_AUTH_TYPES, KILL_SWITCH_METHODS, validateKillSwitchBody } from "./kill_switch_config";
@ -21,6 +23,7 @@ const KeyValueFieldArray = ({
keyPlaceholder: string;
valuePlaceholder: string;
}) => {
const { t } = useTranslation();
const { control } = useFormContext<AgentFormValues>();
const { fields, append, remove } = useFieldArray({ control, name });
@ -28,7 +31,10 @@ const KeyValueFieldArray = ({
<div className="flex flex-col gap-2">
{fields.map((item, index) => (
<div key={item.id} className="flex items-start gap-2">
<AgentFormField name={`${name}.${index}.key`} rules={{ required: "Name required" }}>
<AgentFormField
name={`${name}.${index}.key`}
rules={{ required: t("agents.killSwitchFields.errors.nameRequired") }}
>
{({ value, onChange, ref, ...control }) => (
<Input
{...control}
@ -56,7 +62,7 @@ const KeyValueFieldArray = ({
type="button"
variant="ghost"
size="icon"
aria-label={`Remove ${addLabel.replace(/^Add /, "").toLowerCase()}`}
aria-label={t("agents.killSwitchFields.removeItem", { item: addLabel.replace(/^Add /, "").toLowerCase() })}
className="text-destructive hover:text-destructive/80"
onClick={() => remove(index)}
>
@ -109,24 +115,50 @@ const TextField = ({
);
const KillSwitchAuthFields = () => {
const { t } = useTranslation();
const { control } = useFormContext<AgentFormValues>();
const authType = useWatch({ control, name: "kill_switch.auth_type" });
switch (authType) {
case "bearer":
return <TextField name="kill_switch.auth_token" label="Bearer token" required="Token required" secret />;
return (
<TextField
name="kill_switch.auth_token"
label={t("agents.killSwitchFields.auth.bearerToken")}
required={t("agents.killSwitchFields.errors.tokenRequired")}
secret
/>
);
case "api_key":
return (
<>
<TextField name="kill_switch.auth_header_name" label="Header name" placeholder="X-API-Key" />
<TextField name="kill_switch.auth_api_key" label="API key" required="API key required" secret />
<TextField
name="kill_switch.auth_header_name"
label={t("agents.killSwitchFields.auth.headerName")}
placeholder="X-API-Key"
/>
<TextField
name="kill_switch.auth_api_key"
label={t("agents.killSwitchFields.auth.apiKey")}
required={t("agents.killSwitchFields.errors.apiKeyRequired")}
secret
/>
</>
);
case "basic":
return (
<>
<TextField name="kill_switch.auth_username" label="Username" required="Username required" />
<TextField name="kill_switch.auth_password" label="Password" required="Password required" secret />
<TextField
name="kill_switch.auth_username"
label={t("agents.killSwitchFields.auth.username")}
required={t("agents.killSwitchFields.errors.usernameRequired")}
/>
<TextField
name="kill_switch.auth_password"
label={t("agents.killSwitchFields.auth.password")}
required={t("agents.killSwitchFields.errors.passwordRequired")}
secret
/>
</>
);
default:
@ -139,13 +171,13 @@ const KillSwitchFormFields = () => (
<TextField
name="kill_switch.url"
label={labelWithHint(
"Webhook URL",
"Absolute http(s) URL LiteLLM calls when the kill switch is triggered. Leave empty to remove the kill switch.",
i18n.t("agents.killSwitchFields.webhookUrl.label"),
i18n.t("agents.killSwitchFields.webhookUrl.hint"),
)}
placeholder="https://example.com/hooks/kill-agent"
/>
<AgentFormField name="kill_switch.method" label="Method">
<AgentFormField name="kill_switch.method" label={i18n.t("agents.killSwitchFields.method")}>
{({ value, onChange, ref: _ref, ...control }) => (
<Select value={typeof value === "string" ? value : "POST"} onValueChange={onChange}>
<SelectTrigger {...control} className="w-full">
@ -163,28 +195,28 @@ const KillSwitchFormFields = () => (
</AgentFormField>
<Field>
<FieldTitle>Headers</FieldTitle>
<FieldTitle>{i18n.t("agents.killSwitchFields.headers.title")}</FieldTitle>
<KeyValueFieldArray
name="kill_switch.headers"
addLabel="Add Header"
keyPlaceholder="Header name"
valuePlaceholder="Header value"
addLabel={i18n.t("agents.killSwitchFields.headers.add")}
keyPlaceholder={i18n.t("agents.killSwitchFields.headers.keyPlaceholder")}
valuePlaceholder={i18n.t("agents.killSwitchFields.headers.valuePlaceholder")}
/>
</Field>
<Field>
<FieldTitle>Query Parameters</FieldTitle>
<FieldTitle>{i18n.t("agents.killSwitchFields.queryParams.title")}</FieldTitle>
<KeyValueFieldArray
name="kill_switch.query_params"
addLabel="Add Query Parameter"
keyPlaceholder="Parameter name"
valuePlaceholder="Parameter value"
addLabel={i18n.t("agents.killSwitchFields.queryParams.add")}
keyPlaceholder={i18n.t("agents.killSwitchFields.queryParams.keyPlaceholder")}
valuePlaceholder={i18n.t("agents.killSwitchFields.queryParams.valuePlaceholder")}
/>
</Field>
<AgentFormField
name="kill_switch.body"
label={labelWithHint("JSON Body", "Optional JSON object sent as the request body")}
label={labelWithHint(i18n.t("agents.killSwitchFields.body.label"), i18n.t("agents.killSwitchFields.body.hint"))}
rules={{ validate: (value) => validateKillSwitchBody(typeof value === "string" ? value : "") }}
>
{({ value, onChange, ref, ...control }) => (
@ -199,7 +231,7 @@ const KillSwitchFormFields = () => (
)}
</AgentFormField>
<AgentFormField name="kill_switch.auth_type" label="Authentication">
<AgentFormField name="kill_switch.auth_type" label={i18n.t("agents.killSwitchFields.authentication")}>
{({ value, onChange, ref: _ref, ...control }) => (
<Select value={typeof value === "string" ? value : "none"} onValueChange={onChange}>
<SelectTrigger {...control} className="w-full">
@ -208,7 +240,7 @@ const KillSwitchFormFields = () => (
<SelectContent>
{KILL_SWITCH_AUTH_TYPES.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
{i18n.t(option.labelKey)}
</SelectItem>
))}
</SelectContent>

View file

@ -5,6 +5,7 @@ import { FormProvider, useForm, useWatch } from "react-hook-form";
import { toast } from "@/lib/toast";
import { Logo } from "@/components/molecules/logo/Logo";
import { Bot, Check, CircleCheck, Key, LayoutGrid } from "lucide-react";
import { Trans, useTranslation } from "react-i18next";
import CreatedKeyDisplay from "@/components/shared/CreatedKeyDisplay";
import { Badge } from "@/components/ui/badge";
import { StatusBadge } from "@/components/shared/table_cells/status_badge";
@ -55,10 +56,17 @@ import MCPToolPermissions from "@/components/mcp_server_management/MCPToolPermis
import AccessGroupSelector from "@/components/common_components/AccessGroupSelector";
import GuardrailSelector from "@/components/guardrails/GuardrailSelector";
import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog";
import i18n from "@/i18n/i18n";
const CUSTOM_AGENT_TYPE = "custom";
const STEP_TITLES = ["Configure", "Entitlements", "Governance", "Agent Management", "Ready"] as const;
const STEP_TITLES = [
"agents.addForm.steps.configure",
"agents.addForm.steps.entitlements",
"agents.addForm.steps.governance",
"agents.addForm.steps.agentManagement",
"agents.addForm.steps.ready",
] as const;
const stepMarkerClass = (index: number, current: number): string => {
if (index < current) return "border-primary text-primary";
@ -73,11 +81,12 @@ const stepTitleClass = (index: number, current: number): string => {
};
const AgentTypeLabel: React.FC<{ agentType: string; info: AgentCreateInfo | undefined }> = ({ agentType, info }) => {
const { t } = useTranslation();
if (agentType === CUSTOM_AGENT_TYPE) {
return (
<span className="flex items-center gap-2">
<LayoutGrid className="size-4 text-warning" />
<span>Custom / Other</span>
<span>{t("agents.addForm.agentType.custom")}</span>
</span>
);
}
@ -91,7 +100,7 @@ const AgentTypeLabel: React.FC<{ agentType: string; info: AgentCreateInfo | unde
};
const StepProgress: React.FC<{ current: number }> = ({ current }) => (
<ol aria-label="Agent creation steps" className="mb-8 flex items-center">
<ol aria-label={i18n.t("agents.addForm.steps.ariaLabel")} className="mb-8 flex items-center">
{STEP_TITLES.map((title, index) => (
<li
key={title}
@ -104,7 +113,7 @@ const StepProgress: React.FC<{ current: number }> = ({ current }) => (
>
{index < current ? <Check className="size-3.5" /> : index + 1}
</span>
<span className={`text-xs whitespace-nowrap ${stepTitleClass(index, current)}`}>{title}</span>
<span className={`text-xs whitespace-nowrap ${stepTitleClass(index, current)}`}>{i18n.t(title)}</span>
{index < STEP_TITLES.length - 1 && <span aria-hidden="true" className="mx-2 h-px flex-1 bg-border" />}
</li>
))}
@ -132,6 +141,7 @@ interface AddAgentFormProps {
}
const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessToken, onSuccess, teams }) => {
const { t } = useTranslation();
const { userId, userRole } = useAuthorized();
const form = useForm<AgentFormValues>({ defaultValues: buildInitialValues("a2a") });
const panels = useCollapsiblePanels([AGENT_FORM_CONFIG.basic.key]);
@ -339,12 +349,12 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
const handleCreateAgent = async () => {
if (!accessToken) {
toast.error("No access token available");
toast.error(t("agents.addForm.toasts.noAccessToken"));
return;
}
if (keyAssignOption === "existing_key" && !selectedExistingKey) {
toast.error("Please select an existing key to assign");
toast.error(t("agents.addForm.toasts.selectExistingKey"));
return;
}
@ -358,7 +368,7 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
const values = form.getValues();
const built = buildAgentData(values);
if (!built) {
toast.error("Failed to build agent data");
toast.error(t("agents.addForm.toasts.buildFailed"));
setIsSubmitting(false);
return;
}
@ -434,7 +444,11 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
} catch (error) {
console.error("Error creating agent:", error);
const errorMessage = error instanceof Error ? error.message : String(error);
toast.error(errorMessage ? `Failed to create agent: ${errorMessage}` : "Failed to create agent");
toast.error(
errorMessage
? t("agents.addForm.toasts.createFailedWithError", { error: errorMessage })
: t("agents.addForm.toasts.createFailed"),
);
} finally {
setIsSubmitting(false);
}
@ -461,17 +475,14 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
const renderEntitlementsStep = () => (
<div className="space-y-4">
<p className="text-sm text-muted-foreground">
Configure which models, agents, and MCP tools this agent is allowed to use. Leave fields empty to allow all
(subject to key/team permissions).
</p>
<p className="text-sm text-muted-foreground">{t("agents.addForm.entitlements.description")}</p>
<FieldGroup>
<AgentFormField
name="entitlement_models"
label={labelWithHint(
"Allowed Models",
"Restrict which models this agent can call. Leave empty to allow all.",
t("agents.addForm.entitlements.models.label"),
t("agents.addForm.entitlements.models.hint"),
)}
>
{({ id, value, onChange }) => (
@ -479,7 +490,11 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
id={id}
value={Array.isArray(value) ? (value as string[]) : []}
onValueChange={onChange}
placeholder={loadingModels ? "Loading models..." : "Select models (leave empty for all)"}
placeholder={
loadingModels
? t("agents.addForm.entitlements.models.loading")
: t("agents.addForm.entitlements.models.placeholder")
}
options={availableModels.map((m) => ({ label: getModelDisplayName(m), value: m }))}
/>
)}
@ -488,8 +503,8 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
<AgentFormField
name="entitlement_agents"
label={labelWithHint(
"Allowed Agents (Sub-Agents)",
"Restrict which other agents this agent can invoke as sub-agents. Leave empty to allow all.",
t("agents.addForm.entitlements.agents.label"),
t("agents.addForm.entitlements.agents.hint"),
)}
>
{({ id, value, onChange }) => (
@ -497,7 +512,11 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
id={id}
value={Array.isArray(value) ? (value as string[]) : []}
onValueChange={onChange}
placeholder={loadingAgents ? "Loading agents..." : "Select agents (leave empty for all)"}
placeholder={
loadingAgents
? t("agents.addForm.entitlements.agents.loading")
: t("agents.addForm.entitlements.agents.placeholder")
}
options={availableAgents.map((a) => ({ label: a.agent_name, value: a.agent_id }))}
/>
)}
@ -506,15 +525,15 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
<AgentFormField
name="access_group_ids"
label={labelWithHint(
"Access Groups",
"Attach access groups to this agent. Attached groups cap which models, MCP servers, and agents the agent can reach, on top of its key and team permissions. Leave empty to apply no extra cap.",
t("agents.addForm.entitlements.accessGroups.label"),
t("agents.addForm.entitlements.accessGroups.hint"),
)}
>
{({ value, onChange }) => (
<AccessGroupSelector
value={Array.isArray(value) ? (value as string[]) : []}
onChange={onChange}
placeholder="Select access groups (optional)"
placeholder={t("agents.addForm.entitlements.accessGroups.placeholder")}
/>
)}
</AgentFormField>
@ -524,8 +543,8 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
<AgentFormField
name="allowed_mcp_servers_and_groups"
label={labelWithHint(
"Allowed MCP Servers",
"Select which MCP servers or access groups this agent can access",
t("agents.addForm.entitlements.mcpServers.label"),
t("agents.addForm.entitlements.mcpServers.hint"),
)}
>
{({ value, onChange }) => (
@ -536,7 +555,7 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
accessGroups: (value as McpServerSelection | undefined)?.accessGroups ?? [],
}}
accessToken={accessToken ?? ""}
placeholder="Select MCP servers or access groups (optional)"
placeholder={t("agents.addForm.entitlements.mcpServers.placeholder")}
/>
)}
</AgentFormField>
@ -574,15 +593,15 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
const renderObservabilityStep = () => (
<div className="space-y-6">
<div>
<h4 className="mb-3 text-sm font-medium text-foreground">Tracing</h4>
<h4 className="mb-3 text-sm font-medium text-foreground">{t("agents.addForm.governance.tracing.title")}</h4>
<div className="space-y-4">
<div className="flex items-center justify-between">
<div>
<span className="text-sm font-medium text-foreground">
Require x-litellm-trace-id on calls TO this agent
{t("agents.addForm.governance.tracing.inbound.label", { header: "x-litellm-trace-id" })}
</span>
<p className="mt-1 text-xs text-muted-foreground">
Only accept this agent being invoked with a trace-id (e.g. when used as a sub-agent).
{t("agents.addForm.governance.tracing.inbound.description")}
</p>
</div>
<Switch checked={requireTraceIdInbound} onCheckedChange={setRequireTraceIdInbound} />
@ -591,10 +610,10 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
<div className="flex items-center justify-between">
<div>
<span className="text-sm font-medium text-foreground">
Require x-litellm-trace-id on calls BY this agent
{t("agents.addForm.governance.tracing.outbound.label", { header: "x-litellm-trace-id" })}
</span>
<p className="mt-1 text-xs text-muted-foreground">
Requires LLM/MCP calls made by this agent to include x-litellm-trace-id for session tracking.
{t("agents.addForm.governance.tracing.outbound.description", { header: "x-litellm-trace-id" })}
</p>
</div>
<Switch
@ -614,24 +633,27 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
<Separator />
<div>
<h4 className="mb-3 text-sm font-medium text-foreground">Budgets &amp; Rate Limits</h4>
<h4 className="mb-3 text-sm font-medium text-foreground">{t("agents.addForm.governance.budgets.title")}</h4>
<div className="space-y-4">
{!requireTraceIdOutbound && (
<div className="rounded-lg border border-warning/20 bg-warning/10 p-3 text-sm text-warning">
Enable &quot;Require x-litellm-trace-id on calls BY this agent&quot; in Tracing to configure budgets and
rate limits.
{t("agents.addForm.governance.budgets.requiresOutbound", { header: "x-litellm-trace-id" })}
</div>
)}
<div className="text-sm font-medium text-foreground">Session Budgets</div>
<div className="text-sm font-medium text-foreground">
{t("agents.addForm.governance.budgets.session.title")}
</div>
<div className="grid grid-cols-2 gap-4">
<Field className="gap-1">
<FieldLabel htmlFor="agent-max-iterations">Max Iterations</FieldLabel>
<FieldLabel htmlFor="agent-max-iterations">
{t("agents.addForm.governance.budgets.session.maxIterations.label")}
</FieldLabel>
<Input
id="agent-max-iterations"
type="number"
step="any"
placeholder="e.g. 25"
placeholder={t("agents.addForm.governance.budgets.session.maxIterations.placeholder")}
disabled={!requireTraceIdOutbound}
value={maxIterations ?? ""}
onChange={(event) =>
@ -639,15 +661,19 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
}
onBlur={() => setMaxIterations((current) => (current !== null && current < 1 ? 1 : current))}
/>
<p className="mt-1 text-xs text-muted-foreground">Hard cap on LLM calls per session</p>
<p className="mt-1 text-xs text-muted-foreground">
{t("agents.addForm.governance.budgets.session.maxIterations.hint")}
</p>
</Field>
<Field className="gap-1">
<FieldLabel htmlFor="agent-max-budget-per-session">Max Budget Per Session ($)</FieldLabel>
<FieldLabel htmlFor="agent-max-budget-per-session">
{t("agents.addForm.governance.budgets.session.maxBudget.label")}
</FieldLabel>
<Input
id="agent-max-budget-per-session"
type="number"
step="any"
placeholder="e.g. 5.00"
placeholder={t("agents.addForm.governance.budgets.session.maxBudget.placeholder")}
disabled={!requireTraceIdOutbound}
value={maxBudgetPerSession ?? ""}
onChange={(event) =>
@ -657,26 +683,48 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
setMaxBudgetPerSession((current) => (current !== null && current < 0.01 ? 0.01 : current))
}
/>
<p className="mt-1 text-xs text-muted-foreground">Max spend per trace before returning 429</p>
<p className="mt-1 text-xs text-muted-foreground">
{t("agents.addForm.governance.budgets.session.maxBudget.hint")}
</p>
</Field>
</div>
<Separator className="my-2" />
<div className="text-sm font-medium text-foreground">Agent Rate Limits</div>
<p className="text-xs text-muted-foreground">Global rate limits applied across all callers of this agent.</p>
<div className="text-sm font-medium text-foreground">
{t("agents.addForm.governance.rateLimits.agent.title")}
</div>
<p className="text-xs text-muted-foreground">{t("agents.addForm.governance.rateLimits.agent.description")}</p>
<div className="grid grid-cols-2 gap-4">
{rateLimitField("tpm_limit", "TPM Limit", "e.g. 100000")}
{rateLimitField("rpm_limit", "RPM Limit", "e.g. 100")}
{rateLimitField(
"tpm_limit",
t("agents.addForm.governance.rateLimits.agent.tpm.label"),
t("agents.addForm.governance.rateLimits.agent.tpm.placeholder"),
)}
{rateLimitField(
"rpm_limit",
t("agents.addForm.governance.rateLimits.agent.rpm.label"),
t("agents.addForm.governance.rateLimits.agent.rpm.placeholder"),
)}
</div>
<div className="mt-4 text-sm font-medium text-foreground">Per-Session Rate Limits</div>
<div className="mt-4 text-sm font-medium text-foreground">
{t("agents.addForm.governance.rateLimits.session.title")}
</div>
<p className="text-xs text-muted-foreground">
Rate limits per session (x-litellm-trace-id). Each session gets its own counters.
{t("agents.addForm.governance.rateLimits.session.description", { header: "x-litellm-trace-id" })}
</p>
<div className="grid grid-cols-2 gap-4">
{rateLimitField("session_tpm_limit", "Session TPM Limit", "e.g. 10000")}
{rateLimitField("session_rpm_limit", "Session RPM Limit", "e.g. 20")}
{rateLimitField(
"session_tpm_limit",
t("agents.addForm.governance.rateLimits.session.tpm.label"),
t("agents.addForm.governance.rateLimits.session.tpm.placeholder"),
)}
{rateLimitField(
"session_rpm_limit",
t("agents.addForm.governance.rateLimits.session.rpm.label"),
t("agents.addForm.governance.rateLimits.session.rpm.placeholder"),
)}
</div>
</div>
</div>
@ -684,10 +732,8 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
<Separator />
<div>
<h4 className="mb-3 text-sm font-medium text-foreground">Guardrails</h4>
<p className="mb-3 text-xs text-muted-foreground">
Apply guardrails to this agent. Selected guardrails will run on all calls made by this agent.
</p>
<h4 className="mb-3 text-sm font-medium text-foreground">{t("agents.addForm.governance.guardrails.title")}</h4>
<p className="mb-3 text-xs text-muted-foreground">{t("agents.addForm.governance.guardrails.description")}</p>
<AgentFormField name="guardrails">
{({ value, onChange }) => (
<GuardrailSelector
@ -763,7 +809,7 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
<>
<Field className="gap-1">
<FieldLabel htmlFor="agent-type">
{labelWithHint("Agent Type", "Select the type of agent you want to create")}
{labelWithHint(t("agents.addForm.configure.agentType.label"), t("agents.addForm.configure.agentType.hint"))}
</FieldLabel>
<Select value={agentType} onValueChange={(value) => value !== null && handleAgentTypeChange(value)}>
<SelectTrigger id="agent-type" className="h-10 w-full">
@ -785,18 +831,22 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
))}
<SelectSeparator />
<div className="mb-1 px-2 text-xs font-medium tracking-wide text-muted-foreground uppercase">
Not listed?
{t("agents.addForm.configure.agentType.notListed")}
</div>
<SelectItem value={CUSTOM_AGENT_TYPE} className="focus:bg-warning/10">
<span className="flex items-center gap-3">
<LayoutGrid className="size-4.5 shrink-0 text-warning" />
<span className="block">
<span className="flex items-center gap-2">
<span className="font-medium text-warning">Custom / Other</span>
<StatusBadge tone="warning" label="GENERIC" className="h-4 px-1 text-[10px]" />
<span className="font-medium text-warning">{t("agents.addForm.agentType.custom")}</span>
<StatusBadge
tone="warning"
label={t("agents.addForm.configure.agentType.genericBadge")}
className="h-4 px-1 text-[10px]"
/>
</span>
<span className="block text-xs whitespace-normal text-warning">
For outbound agents using an identity provider or virtual key
{t("agents.addForm.configure.agentType.customDescription")}
</span>
</span>
</span>
@ -810,24 +860,28 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
<div className="mt-4">
{agentType === CUSTOM_AGENT_TYPE ? (
<FieldGroup>
<AgentFormField name="agent_name" label="Agent Name" rules={{ required: "Please enter an agent name" }}>
<AgentFormField
name="agent_name"
label={t("agents.addForm.configure.custom.agentName.label")}
rules={{ required: t("agents.addForm.configure.custom.agentName.required") }}
>
{({ value, onChange, ref, ...control }) => (
<Input
{...control}
ref={ref}
placeholder="e.g. my-custom-agent"
placeholder={t("agents.addForm.configure.custom.agentName.placeholder")}
value={typeof value === "string" ? value : ""}
onChange={onChange}
/>
)}
</AgentFormField>
<AgentFormField name="description" label="Description">
<AgentFormField name="description" label={t("agents.addForm.configure.custom.description.label")}>
{({ value, onChange, ref, ...control }) => (
<Textarea
{...control}
ref={ref}
rows={3}
placeholder="Describe what this agent does…"
placeholder={t("agents.addForm.configure.custom.description.placeholder")}
value={typeof value === "string" ? value : ""}
onChange={onChange}
/>
@ -842,7 +896,7 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
{selectedAgentTypeInfo.credential_fields.length > 0 && (
<div className="mt-4 rounded-lg border border-border p-4">
<h4 className="mb-3 text-sm font-medium text-foreground">
{selectedAgentTypeInfo.agent_type_display_name} Settings
{t("agents.addForm.configure.typeSettings", { type: selectedAgentTypeInfo.agent_type_display_name })}
</h4>
<FieldGroup>
{selectedAgentTypeInfo.credential_fields.map((field) => (
@ -851,7 +905,11 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
name={field.key}
label={field.tooltip ? labelWithHint(field.label, field.tooltip) : field.label}
defaultValue={field.default_value ?? undefined}
rules={field.required ? { required: `Please enter ${field.label}` } : undefined}
rules={
field.required
? { required: t("agents.addForm.configure.enterField", { field: field.label }) }
: undefined
}
>
{({ value, onChange, ref, ...control }) =>
field.field_type === "password" ? (
@ -901,7 +959,7 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
);
const renderAssignKeyStep = () => {
const agentName = form.getValues("agent_name") || "your-agent";
const agentName = form.getValues("agent_name") || t("agents.addForm.assignKey.agentNameFallback");
return (
<div>
{/* Agent name chip */}
@ -914,10 +972,7 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
<AgentFormField
name="team_id"
label={labelWithHint(
"Assign to Team",
"Optionally select a team for the virtual key. The agent identity and its permissions are managed separately.",
)}
label={labelWithHint(t("agents.addForm.assignKey.team.label"), t("agents.addForm.assignKey.team.hint"))}
>
{({ value, onChange }) => (
<TeamDropdown value={typeof value === "string" ? value : undefined} onChange={onChange} />
@ -927,9 +982,7 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
<Separator className="my-4" />
{form.getValues("identity_provider") === "microsoft_entra" && (
<p className="mb-4 text-sm text-muted-foreground">
This agent will authenticate with Microsoft Entra ID. You can skip virtual key creation.
</p>
<p className="mb-4 text-sm text-muted-foreground">{t("agents.addForm.assignKey.entraNotice")}</p>
)}
<RadioGroup
value={keyAssignOption}
@ -947,29 +1000,33 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
>
<div className="flex items-start justify-between">
<div className="flex flex-1 items-start gap-3">
<RadioGroupItem value="create_new" aria-label="Create a new key for this agent" />
<RadioGroupItem value="create_new" aria-label={t("agents.addForm.assignKey.createNew.title")} />
<div className="flex-1">
<div className="flex items-center gap-2">
<Key className="size-4 text-info" />
<span className="font-medium text-foreground">Create a new key for this agent</span>
<span className="font-medium text-foreground">{t("agents.addForm.assignKey.createNew.title")}</span>
</div>
<p className="mt-1 text-sm text-muted-foreground">A dedicated key scoped to this agent.</p>
<p className="mt-1 text-sm text-muted-foreground">
{t("agents.addForm.assignKey.createNew.description")}
</p>
{keyAssignOption === "create_new" && (
<div className="mt-3 space-y-3" onClick={(e) => e.stopPropagation()}>
<Field className="gap-1">
<FieldLabel htmlFor="agent-new-key-name">Key Name</FieldLabel>
<FieldLabel htmlFor="agent-new-key-name">
{t("agents.addForm.assignKey.createNew.keyName.label")}
</FieldLabel>
<Input
id="agent-new-key-name"
value={newKeyName}
onChange={(e) => setNewKeyName(e.target.value)}
placeholder="e.g. my-agent-key"
placeholder={t("agents.addForm.assignKey.createNew.keyName.placeholder")}
/>
</Field>
</div>
)}
</div>
</div>
<StatusBadge tone="success" label="Recommended" />
<StatusBadge tone="success" label={t("agents.addForm.assignKey.createNew.recommended")} />
</div>
</div>
@ -983,18 +1040,24 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
onClick={() => setKeyAssignOption("existing_key")}
>
<div className="flex items-start gap-3">
<RadioGroupItem value="existing_key" aria-label="Assign an existing key" />
<RadioGroupItem value="existing_key" aria-label={t("agents.addForm.assignKey.existing.title")} />
<div className="flex-1">
<div className="flex items-center gap-2">
<Key className="size-4 text-muted-foreground" />
<span className="font-medium text-foreground">Assign an existing key</span>
<span className="font-medium text-foreground">{t("agents.addForm.assignKey.existing.title")}</span>
</div>
<p className="mt-1 text-sm text-muted-foreground">Re-assign a key you already have to this agent.</p>
<p className="mt-1 text-sm text-muted-foreground">
{t("agents.addForm.assignKey.existing.description")}
</p>
{keyAssignOption === "existing_key" && (
<div className="mt-3" onClick={(e) => e.stopPropagation()}>
<SearchSelect
inputId="agent-existing-key"
placeholder={loadingKeys ? "Loading keys…" : "Search by key name…"}
placeholder={
loadingKeys
? t("agents.addForm.assignKey.existing.loading")
: t("agents.addForm.assignKey.existing.placeholder")
}
value={selectedExistingKey}
onValueChange={setSelectedExistingKey}
options={existingKeys.map((k) => ({
@ -1016,8 +1079,8 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
onClick={() => setKeyAssignOption("skip")}
>
{form.getValues("identity_provider") === "microsoft_entra"
? "Use Entra JWT authentication"
: "Skip for now, I’ll assign a key later"}
? t("agents.addForm.assignKey.skip.entra")
: t("agents.addForm.assignKey.skip.later")}
</button>
</div>
</div>
@ -1027,7 +1090,7 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
const renderReadyStep = () => (
<div className="py-6 text-center">
<CircleCheck className="mb-4 size-12 text-success" />
<h3 className="mb-2 text-xl font-semibold text-foreground">Agent Created!</h3>
<h3 className="mb-2 text-xl font-semibold text-foreground">{t("agents.addForm.ready.title")}</h3>
<div className="mb-4 flex justify-center">
<Badge className="h-auto gap-1.5 bg-purple-100 px-3 py-1 text-sm text-purple-700 dark:bg-purple-950 dark:text-purple-300">
<Bot className="size-3.5" />
@ -1041,14 +1104,17 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
)}
{assignedKeyAlias && (
<p className="mt-2 text-sm text-muted-foreground">
Key <span className="font-medium">{assignedKeyAlias}</span> has been assigned to this agent.
<Trans
i18nKey="agents.addForm.ready.keyAssigned"
components={{ alias: <span className="font-medium">{assignedKeyAlias}</span> }}
/>
</p>
)}
{!createdKeyValue && !assignedKeyAlias && keyAssignOption === "skip" && (
<p className="mt-2 text-sm text-muted-foreground">
{form.getValues("identity_provider") === "microsoft_entra"
? "Microsoft Entra ID is configured. Send an authenticated agent request to verify the connection."
: "No key assigned. You can create one from the Virtual Keys page."}
? t("agents.addForm.ready.entraConfigured")
: t("agents.addForm.ready.noKey")}
</p>
)}
</div>
@ -1060,9 +1126,15 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
<DialogHeader>
<div className="flex items-center space-x-3 border-b border-border pb-4">
{selectedLogo && currentStep < 1 && (
<Logo src={selectedLogo} label="Agent" className="h-6 w-6 object-contain" />
<Logo
src={selectedLogo}
label={t("agents.addForm.dialog.logoLabel")}
className="h-6 w-6 object-contain"
/>
)}
<DialogTitle className="text-xl font-semibold text-foreground">Add New Agent</DialogTitle>
<DialogTitle className="text-xl font-semibold text-foreground">
{t("agents.addForm.dialog.title")}
</DialogTitle>
</div>
</DialogHeader>
<TooltipProvider>
@ -1083,24 +1155,24 @@ const AddAgentForm: React.FC<AddAgentFormProps> = ({ visible, onClose, accessTok
<div>
{currentStep > 0 && currentStep < 4 && (
<Button type="button" variant="outline" onClick={handleBack}>
← Back
{t("agents.addForm.dialog.back")}
</Button>
)}
</div>
<div className="flex gap-3">
{currentStep < 4 && (
<Button variant="secondary" onClick={handleClose}>
Cancel
{t("agents.addForm.dialog.cancel")}
</Button>
)}
{currentStep < 3 && <Button onClick={handleNext}>Next →</Button>}
{currentStep < 3 && <Button onClick={handleNext}>{t("agents.addForm.dialog.next")}</Button>}
{currentStep === 3 && (
<Button disabled={isSubmitting} aria-busy={isSubmitting} onClick={handleCreateAgent}>
{isSubmitting && <UiLoadingSpinner className="size-4" />}
{isSubmitting ? "Creating..." : "Create Agent →"}
{isSubmitting ? t("agents.addForm.dialog.creating") : t("agents.addForm.dialog.create")}
</Button>
)}
{currentStep === 4 && <Button onClick={handleClose}>Done</Button>}
{currentStep === 4 && <Button onClick={handleClose}>{t("agents.addForm.dialog.done")}</Button>}
</div>
</div>
</div>

View file

@ -2,6 +2,8 @@
import React, { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { ChevronDown, CircleAlert, CircleCheck, Info, Link as LinkIcon, RotateCw, Search, X } from "lucide-react";
import { Trans, useTranslation } from "react-i18next";
import i18n from "@/i18n/i18n";
import { useDebouncedCallback } from "@tanstack/react-pacer/debouncer";
import { DiscoveredAgentCard, discoverAgentCardCall } from "@/components/networking";
@ -63,6 +65,7 @@ const AgentCardDiscovery: React.FC<AgentCardDiscoveryProps> = ({
discoveryRequest,
savedAgentCard,
}) => {
const { t } = useTranslation();
// When the parent drives discovery, ``manualUrl`` is unused — the URL
// comes from ``discoveryRequest.url`` directly. When the parent hasn't
// supplied a plan, the admin types into this field manually.
@ -117,14 +120,16 @@ const AgentCardDiscovery: React.FC<AgentCardDiscoveryProps> = ({
const handleDiscover = useCallback(async () => {
if (!accessToken) {
setError("No access token available");
setError(i18n.t("agents.cardDiscovery.errors.noAccessToken"));
onApplyRef.current(null);
return;
}
const trimmed = effectiveUrl.trim();
if (!trimmed) {
setError(
isParentDriven ? "Fill in the agent's connection details above first" : "Enter the agent's base URL first",
isParentDriven
? i18n.t("agents.cardDiscovery.errors.fillConnectionDetails")
: i18n.t("agents.cardDiscovery.errors.enterBaseUrl"),
);
setCard(null);
onApplyRef.current(null);
@ -152,7 +157,7 @@ const AgentCardDiscovery: React.FC<AgentCardDiscoveryProps> = ({
resetSelections(response.agent_card);
} catch (e: any) {
if (requestId !== discoverRequestIdRef.current) return;
setError(e?.message ? String(e.message) : "Failed to discover agent card");
setError(e?.message ? String(e.message) : i18n.t("agents.cardDiscovery.errors.discoverFailed"));
setCard(null);
lastSyncedSelectionRef.current = null;
onApplyRef.current(null);
@ -247,13 +252,13 @@ const AgentCardDiscovery: React.FC<AgentCardDiscoveryProps> = ({
if (card) return <RotateCw />;
return <Search />;
};
const discoverLabel = card ? "Re-discover" : "Discover";
const discoverLabel = card ? t("agents.cardDiscovery.rediscover") : t("agents.cardDiscovery.discover");
return (
<div className="mb-4 rounded-lg border border-border bg-muted/50 p-4">
<div className="mb-2 flex items-center gap-2">
<LinkIcon className="size-4 text-primary" />
<span className="text-sm font-medium text-foreground">Discover from agent URL</span>
<span className="text-sm font-medium text-foreground">{t("agents.cardDiscovery.title")}</span>
<TooltipProvider delay={300}>
<Tooltip>
<TooltipTrigger
@ -264,20 +269,17 @@ const AgentCardDiscovery: React.FC<AgentCardDiscoveryProps> = ({
}
/>
<TooltipContent>
LiteLLM will fetch /.well-known/agent-card.json from this URL and let you pick which skills and
capabilities to expose through the proxy.
{t("agents.cardDiscovery.titleHint", { path: "/.well-known/agent-card.json" })}
</TooltipContent>
</Tooltip>
</TooltipProvider>
</div>
{isParentDriven ? (
<>
<p className="mb-2 text-xs text-muted-foreground">
Using the connection details you entered above. We&apos;ll fetch:
</p>
<p className="mb-2 text-xs text-muted-foreground">{t("agents.cardDiscovery.parentDrivenIntro")}</p>
<div className="mb-3 rounded-sm border border-border bg-background px-3 py-2 font-mono text-xs break-all text-foreground">
{discoveryRequest!.display_url || effectiveUrl || (
<span className="text-muted-foreground italic">Fill in the fields above first</span>
<span className="text-muted-foreground italic">{t("agents.cardDiscovery.fillFieldsFirst")}</span>
)}
</div>
<div className="flex justify-end">
@ -290,8 +292,14 @@ const AgentCardDiscovery: React.FC<AgentCardDiscoveryProps> = ({
) : (
<>
<p className="mb-3 text-xs text-muted-foreground">
Paste the upstream agent&apos;s base URL. We&apos;ll try <code>/.well-known/agent-card.json</code>,{" "}
<code>/.well-known/agent.json</code>, and <code>/agent.json</code> in order.
<Trans
i18nKey="agents.cardDiscovery.manualIntro"
components={{
cardfile: <code>/.well-known/agent-card.json</code>,
agentfile: <code>/.well-known/agent.json</code>,
rootfile: <code>/agent.json</code>,
}}
/>
</p>
<div className="flex w-full items-center gap-2">
@ -315,10 +323,15 @@ const AgentCardDiscovery: React.FC<AgentCardDiscoveryProps> = ({
{error && (
<Alert variant="destructive" className="mt-3">
<CircleAlert />
<AlertTitle>Discovery failed</AlertTitle>
<AlertTitle>{t("agents.cardDiscovery.failedTitle")}</AlertTitle>
<AlertDescription>{error}</AlertDescription>
<AlertAction>
<Button variant="ghost" size="icon-xs" aria-label="Dismiss error" onClick={() => setError(null)}>
<Button
variant="ghost"
size="icon-xs"
aria-label={t("agents.cardDiscovery.dismissError")}
onClick={() => setError(null)}
>
<X />
</Button>
</AlertAction>
@ -335,7 +348,7 @@ const AgentCardDiscovery: React.FC<AgentCardDiscoveryProps> = ({
<div className="mt-4 rounded-lg border border-border bg-background p-4">
<div className="mb-3 flex flex-wrap items-center gap-2">
<CircleCheck className="size-4 text-success" />
<span className="text-sm font-medium text-foreground">Upstream card loaded</span>
<span className="text-sm font-medium text-foreground">{t("agents.cardDiscovery.cardLoaded")}</span>
{card.version && <Badge variant="secondary">v{card.version}</Badge>}
{card.provider?.organization && <Badge variant="secondary">{card.provider.organization}</Badge>}
</div>
@ -343,18 +356,24 @@ const AgentCardDiscovery: React.FC<AgentCardDiscoveryProps> = ({
<div className="mb-4 grid grid-cols-1 gap-3 md:grid-cols-2">
<div>
<label className="mb-1 block text-xs font-medium text-muted-foreground">
Name (shown to API clients)
{t("agents.cardDiscovery.nameLabel")}
</label>
<Input value={editedName} onChange={(e) => setEditedName(e.target.value)} placeholder="Agent name" />
<Input
value={editedName}
onChange={(e) => setEditedName(e.target.value)}
placeholder={t("agents.cardDiscovery.namePlaceholder")}
/>
</div>
<div>
<label className="mb-1 block text-xs font-medium text-muted-foreground">Description</label>
<label className="mb-1 block text-xs font-medium text-muted-foreground">
{t("agents.cardDiscovery.descriptionLabel")}
</label>
<Textarea
className="field-sizing-fixed min-h-0"
value={editedDescription}
onChange={(e) => setEditedDescription(e.target.value)}
rows={2}
placeholder="What this agent does"
placeholder={t("agents.cardDiscovery.descriptionPlaceholder")}
/>
</div>
</div>
@ -366,17 +385,19 @@ const AgentCardDiscovery: React.FC<AgentCardDiscoveryProps> = ({
render={
<button type="button" className="group flex items-center gap-2">
<ChevronDown className="size-4 text-muted-foreground transition-transform group-data-[panel-open]:rotate-180" />
<span className="text-sm font-medium text-foreground">Skills</span>
<span className="text-sm font-medium text-foreground">{t("agents.cardDiscovery.skills")}</span>
</button>
}
/>
<Badge variant="secondary">
{selectedSkillCount} / {skillCount} selected
{t("agents.cardDiscovery.skillsSelected", { selected: selectedSkillCount, total: skillCount })}
</Badge>
</div>
<CollapsibleContent className="pt-2">
{skillCount === 0 ? (
<div className="py-6 text-center text-sm text-muted-foreground">Upstream card has no skills</div>
<div className="py-6 text-center text-sm text-muted-foreground">
{t("agents.cardDiscovery.noSkills")}
</div>
) : (
<div className="space-y-2">
{(card.skills ?? []).map((skill, idx) => {
@ -394,9 +415,9 @@ const AgentCardDiscovery: React.FC<AgentCardDiscoveryProps> = ({
<div className="flex flex-wrap items-center gap-2">
<span className="text-sm font-medium text-foreground">{skill.name || id}</span>
{skill.id && <Badge variant="secondary">{skill.id}</Badge>}
{(skill.tags ?? []).map((t: string) => (
<Badge key={t} variant="outline">
{t}
{(skill.tags ?? []).map((tag: string) => (
<Badge key={tag} variant="outline">
{tag}
</Badge>
))}
</div>
@ -418,7 +439,9 @@ const AgentCardDiscovery: React.FC<AgentCardDiscoveryProps> = ({
render={
<button type="button" className="group flex items-center gap-2">
<ChevronDown className="size-4 text-muted-foreground transition-transform group-data-[panel-open]:rotate-180" />
<span className="text-sm font-medium text-foreground">Capabilities</span>
<span className="text-sm font-medium text-foreground">
{t("agents.cardDiscovery.capabilities")}
</span>
</button>
}
/>
@ -431,10 +454,7 @@ const AgentCardDiscovery: React.FC<AgentCardDiscoveryProps> = ({
</span>
}
/>
<TooltipContent>
Only capabilities LiteLLM can faithfully proxy today are listed. Others (push notifications,
extensions) are coming soon.
</TooltipContent>
<TooltipContent>{t("agents.cardDiscovery.capabilitiesHint")}</TooltipContent>
</Tooltip>
</TooltipProvider>
</div>
@ -449,7 +469,7 @@ const AgentCardDiscovery: React.FC<AgentCardDiscoveryProps> = ({
>
<div className="flex items-center gap-2">
<span className="text-sm font-medium text-foreground capitalize">{key}</span>
{!upstreamHas && <Badge variant="outline">not advertised upstream</Badge>}
{!upstreamHas && <Badge variant="outline">{t("agents.cardDiscovery.notAdvertised")}</Badge>}
</div>
<Switch
checked={Boolean(selectedCapabilities[key])}

View file

@ -1,3 +1,4 @@
import type { ParseKeys } from "i18next";
import { parseIdentityForForm } from "./agent_identity";
/**
* Shared configuration for agent form fields
@ -23,12 +24,12 @@ export interface FieldConfig {
rows?: number;
validation?: any[];
options?: string[];
helpText?: string;
helpTextKey?: ParseKeys;
}
export interface SectionConfig {
key: string;
title: string;
titleKey: ParseKeys;
fields: FieldConfig[];
defaultExpanded?: boolean;
}
@ -44,7 +45,7 @@ export const AGENT_FORM_CONFIG: {
} = {
basic: {
key: "basic",
title: "Basic Information",
titleKey: "agents.formConfig.basic.title",
defaultExpanded: true,
fields: [
{
@ -85,14 +86,13 @@ export const AGENT_FORM_CONFIG: {
defaultValue: "1.0",
tooltip:
"The A2A protocol version LiteLLM serves to clients for this agent. LiteLLM converts the upstream agent's responses to this version, so clients always see the version you pick here regardless of the original agent's version.",
helpText:
"LiteLLM serves this version to clients and converts the upstream agent's responses to match it, regardless of the original agent's version.",
helpTextKey: "agents.formConfig.basic.protocolVersionHelp",
},
],
},
skills: {
key: "skills",
title: "Skills",
titleKey: "agents.formConfig.skills.title",
fields: [
{
name: "skills",
@ -104,7 +104,7 @@ export const AGENT_FORM_CONFIG: {
},
capabilities: {
key: "capabilities",
title: "Capabilities",
titleKey: "agents.formConfig.capabilities.title",
fields: [
{
name: "streaming",
@ -126,7 +126,7 @@ export const AGENT_FORM_CONFIG: {
},
optional: {
key: "optional",
title: "Optional Settings",
titleKey: "agents.formConfig.optional.title",
fields: [
{
name: "iconUrl",
@ -149,7 +149,7 @@ export const AGENT_FORM_CONFIG: {
},
litellm: {
key: "litellm",
title: "LiteLLM Parameters",
titleKey: "agents.formConfig.litellm.title",
fields: [
{
name: "model",
@ -165,7 +165,7 @@ export const AGENT_FORM_CONFIG: {
},
cost: {
key: "cost",
title: "Cost Configuration",
titleKey: "agents.formConfig.cost.title",
fields: [
{
name: "cost_per_query",
@ -192,7 +192,7 @@ export const AGENT_FORM_CONFIG: {
},
tracing: {
key: "tracing",
title: "Tracing",
titleKey: "agents.formConfig.tracing.title",
fields: [
{
name: "enable_tracing",
@ -208,33 +208,33 @@ export const AGENT_FORM_CONFIG: {
export const SKILL_FIELD_CONFIG = {
id: {
name: "id",
label: "Skill ID",
labelKey: "agents.formConfig.skillFields.id" as ParseKeys,
required: true,
placeholder: "e.g., hello_world",
placeholderKey: "agents.formConfig.skillFields.idPlaceholder" as ParseKeys,
},
name: {
name: "name",
label: "Skill Name",
labelKey: "agents.formConfig.skillFields.name" as ParseKeys,
required: true,
placeholder: "e.g., Returns hello world",
placeholderKey: "agents.formConfig.skillFields.namePlaceholder" as ParseKeys,
},
description: {
name: "description",
label: "Description",
labelKey: "agents.formConfig.skillFields.description" as ParseKeys,
required: true,
placeholder: "What this skill does",
placeholderKey: "agents.formConfig.skillFields.descriptionPlaceholder" as ParseKeys,
rows: 2,
},
tags: {
name: "tags",
label: "Tags",
labelKey: "agents.formConfig.skillFields.tags" as ParseKeys,
required: true,
placeholder: "Type a tag and press Enter",
placeholderKey: "agents.formConfig.skillFields.tagsPlaceholder" as ParseKeys,
},
examples: {
name: "examples",
label: "Examples",
placeholder: "Type an example and press Enter",
labelKey: "agents.formConfig.skillFields.examples" as ParseKeys,
placeholderKey: "agents.formConfig.skillFields.examplesPlaceholder" as ParseKeys,
},
};

View file

@ -1,4 +1,5 @@
import React from "react";
import { useTranslation } from "react-i18next";
import { Agent } from "@/components/agents/types";
interface AgentCostViewProps {
@ -6,6 +7,7 @@ interface AgentCostViewProps {
}
const AgentCostView: React.FC<AgentCostViewProps> = ({ agent }) => {
const { t } = useTranslation();
const params = agent.litellm_params;
if (params?.cost_per_query == null && params?.input_cost_per_token == null && params?.output_cost_per_token == null) {
@ -14,15 +16,15 @@ const AgentCostView: React.FC<AgentCostViewProps> = ({ agent }) => {
const rows = (
[
["Cost Per Query", params.cost_per_query],
["Input Cost Per Token", params.input_cost_per_token],
["Output Cost Per Token", params.output_cost_per_token],
[t("agents.costView.costPerQuery"), params.cost_per_query],
[t("agents.costView.inputCostPerToken"), params.input_cost_per_token],
[t("agents.costView.outputCostPerToken"), params.output_cost_per_token],
] as const
).filter(([, value]) => value != null);
return (
<div className="mt-6">
<h3 className="text-lg font-semibold text-foreground">Cost Configuration</h3>
<h3 className="text-lg font-semibold text-foreground">{t("agents.costView.title")}</h3>
<dl className="mt-4 divide-y divide-border overflow-hidden rounded-lg border border-border">
{rows.map(([label, value]) => (
<div key={label} className="grid grid-cols-1 sm:grid-cols-3">

View file

@ -1,5 +1,6 @@
import React from "react";
import { useFieldArray, useFormContext } from "react-hook-form";
import { useTranslation } from "react-i18next";
import { Plus, Trash2 } from "lucide-react";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
@ -47,6 +48,7 @@ interface AgentFormFieldsProps {
}
const SkillsFieldArray = () => {
const { t } = useTranslation();
const { control } = useFormContext<AgentFormValues>();
const { fields, append, remove } = useFieldArray({ control, name: "skills" });
@ -57,14 +59,14 @@ const SkillsFieldArray = () => {
<FieldGroup>
<AgentFormField
name={`skills.${index}.id`}
label={SKILL_FIELD_CONFIG.id.label}
rules={SKILL_FIELD_CONFIG.id.required ? { required: "Required" } : undefined}
label={t(SKILL_FIELD_CONFIG.id.labelKey)}
rules={SKILL_FIELD_CONFIG.id.required ? { required: t("agents.formFields.errors.required") } : undefined}
>
{({ value, onChange, ref, ...control }) => (
<Input
{...control}
ref={ref}
placeholder={SKILL_FIELD_CONFIG.id.placeholder}
placeholder={t(SKILL_FIELD_CONFIG.id.placeholderKey)}
value={typeof value === "string" ? value : ""}
onChange={onChange}
/>
@ -73,14 +75,16 @@ const SkillsFieldArray = () => {
<AgentFormField
name={`skills.${index}.name`}
label={SKILL_FIELD_CONFIG.name.label}
rules={SKILL_FIELD_CONFIG.name.required ? { required: "Required" } : undefined}
label={t(SKILL_FIELD_CONFIG.name.labelKey)}
rules={
SKILL_FIELD_CONFIG.name.required ? { required: t("agents.formFields.errors.required") } : undefined
}
>
{({ value, onChange, ref, ...control }) => (
<Input
{...control}
ref={ref}
placeholder={SKILL_FIELD_CONFIG.name.placeholder}
placeholder={t(SKILL_FIELD_CONFIG.name.placeholderKey)}
value={typeof value === "string" ? value : ""}
onChange={onChange}
/>
@ -89,15 +93,19 @@ const SkillsFieldArray = () => {
<AgentFormField
name={`skills.${index}.description`}
label={SKILL_FIELD_CONFIG.description.label}
rules={SKILL_FIELD_CONFIG.description.required ? { required: "Required" } : undefined}
label={t(SKILL_FIELD_CONFIG.description.labelKey)}
rules={
SKILL_FIELD_CONFIG.description.required
? { required: t("agents.formFields.errors.required") }
: undefined
}
>
{({ value, onChange, ref, ...control }) => (
<Textarea
{...control}
ref={ref}
rows={SKILL_FIELD_CONFIG.description.rows}
placeholder={SKILL_FIELD_CONFIG.description.placeholder}
placeholder={t(SKILL_FIELD_CONFIG.description.placeholderKey)}
value={typeof value === "string" ? value : ""}
onChange={onChange}
/>
@ -106,26 +114,28 @@ const SkillsFieldArray = () => {
<AgentFormField
name={`skills.${index}.tags`}
label={SKILL_FIELD_CONFIG.tags.label}
rules={SKILL_FIELD_CONFIG.tags.required ? { required: "Required" } : undefined}
label={t(SKILL_FIELD_CONFIG.tags.labelKey)}
rules={
SKILL_FIELD_CONFIG.tags.required ? { required: t("agents.formFields.errors.required") } : undefined
}
>
{({ id, value, onChange }) => (
<AgentTagsInput
id={id}
value={Array.isArray(value) ? (value as string[]) : []}
onValueChange={onChange}
placeholder={SKILL_FIELD_CONFIG.tags.placeholder}
placeholder={t(SKILL_FIELD_CONFIG.tags.placeholderKey)}
/>
)}
</AgentFormField>
<AgentFormField name={`skills.${index}.examples`} label={SKILL_FIELD_CONFIG.examples.label}>
<AgentFormField name={`skills.${index}.examples`} label={t(SKILL_FIELD_CONFIG.examples.labelKey)}>
{({ id, value, onChange }) => (
<AgentTagsInput
id={id}
value={Array.isArray(value) ? (value as string[]) : []}
onValueChange={onChange}
placeholder={SKILL_FIELD_CONFIG.examples.placeholder}
placeholder={t(SKILL_FIELD_CONFIG.examples.placeholderKey)}
/>
)}
</AgentFormField>
@ -138,19 +148,20 @@ const SkillsFieldArray = () => {
onClick={() => remove(index)}
>
<Trash2 />
Remove Skill
{t("agents.formFields.skills.remove")}
</Button>
</div>
))}
<Button type="button" variant="outline" className="w-full border-dashed" onClick={() => append({})}>
<Plus />
Add Skill
{t("agents.formFields.skills.add")}
</Button>
</>
);
};
const StaticHeadersFieldArray = () => {
const { t } = useTranslation();
const { control } = useFormContext<AgentFormValues>();
const { fields, append, remove } = useFieldArray({ control, name: "static_headers" });
@ -158,25 +169,31 @@ const StaticHeadersFieldArray = () => {
<>
{fields.map((item, index) => (
<div key={item.id} className="flex items-start gap-2">
<AgentFormField name={`static_headers.${index}.header`} rules={{ required: "Header name required" }}>
<AgentFormField
name={`static_headers.${index}.header`}
rules={{ required: t("agents.formFields.staticHeaders.errors.nameRequired") }}
>
{({ value, onChange, ref, ...control }) => (
<Input
{...control}
ref={ref}
className="w-55"
placeholder="Header name (e.g. Authorization)"
placeholder={t("agents.formFields.staticHeaders.namePlaceholder")}
value={typeof value === "string" ? value : ""}
onChange={onChange}
/>
)}
</AgentFormField>
<AgentFormField name={`static_headers.${index}.value`} rules={{ required: "Value required" }}>
<AgentFormField
name={`static_headers.${index}.value`}
rules={{ required: t("agents.formFields.staticHeaders.errors.valueRequired") }}
>
{({ value, onChange, ref, ...control }) => (
<Input
{...control}
ref={ref}
className="w-65"
placeholder="Value (e.g. Bearer token123)"
placeholder={t("agents.formFields.staticHeaders.valuePlaceholder")}
value={typeof value === "string" ? value : ""}
onChange={onChange}
/>
@ -186,7 +203,7 @@ const StaticHeadersFieldArray = () => {
type="button"
variant="ghost"
size="icon"
aria-label="Remove static header"
aria-label={t("agents.formFields.staticHeaders.remove")}
className="text-destructive hover:text-destructive/80"
onClick={() => remove(index)}
>
@ -196,13 +213,14 @@ const StaticHeadersFieldArray = () => {
))}
<Button type="button" variant="outline" className="w-full border-dashed" onClick={() => append({})}>
<Plus />
Add Static Header
{t("agents.formFields.staticHeaders.add")}
</Button>
</>
);
};
const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName = true, visiblePanels }) => {
const { t } = useTranslation();
const shouldShow = (key: string) => !visiblePanels || visiblePanels.includes(key);
return (
@ -211,14 +229,14 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
<FieldGroup className="mb-4">
<AgentFormField
name="agent_name"
label={labelWithHint("Agent Name", "Unique identifier for the agent")}
rules={{ required: "Please enter a unique agent name" }}
label={labelWithHint(t("agents.formFields.agentName.label"), t("agents.formFields.agentName.hint"))}
rules={{ required: t("agents.formFields.agentName.required") }}
>
{({ value, onChange, ref, ...control }) => (
<Input
{...control}
ref={ref}
placeholder="e.g., customer-support-agent"
placeholder={t("agents.formFields.agentName.placeholder")}
value={typeof value === "string" ? value : ""}
onChange={onChange}
/>
@ -231,7 +249,7 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
{shouldShow(AGENT_FORM_CONFIG.basic.key) && (
<AgentFormPanel
panelKey={AGENT_FORM_CONFIG.basic.key}
title={`${AGENT_FORM_CONFIG.basic.title} (Required)`}
title={t("agents.formFields.requiredSection", { section: t(AGENT_FORM_CONFIG.basic.titleKey) })}
panels={panels}
>
{AGENT_FORM_CONFIG.basic.fields.map((field) => (
@ -239,8 +257,12 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
key={field.name}
name={field.name}
label={field.tooltip ? labelWithHint(field.label, field.tooltip) : field.label}
description={field.helpText}
rules={field.required ? { required: `Please enter ${field.label.toLowerCase()}` } : undefined}
description={field.helpTextKey ? t(field.helpTextKey) : undefined}
rules={
field.required
? { required: t("agents.formFields.errors.enterField", { field: field.label.toLowerCase() }) }
: undefined
}
>
{({ value, onChange, ref, ...control }) => {
const text = typeof value === "string" ? value : "";
@ -284,7 +306,7 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
{shouldShow(AGENT_FORM_CONFIG.skills.key) && (
<AgentFormPanel
panelKey={AGENT_FORM_CONFIG.skills.key}
title={AGENT_FORM_CONFIG.skills.title}
title={t(AGENT_FORM_CONFIG.skills.titleKey)}
panels={panels}
>
<SkillsFieldArray />
@ -294,7 +316,7 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
{shouldShow(AGENT_FORM_CONFIG.capabilities.key) && (
<AgentFormPanel
panelKey={AGENT_FORM_CONFIG.capabilities.key}
title={AGENT_FORM_CONFIG.capabilities.title}
title={t(AGENT_FORM_CONFIG.capabilities.titleKey)}
panels={panels}
>
{AGENT_FORM_CONFIG.capabilities.fields.map((field) => (
@ -310,7 +332,7 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
{shouldShow(AGENT_FORM_CONFIG.optional.key) && (
<AgentFormPanel
panelKey={AGENT_FORM_CONFIG.optional.key}
title={AGENT_FORM_CONFIG.optional.title}
title={t(AGENT_FORM_CONFIG.optional.titleKey)}
panels={panels}
>
{AGENT_FORM_CONFIG.optional.fields.map((field) => (
@ -334,7 +356,11 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
)}
{shouldShow(AGENT_FORM_CONFIG.cost.key) && (
<AgentFormPanel panelKey={AGENT_FORM_CONFIG.cost.key} title={AGENT_FORM_CONFIG.cost.title} panels={panels}>
<AgentFormPanel
panelKey={AGENT_FORM_CONFIG.cost.key}
title={t(AGENT_FORM_CONFIG.cost.titleKey)}
panels={panels}
>
<CostConfigFields />
</AgentFormPanel>
)}
@ -342,7 +368,7 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
{shouldShow(AGENT_FORM_CONFIG.litellm.key) && (
<AgentFormPanel
panelKey={AGENT_FORM_CONFIG.litellm.key}
title={AGENT_FORM_CONFIG.litellm.title}
title={t(AGENT_FORM_CONFIG.litellm.titleKey)}
panels={panels}
>
{AGENT_FORM_CONFIG.litellm.fields.map((field) => (
@ -366,12 +392,16 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
)}
{shouldShow(AUTH_HEADERS_PANEL_KEY) && (
<AgentFormPanel panelKey={AUTH_HEADERS_PANEL_KEY} title="Authentication Headers" panels={panels}>
<AgentFormPanel
panelKey={AUTH_HEADERS_PANEL_KEY}
title={t("agents.formFields.authHeaders.title")}
panels={panels}
>
<Field>
<FieldTitle>
{labelWithHint(
"Static Headers",
"Headers always sent to the backend agent, regardless of the client request. Admin-configured, static wins on conflict.",
t("agents.formFields.authHeaders.staticHeaders"),
t("agents.formFields.authHeaders.staticHeadersHint"),
)}
</FieldTitle>
<div className="flex flex-col gap-2">
@ -382,8 +412,8 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
<AgentFormField
name="extra_headers"
label={labelWithHint(
"Forward Client Headers",
"Header names to extract from the client's request and forward to the agent. Type a name and press Enter.",
t("agents.formFields.authHeaders.forwardHeaders"),
t("agents.formFields.authHeaders.forwardHeadersHint"),
)}
>
{({ id, value, onChange }) => (
@ -391,7 +421,7 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
id={id}
value={Array.isArray(value) ? (value as string[]) : []}
onValueChange={onChange}
placeholder="e.g. x-api-key, Authorization"
placeholder={t("agents.formFields.authHeaders.forwardHeadersPlaceholder")}
/>
)}
</AgentFormField>
@ -399,7 +429,7 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
)}
{shouldShow(KILL_SWITCH_PANEL_KEY) && (
<AgentFormPanel panelKey={KILL_SWITCH_PANEL_KEY} title="Kill Switch" panels={panels}>
<AgentFormPanel panelKey={KILL_SWITCH_PANEL_KEY} title={t("agents.formFields.killSwitch")} panels={panels}>
<KillSwitchFormFields />
</AgentFormPanel>
)}

View file

@ -14,6 +14,7 @@ import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
import { Field, FieldGroup, FieldLabel } from "@/components/ui/field";
import { toast } from "@/lib/toast";
import { ArrowLeft } from "lucide-react";
import { useTranslation } from "react-i18next";
import { getAgentInfo, patchAgentCall, getAgentCreateMetadata, AgentCreateInfo } from "@/components/networking";
import { Agent } from "@/components/agents/types";
import { KeyResponse } from "@/components/key_team_helpers/key_list";
@ -79,6 +80,7 @@ const DetailItem: React.FC<{ label: React.ReactNode; children: React.ReactNode }
);
const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessToken, isAdmin }) => {
const { t } = useTranslation();
const [agent, setAgent] = useState<Agent | null>(null);
const [selectedKey, setSelectedKey] = useState<KeyResponse | null>(null);
const { data: keysData, isLoading: keysLoading, refetch: refetchAgentKeys } = useKeys(1, 100, { agentID: agentId });
@ -127,7 +129,7 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
if (agentType === "a2a") {
form.reset(parseAgentForForm(data));
} else {
const typeInfo = agentTypeMetadata.find((t) => t.agent_type === agentType);
const typeInfo = agentTypeMetadata.find((info) => info.agent_type === agentType);
if (typeInfo) {
form.reset({
...parseDynamicAgentForForm(data, typeInfo),
@ -140,7 +142,7 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
}
} catch (error) {
console.error("Error fetching agent info:", error);
toast.error("Failed to load agent information");
toast.error(t("agents.info.toasts.loadFailed"));
} finally {
setIsLoading(false);
}
@ -151,7 +153,7 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
if (agent && agentTypeMetadata.length > 0) {
const agentType = detectAgentType(agent);
if (agentType !== "a2a") {
const typeInfo = agentTypeMetadata.find((t) => t.agent_type === agentType);
const typeInfo = agentTypeMetadata.find((info) => info.agent_type === agentType);
if (typeInfo) {
form.reset({
...parseDynamicAgentForForm(agent, typeInfo),
@ -163,7 +165,7 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
}
}, [agentTypeMetadata, agent]);
const selectedAgentTypeInfo = agentTypeMetadata.find((t) => t.agent_type === detectedAgentType);
const selectedAgentTypeInfo = agentTypeMetadata.find((info) => info.agent_type === detectedAgentType);
const watchedFormValues = useWatch({ control: form.control });
const mcpSelection = useWatch({ control: form.control, name: "allowed_mcp_servers_and_groups" });
const mcpToolPermissions = useWatch({ control: form.control, name: "mcp_tool_permissions" });
@ -249,12 +251,12 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
object_permission: buildMcpObjectPermission(values),
access_group_ids: values.access_group_ids ?? [],
});
toast.success("Agent updated successfully");
toast.success(t("agents.info.toasts.updated"));
setIsEditing(false);
fetchAgentInfo();
} catch (error) {
console.error("Error updating agent:", error);
toast.error("Failed to update agent");
toast.error(t("agents.info.toasts.updateFailed"));
} finally {
setIsSaving(false);
}
@ -273,9 +275,9 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
if (!agent) {
return (
<div className="p-4">
<div className="text-center">Agent not found</div>
<div className="text-center">{t("agents.info.notFound")}</div>
<Button onClick={onClose} className="mt-4">
Back to Agents List
{t("agents.info.backToList")}
</Button>
</div>
);
@ -297,7 +299,7 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
onChange={onChange}
inputRef={ref}
min={0}
placeholder="Unlimited"
placeholder={t("agents.info.unlimited")}
/>
)}
</AgentFormField>
@ -314,7 +316,7 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
refetchAgentKeys();
}}
teams={null}
backButtonText="Back to Agent"
backButtonText={t("agents.info.backToAgent")}
/>
);
}
@ -324,20 +326,20 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
<div>
<Button variant="ghost" onClick={onClose} className="mb-4">
<ArrowLeft className="size-4" />
Back to Agents
{t("agents.info.backToAgents")}
</Button>
<h1 className="text-2xl font-semibold">{agent.agent_name || "Unnamed Agent"}</h1>
<h1 className="text-2xl font-semibold">{agent.agent_name || t("agents.info.unnamed")}</h1>
<p className="text-sm text-muted-foreground font-mono">{agent.agent_id}</p>
</div>
<Tabs value={activeTab} onValueChange={setActiveTab}>
<TabsList variant="line" className="mb-4 h-auto w-full justify-start rounded-none border-b p-0">
<TabsTrigger value="overview" className="flex-none rounded-none px-4 py-2">
Overview
{t("agents.info.tabs.overview")}
</TabsTrigger>
{isAdmin && (
<TabsTrigger value="settings" className="flex-none rounded-none px-4 py-2">
Settings
{t("agents.info.tabs.settings")}
</TabsTrigger>
)}
</TabsList>
@ -352,38 +354,62 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
isAdmin={isAdmin}
/>
<DetailList>
<DetailItem label="Agent ID">{agent.agent_id}</DetailItem>
<DetailItem label="Agent Name">{agent.agent_name}</DetailItem>
<DetailItem label="Display Name">{agent.agent_card_params?.name || "-"}</DetailItem>
<DetailItem label="Description">{agent.agent_card_params?.description || "-"}</DetailItem>
<DetailItem label="URL">{agent.agent_card_params?.url || "-"}</DetailItem>
<DetailItem label="Version">{agent.agent_card_params?.version || "-"}</DetailItem>
<DetailItem label="Protocol Version">{agent.agent_card_params?.protocolVersion || "-"}</DetailItem>
<DetailItem label="Streaming">
{agent.agent_card_params?.capabilities?.streaming ? "Yes" : "No"}
<DetailItem label={t("agents.info.details.agentId")}>{agent.agent_id}</DetailItem>
<DetailItem label={t("agents.info.details.agentName")}>{agent.agent_name}</DetailItem>
<DetailItem label={t("agents.info.details.displayName")}>
{agent.agent_card_params?.name || "-"}
</DetailItem>
<DetailItem label={t("agents.info.details.description")}>
{agent.agent_card_params?.description || "-"}
</DetailItem>
<DetailItem label={t("agents.info.details.url")}>{agent.agent_card_params?.url || "-"}</DetailItem>
<DetailItem label={t("agents.info.details.version")}>
{agent.agent_card_params?.version || "-"}
</DetailItem>
<DetailItem label={t("agents.info.details.protocolVersion")}>
{agent.agent_card_params?.protocolVersion || "-"}
</DetailItem>
<DetailItem label={t("agents.info.details.streaming")}>
{agent.agent_card_params?.capabilities?.streaming ? t("agents.info.yes") : t("agents.info.no")}
</DetailItem>
{agent.agent_card_params?.capabilities?.pushNotifications && (
<DetailItem label="Push Notifications">Yes</DetailItem>
<DetailItem label={t("agents.info.details.pushNotifications")}>{t("agents.info.yes")}</DetailItem>
)}
{agent.agent_card_params?.capabilities?.stateTransitionHistory && (
<DetailItem label="State Transition History">Yes</DetailItem>
<DetailItem label={t("agents.info.details.stateTransitionHistory")}>{t("agents.info.yes")}</DetailItem>
)}
<DetailItem label={t("agents.info.details.skills")}>
{t("agents.info.skillsConfigured", { count: agent.agent_card_params?.skills?.length || 0 })}
</DetailItem>
{agent.litellm_params?.model && (
<DetailItem label={t("agents.info.details.model")}>{agent.litellm_params.model}</DetailItem>
)}
<DetailItem label="Skills">{agent.agent_card_params?.skills?.length || 0} configured</DetailItem>
{agent.litellm_params?.model && <DetailItem label="Model">{agent.litellm_params.model}</DetailItem>}
{agent.litellm_params?.make_public !== undefined && (
<DetailItem label="Make Public">{agent.litellm_params.make_public ? "Yes" : "No"}</DetailItem>
<DetailItem label={t("agents.info.details.makePublic")}>
{agent.litellm_params.make_public ? t("agents.info.yes") : t("agents.info.no")}
</DetailItem>
)}
{agent.agent_card_params?.iconUrl && (
<DetailItem label="Icon URL">{agent.agent_card_params.iconUrl}</DetailItem>
<DetailItem label={t("agents.info.details.iconUrl")}>{agent.agent_card_params.iconUrl}</DetailItem>
)}
{agent.agent_card_params?.documentationUrl && (
<DetailItem label="Documentation URL">{agent.agent_card_params.documentationUrl}</DetailItem>
<DetailItem label={t("agents.info.details.documentationUrl")}>
{agent.agent_card_params.documentationUrl}
</DetailItem>
)}
<DetailItem label="TPM Limit">{agent.tpm_limit ?? "Unlimited"}</DetailItem>
<DetailItem label="RPM Limit">{agent.rpm_limit ?? "Unlimited"}</DetailItem>
<DetailItem label="Session TPM Limit">{agent.session_tpm_limit ?? "Unlimited"}</DetailItem>
<DetailItem label="Session RPM Limit">{agent.session_rpm_limit ?? "Unlimited"}</DetailItem>
<DetailItem label="Access Groups">
<DetailItem label={t("agents.info.details.tpmLimit")}>
{agent.tpm_limit ?? t("agents.info.unlimited")}
</DetailItem>
<DetailItem label={t("agents.info.details.rpmLimit")}>
{agent.rpm_limit ?? t("agents.info.unlimited")}
</DetailItem>
<DetailItem label={t("agents.info.details.sessionTpmLimit")}>
{agent.session_tpm_limit ?? t("agents.info.unlimited")}
</DetailItem>
<DetailItem label={t("agents.info.details.sessionRpmLimit")}>
{agent.session_rpm_limit ?? t("agents.info.unlimited")}
</DetailItem>
<DetailItem label={t("agents.info.details.accessGroups")}>
{agent.access_group_ids?.length ? (
<div className="space-y-1">
{agent.access_group_ids.map((accessGroupId) => (
@ -391,11 +417,11 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
))}
</div>
) : (
"None"
t("agents.info.none")
)}
</DetailItem>
<DetailItem label="Created At">{formatDate(agent.created_at)}</DetailItem>
<DetailItem label="Updated At">{formatDate(agent.updated_at)}</DetailItem>
<DetailItem label={t("agents.info.details.createdAt")}>{formatDate(agent.created_at)}</DetailItem>
<DetailItem label={t("agents.info.details.updatedAt")}>{formatDate(agent.updated_at)}</DetailItem>
</DetailList>
<AgentVirtualKeys keys={agentKeys} isLoading={keysLoading} onKeyClick={setSelectedKey} />
@ -407,10 +433,10 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
(agent.object_permission.mcp_tool_permissions &&
Object.keys(agent.object_permission.mcp_tool_permissions).length > 0)) && (
<div style={{ marginTop: 24 }}>
<h3 className="text-lg font-medium">MCP Tool Permissions</h3>
<h3 className="text-lg font-medium">{t("agents.info.mcpPermissions.title")}</h3>
<DetailList className="mt-4">
{agent.object_permission.mcp_servers && agent.object_permission.mcp_servers.length > 0 && (
<DetailItem label="MCP Servers">
<DetailItem label={t("agents.info.mcpPermissions.servers")}>
<div className="space-y-1">
{agent.object_permission.mcp_servers.map((serverId) => (
<div key={serverId}>{mcpServerLabel(serverId)}</div>
@ -420,16 +446,18 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
)}
{agent.object_permission.mcp_access_groups &&
agent.object_permission.mcp_access_groups.length > 0 && (
<DetailItem label="MCP Access Groups">
<DetailItem label={t("agents.info.mcpPermissions.accessGroups")}>
{agent.object_permission.mcp_access_groups.join(", ")}
</DetailItem>
)}
{agent.object_permission.mcp_toolsets && agent.object_permission.mcp_toolsets.length > 0 && (
<DetailItem label="MCP Toolsets">{agent.object_permission.mcp_toolsets.join(", ")}</DetailItem>
<DetailItem label={t("agents.info.mcpPermissions.toolsets")}>
{agent.object_permission.mcp_toolsets.join(", ")}
</DetailItem>
)}
{agent.object_permission.mcp_tool_permissions &&
Object.keys(agent.object_permission.mcp_tool_permissions).length > 0 && (
<DetailItem label="Tool permissions per server">
<DetailItem label={t("agents.info.mcpPermissions.toolsPerServer")}>
<div className="space-y-1">
{Object.entries(agent.object_permission.mcp_tool_permissions).map(([serverId, tools]) => (
<div key={serverId}>
@ -448,23 +476,27 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
{agent.agent_card_params?.skills && agent.agent_card_params.skills.length > 0 && (
<div style={{ marginTop: 24 }}>
<h3 className="text-lg font-medium">Skills</h3>
<h3 className="text-lg font-medium">{t("agents.info.skillsSection.title")}</h3>
<DetailList className="mt-4">
{agent.agent_card_params.skills.map((skill, index) => (
<DetailItem label={skill.name || `Skill ${index + 1}`} key={index}>
<DetailItem
label={skill.name || t("agents.info.skillsSection.fallbackName", { number: index + 1 })}
key={index}
>
<div>
<div>
<strong>ID:</strong> {skill.id}
<strong>{t("agents.info.skillsSection.id")}</strong> {skill.id}
</div>
<div>
<strong>Description:</strong> {skill.description}
<strong>{t("agents.info.skillsSection.description")}</strong> {skill.description}
</div>
<div>
<strong>Tags:</strong> {Array.isArray(skill.tags) ? skill.tags.join(", ") : skill.tags}
<strong>{t("agents.info.skillsSection.tags")}</strong>{" "}
{Array.isArray(skill.tags) ? skill.tags.join(", ") : skill.tags}
</div>
{skill.examples && skill.examples.length > 0 && (
<div>
<strong>Examples:</strong>{" "}
<strong>{t("agents.info.skillsSection.examples")}</strong>{" "}
{Array.isArray(skill.examples) ? skill.examples.join(", ") : skill.examples}
</div>
)}
@ -489,7 +521,7 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
<TabsContent value="settings" keepMounted>
<Card className="block p-6">
<div className="flex justify-between items-center mb-4">
<h3 className="text-lg font-medium">Agent Settings</h3>
<h3 className="text-lg font-medium">{t("agents.info.settings.title")}</h3>
{!isEditing && (
<Button
onClick={() => {
@ -497,7 +529,7 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
setIsEditing(true);
}}
>
Edit Settings
{t("agents.info.settings.edit")}
</Button>
)}
</div>
@ -508,7 +540,7 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
<form onSubmit={form.handleSubmit(handleUpdate)}>
<FieldGroup className="mb-4">
<Field>
<FieldLabel htmlFor="agent-id">Agent ID</FieldLabel>
<FieldLabel htmlFor="agent-id">{t("agents.info.settings.agentId")}</FieldLabel>
<Input id="agent-id" value={agent.agent_id} disabled readOnly />
</Field>
</FieldGroup>
@ -533,44 +565,44 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
)}
<Separator className="my-6" />
<h3 className="text-lg font-medium mb-4">Rate Limits</h3>
<h3 className="text-lg font-medium mb-4">{t("agents.info.settings.rateLimits")}</h3>
<div className="grid grid-cols-2 gap-4">
{rateLimitField("tpm_limit", "TPM Limit")}
{rateLimitField("rpm_limit", "RPM Limit")}
{rateLimitField("tpm_limit", t("agents.info.settings.tpmLimit"))}
{rateLimitField("rpm_limit", t("agents.info.settings.rpmLimit"))}
</div>
<div className="mt-4 grid grid-cols-2 gap-4">
{rateLimitField("session_tpm_limit", "Session TPM Limit")}
{rateLimitField("session_rpm_limit", "Session RPM Limit")}
{rateLimitField("session_tpm_limit", t("agents.info.settings.sessionTpmLimit"))}
{rateLimitField("session_rpm_limit", t("agents.info.settings.sessionRpmLimit"))}
</div>
<Separator className="my-6" />
<h3 className="text-lg font-medium mb-4">Access Groups</h3>
<h3 className="text-lg font-medium mb-4">{t("agents.info.settings.accessGroups.title")}</h3>
<FieldGroup>
<AgentFormField
name="access_group_ids"
label={labelWithHint(
"Access Groups",
"Attached groups cap which models, MCP servers, and agents this agent can reach, on top of its key and team permissions. Leave empty to apply no extra cap.",
t("agents.info.settings.accessGroups.label"),
t("agents.info.settings.accessGroups.hint"),
)}
>
{({ value, onChange }) => (
<AccessGroupSelector
value={Array.isArray(value) ? (value as string[]) : []}
onChange={onChange}
placeholder="Select access groups (optional)"
placeholder={t("agents.info.settings.accessGroups.placeholder")}
/>
)}
</AgentFormField>
</FieldGroup>
<Separator className="my-6" />
<h3 className="text-lg font-medium mb-4">MCP Servers</h3>
<h3 className="text-lg font-medium mb-4">{t("agents.info.settings.mcpServers.title")}</h3>
<FieldGroup>
<AgentFormField
name="allowed_mcp_servers_and_groups"
label={labelWithHint(
"Allowed MCP Servers",
"Select which MCP servers or access groups this agent can access. Keys bound to this agent can only reach servers granted here.",
t("agents.info.settings.mcpServers.label"),
t("agents.info.settings.mcpServers.hint"),
)}
>
{({ value, onChange }) => (
@ -582,7 +614,7 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
toolsets: (value as McpServerSelection | undefined)?.toolsets ?? [],
}}
accessToken={accessToken ?? ""}
placeholder="Select MCP servers or access groups (optional)"
placeholder={t("agents.info.settings.mcpServers.placeholder")}
/>
)}
</AgentFormField>
@ -608,18 +640,18 @@ const AgentInfoView: React.FC<AgentInfoViewProps> = ({ agentId, onClose, accessT
fetchAgentInfo();
}}
>
Cancel
{t("agents.info.settings.cancel")}
</Button>
<Button type="submit" disabled={isSaving} aria-busy={isSaving}>
{isSaving && <UiLoadingSpinner className="size-4" />}
Save Changes
{t("agents.info.settings.save")}
</Button>
</div>
</form>
</FormProvider>
</TooltipProvider>
) : (
<p>Click &quot;Edit Settings&quot; to modify agent configuration.</p>
<p>{t("agents.info.settings.editHint")}</p>
)}
</Card>
</TabsContent>

View file

@ -1,4 +1,5 @@
import React from "react";
import { useTranslation } from "react-i18next";
import { KeyRound } from "lucide-react";
import { KeyResponse } from "@/components/key_team_helpers/key_list";
import { Button } from "@/components/ui/button";
@ -11,19 +12,22 @@ interface AgentVirtualKeysProps {
}
const AgentVirtualKeys: React.FC<AgentVirtualKeysProps> = ({ keys, isLoading, onKeyClick }) => {
const { t } = useTranslation();
return (
<div className="mt-6">
<h4 className="text-base font-semibold text-foreground">Virtual Keys</h4>
<h4 className="text-base font-semibold text-foreground">{t("agents.virtualKeys.title")}</h4>
{isLoading ? (
<p className="mt-2 text-sm text-muted-foreground">Loading keys...</p>
<p className="mt-2 text-sm text-muted-foreground">{t("agents.virtualKeys.loading")}</p>
) : keys.length === 0 ? (
<p className="mt-2 text-sm text-muted-foreground">No virtual key assigned to this agent.</p>
<p className="mt-2 text-sm text-muted-foreground">{t("agents.virtualKeys.empty")}</p>
) : (
<div className="mt-3 flex flex-col gap-2">
{keys.map((key) => (
<div key={key.token} className="flex items-center gap-3 rounded-sm border border-border px-3 py-2">
<KeyRound className="size-4 text-muted-foreground" />
<span className="text-sm font-medium text-foreground">{key.key_alias || "Unnamed key"}</span>
<span className="text-sm font-medium text-foreground">
{key.key_alias || t("agents.virtualKeys.unnamed")}
</span>
{key.key_name && <span className="font-mono text-xs text-muted-foreground">{key.key_name}</span>}
<TooltipProvider delay={300}>
<Tooltip>

View file

@ -1,10 +1,13 @@
import React from "react";
import type { TFunction } from "i18next";
import { useTranslation } from "react-i18next";
import { Input } from "@/components/ui/input";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { Textarea } from "@/components/ui/textarea";
import { FieldGroup } from "@/components/ui/field";
import { AgentCreateInfo, AgentCredentialFieldMetadata } from "@/components/networking";
import { PasswordInput } from "@/components/shared/PasswordInput";
import i18n from "@/i18n/i18n";
import { AGENT_FORM_CONFIG, applyKillSwitchToPayload } from "./agent_config";
import CostConfigFields, { COST_FIELD_NAMES } from "./cost_config_fields";
import KillSwitchFormFields from "./KillSwitchFormFields";
@ -32,12 +35,13 @@ export const unmountedDynamicFieldNames = (mountedPanels: readonly string[]): re
// validation rather than throwing during render and taking the whole form down with it.
const buildValidationPatternRule = (
field: AgentCredentialFieldMetadata,
t: TFunction,
): { value: RegExp; message: string } | undefined => {
if (!field.validation_pattern) return undefined;
try {
return {
value: new RegExp(field.validation_pattern),
message: field.validation_message || `${field.label} looks incomplete or malformed`,
message: field.validation_message || t("agents.dynamicFields.errors.malformed", { field: field.label }),
};
} catch {
return undefined;
@ -45,14 +49,15 @@ const buildValidationPatternRule = (
};
const CredentialField = ({ field }: { field: AgentCredentialFieldMetadata }) => {
const patternRule = buildValidationPatternRule(field);
const { t } = useTranslation();
const patternRule = buildValidationPatternRule(field, t);
return (
<AgentFormField
name={field.key}
label={field.tooltip ? labelWithHint(field.label, field.tooltip) : field.label}
defaultValue={field.default_value ?? undefined}
rules={{
...(field.required ? { required: `Please enter ${field.label}` } : {}),
...(field.required ? { required: t("agents.dynamicFields.errors.enterField", { field: field.label }) } : {}),
...(patternRule ? { pattern: patternRule } : {}),
}}
>
@ -108,14 +113,17 @@ const DynamicAgentFormFields: React.FC<DynamicAgentFormFieldsProps> = ({ agentTy
<FieldGroup className="mb-4">
<AgentFormField
name="agent_name"
label={labelWithHint("Agent Name", "Unique identifier for the agent")}
rules={{ required: "Please enter a unique agent name" }}
label={labelWithHint(
i18n.t("agents.dynamicFields.agentName.label"),
i18n.t("agents.dynamicFields.agentName.hint"),
)}
rules={{ required: i18n.t("agents.dynamicFields.agentName.required") }}
>
{({ value, onChange, ref, ...control }) => (
<Input
{...control}
ref={ref}
placeholder="e.g., my-langgraph-agent"
placeholder={i18n.t("agents.dynamicFields.agentName.placeholder")}
value={typeof value === "string" ? value : ""}
onChange={onChange}
/>
@ -124,14 +132,17 @@ const DynamicAgentFormFields: React.FC<DynamicAgentFormFieldsProps> = ({ agentTy
<AgentFormField
name="description"
label={labelWithHint("Description", "Brief description of what this agent does")}
label={labelWithHint(
i18n.t("agents.dynamicFields.description.label"),
i18n.t("agents.dynamicFields.description.hint"),
)}
>
{({ value, onChange, ref, ...control }) => (
<Textarea
{...control}
ref={ref}
rows={2}
placeholder="Describe what this agent does..."
placeholder={i18n.t("agents.dynamicFields.description.placeholder")}
value={typeof value === "string" ? value : ""}
onChange={onChange}
/>
@ -144,10 +155,18 @@ const DynamicAgentFormFields: React.FC<DynamicAgentFormFieldsProps> = ({ agentTy
</FieldGroup>
<div className="mb-4 rounded-md border border-border px-4">
<AgentFormPanel panelKey={AGENT_FORM_CONFIG.cost.key} title={AGENT_FORM_CONFIG.cost.title} panels={panels}>
<AgentFormPanel
panelKey={AGENT_FORM_CONFIG.cost.key}
title={i18n.t(AGENT_FORM_CONFIG.cost.titleKey)}
panels={panels}
>
<CostConfigFields />
</AgentFormPanel>
<AgentFormPanel panelKey={KILL_SWITCH_PANEL_KEY} title="Kill Switch" panels={panels}>
<AgentFormPanel
panelKey={KILL_SWITCH_PANEL_KEY}
title={i18n.t("agents.dynamicFields.killSwitch")}
panels={panels}
>
<KillSwitchFormFields />
</AgentFormPanel>
</div>

View file

@ -1,3 +1,5 @@
import type { ParseKeys, TFunction } from "i18next";
import i18n from "@/i18n/i18n";
import type { components } from "@/lib/http/schema";
export type KillSwitchConfig = components["schemas"]["AgentKillSwitchConfig"];
@ -6,11 +8,11 @@ export type KillSwitchMethod = NonNullable<KillSwitchConfig["method"]>;
export type KillSwitchAuthType = KillSwitchAuth["type"] | "none";
export const KILL_SWITCH_METHODS: readonly KillSwitchMethod[] = ["POST", "PUT", "PATCH", "DELETE", "GET"];
export const KILL_SWITCH_AUTH_TYPES: readonly { value: KillSwitchAuthType; label: string }[] = [
{ value: "none", label: "None" },
{ value: "bearer", label: "Bearer token" },
{ value: "api_key", label: "API key header" },
{ value: "basic", label: "Basic auth" },
export const KILL_SWITCH_AUTH_TYPES: readonly { value: KillSwitchAuthType; labelKey: ParseKeys }[] = [
{ value: "none", labelKey: "agents.killSwitchConfig.authTypes.none" },
{ value: "bearer", labelKey: "agents.killSwitchConfig.authTypes.bearer" },
{ value: "api_key", labelKey: "agents.killSwitchConfig.authTypes.apiKey" },
{ value: "basic", labelKey: "agents.killSwitchConfig.authTypes.basic" },
];
export interface KeyValueFormValue {
@ -61,12 +63,12 @@ export const parseKillSwitchBody = (text: string | undefined): Record<string, un
return parsed as Record<string, unknown>;
};
export const validateKillSwitchBody = (text: string | undefined): true | string => {
export const validateKillSwitchBody = (text: string | undefined, t: TFunction = i18n.t.bind(i18n)): true | string => {
try {
parseKillSwitchBody(text);
return true;
} catch (error) {
return error instanceof Error ? error.message : "Body must be valid JSON";
return error instanceof Error ? error.message : t("agents.killSwitchConfig.errors.invalidJson");
}
};

View file

@ -2,6 +2,7 @@
import React from "react";
import { X } from "lucide-react";
import { useTranslation } from "react-i18next";
import { BarChart } from "@/components/shared/charts";
import type { ChartTooltipProps } from "@/components/shared/charts/chart_tooltip";
import { Button } from "@/components/ui/button";
@ -40,6 +41,7 @@ export const groupErrorBuckets = (buckets: readonly CacheActivityErrorBucket[],
};
export const ErrorCodeTooltip = ({ active, payload, label }: ChartTooltipProps) => {
const { t } = useTranslation();
if (!active || !payload || payload.length === 0) return null;
const datum = payload[0]?.payload as ErrorCodeDatum | undefined;
if (!datum) return null;
@ -47,7 +49,10 @@ export const ErrorCodeTooltip = ({ active, payload, label }: ChartTooltipProps)
return (
<div className="min-w-40 rounded-lg border border-border/50 bg-background px-2.5 py-1.5 text-xs shadow-xl">
<p className="mb-1.5 font-medium text-foreground">
Error code {String(label)}: {datum[FAILED_REQUESTS_SERIES].toLocaleString()} failed
{t("caching.errorDrilldown.tooltipTitle", {
code: String(label),
total: datum[FAILED_REQUESTS_SERIES].toLocaleString(),
})}
</p>
<div className="grid gap-1.5">
{datum.classes.map((errorClass) => (
@ -70,27 +75,33 @@ interface ErrorDrilldownCardProps {
onClose: () => void;
}
export const ErrorDrilldownCard = ({ callType, buckets, valueFormatter, onClose }: ErrorDrilldownCardProps) => (
<Card className="mt-4">
<CardHeader className="flex flex-row items-center justify-between">
<CardTitle className="text-base font-semibold">Failed requests by error code: {callType}</CardTitle>
<Button variant="outline" size="icon-sm" onClick={onClose} aria-label="Close error breakdown">
<X />
</Button>
</CardHeader>
<CardContent>
<p className="text-sm text-muted-foreground">Hover a bar to see the error classes behind that code.</p>
<BarChart
data={groupErrorBuckets(buckets, callType)}
index="error_code"
categories={[FAILED_REQUESTS_SERIES]}
colors={["red"]}
valueFormatter={valueFormatter}
showLegend={false}
customTooltip={ErrorCodeTooltip}
yAxisWidth={48}
className="mt-2"
/>
</CardContent>
</Card>
);
export const ErrorDrilldownCard = ({ callType, buckets, valueFormatter, onClose }: ErrorDrilldownCardProps) => {
const { t } = useTranslation();
return (
<Card className="mt-4">
<CardHeader className="flex flex-row items-center justify-between">
<CardTitle className="text-base font-semibold">
{t("caching.errorDrilldown.title")}
{callType}
</CardTitle>
<Button variant="outline" size="icon-sm" onClick={onClose} aria-label={t("caching.errorDrilldown.close")}>
<X />
</Button>
</CardHeader>
<CardContent>
<p className="text-sm text-muted-foreground">{t("caching.errorDrilldown.hint")}</p>
<BarChart
data={groupErrorBuckets(buckets, callType)}
index="error_code"
categories={[FAILED_REQUESTS_SERIES]}
colors={["red"]}
valueFormatter={valueFormatter}
showLegend={false}
customTooltip={ErrorCodeTooltip}
yAxisWidth={48}
className="mt-2"
/>
</CardContent>
</Card>
);
};

View file

@ -1,5 +1,7 @@
import type { DateRangePickerValue } from "@/components/shared/date_picker_types";
import type { TFunction } from "i18next";
import React, { useEffect, useState } from "react";
import { Trans, useTranslation } from "react-i18next";
import { toast } from "@/lib/toast";
import AdvancedDatePicker from "@/components/shared/advanced_date_picker";
import { BarChart } from "@/components/shared/charts";
@ -32,23 +34,22 @@ import CoordinationRedisSettings from "./coordination_redis_settings";
import { ErrorDrilldownCard } from "./ErrorDrilldown";
const REQUEST_SERIES = {
apiRequests: "LLM API requests",
cacheHits: "Cache hit",
failed: "Failed requests",
apiRequests: "caching.dashboard.series.apiRequests",
cacheHits: "caching.dashboard.series.cacheHits",
failed: "caching.dashboard.series.failed",
} as const;
const UNKNOWN_CALL_TYPE = "Unknown";
const UNKNOWN_CALL_TYPE_NOTE =
"Unknown groups spend logs that recorded no endpoint. Older proxy versions wrote those for requests rejected before routing, so they are not necessarily LLM API requests.";
const UNKNOWN_CALL_TYPE_NOTE = "caching.dashboard.requestsChart.unknownCallTypeNote";
const toChartDatum = (group: CacheActivityGroup) => ({
const toChartDatum = (group: CacheActivityGroup, t: TFunction) => ({
name: group.call_type,
[REQUEST_SERIES.apiRequests]: group.api_requests,
[REQUEST_SERIES.cacheHits]: group.cache_hits,
[REQUEST_SERIES.failed]: group.failed_requests,
"Cached Completion Tokens": group.cached_completion_tokens,
"Generated Completion Tokens": group.generated_completion_tokens,
[t(REQUEST_SERIES.apiRequests)]: group.api_requests,
[t(REQUEST_SERIES.cacheHits)]: group.cache_hits,
[t(REQUEST_SERIES.failed)]: group.failed_requests,
[t("caching.dashboard.series.cachedCompletionTokens")]: group.cached_completion_tokens,
[t("caching.dashboard.series.generatedCompletionTokens")]: group.generated_completion_tokens,
});
const formatDateWithoutTZ = (date: Date | undefined) => {
@ -82,6 +83,7 @@ interface CachePageProps {
// Helper function to deep-parse a JSON string if possible
const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole, userID, premiumUser }) => {
const { t } = useTranslation();
const anchor1 = useComboboxAnchor();
const anchor2 = useComboboxAnchor();
const [selectedApiKeys, setSelectedApiKeys] = useState<string[]>([]);
@ -114,7 +116,7 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
const uniqueApiKeys = activity?.filter_options.key_aliases ?? [];
const uniqueModels = activity?.filter_options.models ?? [];
const chartData = (activity?.groups ?? []).map(toChartDatum);
const chartData = (activity?.groups ?? []).map((group) => toChartDatum(group, t));
const hasUnknownGroup = (activity?.groups ?? []).some((group) => group.call_type === UNKNOWN_CALL_TYPE);
const activeDrilldownCallType = resolveDrilldownCallType(errorDrilldownCallType, activity?.groups ?? []);
@ -125,7 +127,7 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
const runCachingHealthCheck = async () => {
try {
toast.info("Running cache health check...");
toast.info(t("caching.dashboard.healthCheck.running"));
setHealthCheckResponse("");
const response = await cachingHealthCheckCall(accessToken !== null ? accessToken : "");
setHealthCheckResponse(response);
@ -145,7 +147,7 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
errorData = { message: error.message };
}
} else {
errorData = { message: "Unknown error occurred" };
errorData = { message: t("caching.dashboard.healthCheck.unknownError") };
}
setHealthCheckResponse({ error: errorData });
}
@ -154,9 +156,15 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
const totals = activity?.totals;
const hasRequests = totals != null && totals.api_requests + totals.cache_hits + totals.failed_requests > 0;
const statCards = [
{ label: "Cache Hit Ratio", value: `${hasRequests ? totals.cache_hit_ratio.toFixed(2) : "0"}%` },
{ label: "Cache Hits", value: valueFormatterNumbers(totals?.cache_hits ?? 0) },
{ label: "Cached Completion Tokens", value: valueFormatterNumbers(totals?.cached_completion_tokens ?? 0) },
{
label: t("caching.dashboard.stats.cacheHitRatio"),
value: `${hasRequests ? totals.cache_hit_ratio.toFixed(2) : "0"}%`,
},
{ label: t("caching.dashboard.stats.cacheHits"), value: valueFormatterNumbers(totals?.cache_hits ?? 0) },
{
label: t("caching.dashboard.stats.cachedCompletionTokens"),
value: valueFormatterNumbers(totals?.cached_completion_tokens ?? 0),
},
];
return (
@ -164,22 +172,32 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
<div className="mt-2 flex w-full items-center justify-between border-b">
<TabsList variant="line" className="h-auto rounded-none p-0">
<TabsTrigger value="analytics" className="flex-none rounded-none px-4 py-2">
Cache Analytics
{t("caching.dashboard.tabs.analytics")}
</TabsTrigger>
<TabsTrigger value="health" className="flex-none rounded-none px-4 py-2">
Cache Health
{t("caching.dashboard.tabs.health")}
</TabsTrigger>
<TabsTrigger value="settings" className="flex-none rounded-none px-4 py-2">
Cache Settings
{t("caching.dashboard.tabs.settings")}
</TabsTrigger>
<TabsTrigger value="coordination" className="flex-none rounded-none px-4 py-2">
Coordination Redis
{t("caching.dashboard.tabs.coordination")}
</TabsTrigger>
</TabsList>
<div className="flex items-center space-x-2">
{lastRefreshed && <p className="text-sm text-muted-foreground">Last Refreshed: {lastRefreshed}</p>}
<Button variant="outline" size="icon-sm" onClick={handleRefreshClick} aria-label="Refresh">
{lastRefreshed && (
<p className="text-sm text-muted-foreground">
{t("caching.dashboard.lastRefreshed")}
{lastRefreshed}
</p>
)}
<Button
variant="outline"
size="icon-sm"
onClick={handleRefreshClick}
aria-label={t("caching.dashboard.refresh")}
>
<RefreshCw />
</Button>
</div>
@ -189,26 +207,27 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
<Card>
<CardContent>
<p className="text-sm text-muted-foreground">
Analytics for LiteLLM&apos;s{" "}
<a
href="https://docs.litellm.ai/docs/proxy/caching"
target="_blank"
rel="noreferrer"
className="underline"
>
response cache
</a>{" "}
(e.g. Redis / in-memory): requests answered from cache without calling the LLM provider. Provider-side{" "}
<a
href="https://docs.litellm.ai/docs/completion/prompt_caching"
target="_blank"
rel="noreferrer"
className="underline"
>
prompt caching
</a>{" "}
(cached input tokens from Anthropic, OpenAI, etc.) is not shown here; see &quot;Prompt Caching
Metrics&quot; on the Usage page or individual requests in the Logs page.
<Trans
i18nKey="caching.dashboard.description"
components={{
cacheDocs: (
<a
href="https://docs.litellm.ai/docs/proxy/caching"
target="_blank"
rel="noreferrer"
className="underline"
/>
),
promptDocs: (
<a
href="https://docs.litellm.ai/docs/completion/prompt_caching"
target="_blank"
rel="noreferrer"
className="underline"
/>
),
}}
/>
</p>
<div className="mt-4 grid grid-cols-1 items-center gap-4 md:grid-cols-[1fr_1fr_auto]">
@ -228,10 +247,10 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
))
}
</ComboboxValue>
<ComboboxChipsInput placeholder="Select Virtual Keys" />
<ComboboxChipsInput placeholder={t("caching.dashboard.filters.virtualKeysPlaceholder")} />
</ComboboxChips>
<ComboboxContent anchor={anchor1}>
<ComboboxEmpty>No virtual keys found</ComboboxEmpty>
<ComboboxEmpty>{t("caching.dashboard.filters.noVirtualKeys")}</ComboboxEmpty>
<ComboboxList>
{(key: string) => (
<ComboboxItem key={key} value={key}>
@ -258,10 +277,10 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
))
}
</ComboboxValue>
<ComboboxChipsInput placeholder="Select Models" />
<ComboboxChipsInput placeholder={t("caching.dashboard.filters.modelsPlaceholder")} />
</ComboboxChips>
<ComboboxContent anchor={anchor2}>
<ComboboxEmpty>No models found</ComboboxEmpty>
<ComboboxEmpty>{t("caching.dashboard.filters.noModels")}</ComboboxEmpty>
<ComboboxList>
{(model: string) => (
<ComboboxItem key={model} value={model}>
@ -295,16 +314,14 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
<Card className="mt-4">
<CardHeader>
<CardTitle className="text-base font-semibold">Cache Hits vs API Requests</CardTitle>
<CardTitle className="text-base font-semibold">{t("caching.dashboard.requestsChart.title")}</CardTitle>
</CardHeader>
<CardContent>
<p className="text-sm text-muted-foreground">
Click a red failed-requests segment to see which error codes caused those failures.
</p>
{hasUnknownGroup && <p className="mt-1 text-sm text-muted-foreground">{UNKNOWN_CALL_TYPE_NOTE}</p>}
<p className="text-sm text-muted-foreground">{t("caching.dashboard.requestsChart.hint")}</p>
{hasUnknownGroup && <p className="mt-1 text-sm text-muted-foreground">{t(UNKNOWN_CALL_TYPE_NOTE)}</p>}
{activityFailed ? (
<ErrorState
title="Failed to load cache activity"
title={t("caching.dashboard.requestsChart.loadFailed")}
message={deriveErrorMessage(activityError)}
onRetry={() => void refetch()}
/>
@ -314,12 +331,12 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
stack={true}
index="name"
valueFormatter={valueFormatterNumbers}
categories={[REQUEST_SERIES.apiRequests, REQUEST_SERIES.cacheHits, REQUEST_SERIES.failed]}
categories={[t(REQUEST_SERIES.apiRequests), t(REQUEST_SERIES.cacheHits), t(REQUEST_SERIES.failed)]}
colors={["sky", "teal", "red"]}
yAxisWidth={48}
className="mt-2"
onValueChange={(item) => {
if (item.categoryClicked === REQUEST_SERIES.failed) setErrorDrilldownCallType(item.name);
if (item.categoryClicked === t(REQUEST_SERIES.failed)) setErrorDrilldownCallType(item.name);
}}
/>
)}
@ -337,9 +354,7 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
<Card className="mt-6">
<CardHeader>
<CardTitle className="text-base font-semibold">
Cached Completion Tokens vs Generated Completion Tokens
</CardTitle>
<CardTitle className="text-base font-semibold">{t("caching.dashboard.tokensChart.title")}</CardTitle>
</CardHeader>
<CardContent>
<BarChart
@ -347,7 +362,10 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
stack={true}
index="name"
valueFormatter={valueFormatterNumbers}
categories={["Generated Completion Tokens", "Cached Completion Tokens"]}
categories={[
t("caching.dashboard.series.generatedCompletionTokens"),
t("caching.dashboard.series.cachedCompletionTokens"),
]}
colors={["sky", "teal"]}
yAxisWidth={48}
/>

View file

@ -1,4 +1,5 @@
import React from "react";
import { useTranslation } from "react-i18next";
import { Button } from "@/components/ui/button";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { CheckCircle2, XCircle, ClipboardCopy } from "lucide-react";
@ -19,8 +20,9 @@ const deepParse = (input: any) => {
// TableClickableErrorField component with copy-to-clipboard functionality
const TableClickableErrorField: React.FC<{ label: string; value: string | null | undefined }> = ({ label, value }) => {
const { t } = useTranslation();
const [isExpanded, setIsExpanded] = React.useState(false);
const safeValue = value?.toString() || "N/A";
const safeValue = value?.toString() || t("caching.health.notAvailable");
const truncated = safeValue.length > 50 ? safeValue.substring(0, 50) + "..." : safeValue;
const handleCopy = () => {
@ -74,6 +76,7 @@ interface ErrorDetails {
// Update HealthCheckDetails component to handle errors
const HealthCheckDetails: React.FC<{ response: any }> = ({ response }) => {
const { t } = useTranslation();
// Initialize with safe default values
let errorDetails: ErrorDetails | null = null;
let parsedLitellmParams: any = {};
@ -86,8 +89,8 @@ const HealthCheckDetails: React.FC<{ response: any }> = ({ response }) => {
typeof response.error.message === "string" ? JSON.parse(response.error.message) : response.error.message;
errorDetails = {
message: errorMessage?.message || "Unknown error",
traceback: errorMessage?.traceback || "No traceback available",
message: errorMessage?.message || t("caching.health.unknownError"),
traceback: errorMessage?.traceback || t("caching.health.noTraceback"),
litellm_params: errorMessage?.litellm_cache_params || {},
health_check_cache_params: errorMessage?.health_check_cache_params || {},
};
@ -97,8 +100,8 @@ const HealthCheckDetails: React.FC<{ response: any }> = ({ response }) => {
} catch (e) {
console.warn("Error parsing error details:", e);
errorDetails = {
message: String(response.error.message || "Unknown error"),
traceback: "Error parsing details",
message: String(response.error.message || t("caching.health.unknownError")),
traceback: t("caching.health.parseFailed"),
litellm_params: {},
health_check_cache_params: {},
};
@ -121,16 +124,16 @@ const HealthCheckDetails: React.FC<{ response: any }> = ({ response }) => {
parsedRedisParams?.redis_async_client?.connection_pool?.connection_kwargs?.host ||
parsedRedisParams?.connection_kwargs?.host ||
parsedRedisParams?.host ||
"N/A",
t("caching.health.notAvailable"),
redis_port:
parsedRedisParams?.redis_client?.connection_pool?.connection_kwargs?.port ||
parsedRedisParams?.redis_async_client?.connection_pool?.connection_kwargs?.port ||
parsedRedisParams?.connection_kwargs?.port ||
parsedRedisParams?.port ||
"N/A",
t("caching.health.notAvailable"),
redis_version: parsedRedisParams?.redis_version || "N/A",
redis_version: parsedRedisParams?.redis_version || t("caching.health.notAvailable"),
startup_nodes: (() => {
try {
@ -143,13 +146,13 @@ const HealthCheckDetails: React.FC<{ response: any }> = ({ response }) => {
const port =
parsedRedisParams?.redis_client?.connection_pool?.connection_kwargs?.port ||
parsedRedisParams?.redis_async_client?.connection_pool?.connection_kwargs?.port;
return host && port ? JSON.stringify([{ host, port }]) : "N/A";
return host && port ? JSON.stringify([{ host, port }]) : t("caching.health.notAvailable");
} catch (e) {
return "N/A";
return t("caching.health.notAvailable");
}
})(),
namespace: parsedRedisParams?.namespace || "N/A",
namespace: parsedRedisParams?.namespace || t("caching.health.notAvailable"),
};
return (
@ -157,10 +160,10 @@ const HealthCheckDetails: React.FC<{ response: any }> = ({ response }) => {
<Tabs defaultValue="summary">
<TabsList className="border-b border-border px-4">
<TabsTrigger value="summary" className="flex-none">
Summary
{t("caching.health.tabs.summary")}
</TabsTrigger>
<TabsTrigger value="raw" className="flex-none">
Raw Response
{t("caching.health.tabs.raw")}
</TabsTrigger>
</TabsList>
@ -175,7 +178,8 @@ const HealthCheckDetails: React.FC<{ response: any }> = ({ response }) => {
<p
className={`text-sm font-medium ${response?.status === "healthy" ? "text-success" : "text-destructive"}`}
>
Cache Status: {response?.status || "unhealthy"}
{t("caching.health.cacheStatus")}
{response?.status || "unhealthy"}
</p>
</div>
@ -186,23 +190,32 @@ const HealthCheckDetails: React.FC<{ response: any }> = ({ response }) => {
<>
<tr>
<td colSpan={2} className="pt-4 pb-2 font-semibold text-destructive">
Error Details
{t("caching.health.errorDetails")}
</td>
</tr>
<TableClickableErrorField label="Error Message" value={errorDetails.message} />
<TableClickableErrorField label="Traceback" value={errorDetails.traceback} />
<TableClickableErrorField label={t("caching.health.errorMessage")} value={errorDetails.message} />
<TableClickableErrorField label={t("caching.health.traceback")} value={errorDetails.traceback} />
</>
)}
{/* Always show cache details, regardless of error state */}
<tr>
<td colSpan={2} className="pt-4 pb-2 font-semibold">
Cache Details
{t("caching.health.cacheDetails")}
</td>
</tr>
<TableClickableErrorField label="Cache Configuration" value={String(parsedLitellmParams?.type)} />
<TableClickableErrorField label="Ping Response" value={String(response.ping_response)} />
<TableClickableErrorField label="Set Cache Response" value={response.set_cache_response || "N/A"} />
<TableClickableErrorField
label={t("caching.health.cacheConfiguration")}
value={String(parsedLitellmParams?.type)}
/>
<TableClickableErrorField
label={t("caching.health.pingResponse")}
value={String(response.ping_response)}
/>
<TableClickableErrorField
label={t("caching.health.setCacheResponse")}
value={response.set_cache_response || t("caching.health.notAvailable")}
/>
<TableClickableErrorField
label="litellm_settings.cache_params"
value={JSON.stringify(parsedLitellmParams, null, 2)}
@ -213,14 +226,29 @@ const HealthCheckDetails: React.FC<{ response: any }> = ({ response }) => {
<>
<tr>
<td colSpan={2} className="pt-4 pb-2 font-semibold">
Redis Details
{t("caching.health.redisDetails")}
</td>
</tr>
<TableClickableErrorField label="Redis Host" value={redisDetails.redis_host || "N/A"} />
<TableClickableErrorField label="Redis Port" value={redisDetails.redis_port || "N/A"} />
<TableClickableErrorField label="Redis Version" value={redisDetails.redis_version || "N/A"} />
<TableClickableErrorField label="Startup Nodes" value={redisDetails.startup_nodes || "N/A"} />
<TableClickableErrorField label="Namespace" value={redisDetails.namespace || "N/A"} />
<TableClickableErrorField
label={t("caching.health.redisHost")}
value={redisDetails.redis_host || t("caching.health.notAvailable")}
/>
<TableClickableErrorField
label={t("caching.health.redisPort")}
value={redisDetails.redis_port || t("caching.health.notAvailable")}
/>
<TableClickableErrorField
label={t("caching.health.redisVersion")}
value={redisDetails.redis_version || t("caching.health.notAvailable")}
/>
<TableClickableErrorField
label={t("caching.health.startupNodes")}
value={redisDetails.startup_nodes || t("caching.health.notAvailable")}
/>
<TableClickableErrorField
label={t("caching.health.namespace")}
value={redisDetails.namespace || t("caching.health.notAvailable")}
/>
</>
)}
</tbody>
@ -254,7 +282,7 @@ const HealthCheckDetails: React.FC<{ response: any }> = ({ response }) => {
// Then stringify with proper formatting
return JSON.stringify(prettyData, null, 2);
} catch (e) {
return "Error formatting JSON: " + (e as Error).message;
return t("caching.health.formatFailed", { message: (e as Error).message });
}
})()}
</pre>
@ -271,6 +299,7 @@ export const CacheHealthTab: React.FC<{
runCachingHealthCheck: () => void;
responseTimeMs?: number | null;
}> = ({ accessToken, healthCheckResponse, runCachingHealthCheck, responseTimeMs }) => {
const { t } = useTranslation();
const [localResponseTimeMs, setLocalResponseTimeMs] = React.useState<number | null>(null);
const [isLoading, setIsLoading] = React.useState<boolean>(false);
@ -287,7 +316,7 @@ export const CacheHealthTab: React.FC<{
<div className="space-y-4">
<div className="flex items-center justify-between">
<Button onClick={handleHealthCheck} disabled={isLoading}>
{isLoading ? "Running Health Check..." : "Run Health Check"}
{isLoading ? t("caching.health.running") : t("caching.health.runButton")}
</Button>
<ResponseTimeIndicator responseTimeMs={localResponseTimeMs} />
</div>

View file

@ -1,5 +1,6 @@
import React from "react";
import { useFormContext } from "react-hook-form";
import { useTranslation } from "react-i18next";
import { FormField } from "@/components/shared/form/FormField";
import { PasswordInput } from "@/components/shared/PasswordInput";
import {
@ -23,7 +24,7 @@ export interface EmbeddingModelOption {
label: string;
}
export const SECRET_ALREADY_SET_PLACEHOLDER = "Already set. Enter a new value to replace it.";
export const SECRET_ALREADY_SET_PLACEHOLDER = "caching.settings.field.secretAlreadySet";
interface CacheFormFieldProps {
field: CacheField;
@ -32,11 +33,12 @@ interface CacheFormFieldProps {
}
const CacheFormField: React.FC<CacheFormFieldProps> = ({ field, embeddingModels, isSecretConfigured = false }) => {
const { t } = useTranslation();
const form = useFormContext<CacheFormValues>();
const placeholder = isSecretConfigured ? SECRET_ALREADY_SET_PLACEHOLDER : field.helpText;
const placeholder = isSecretConfigured ? t(SECRET_ALREADY_SET_PLACEHOLDER) : t(field.helpText);
return (
<FormField control={form.control} name={field.name} label={field.label} description={field.helpText}>
<FormField control={form.control} name={field.name} label={t(field.label)} description={t(field.helpText)}>
{({ ref, value, onChange, ...rest }) => {
if (field.type === "boolean") {
return <Switch {...rest} checked={value === true} onCheckedChange={(checked) => onChange(checked)} />;
@ -70,7 +72,7 @@ const CacheFormField: React.FC<CacheFormFieldProps> = ({ field, embeddingModels,
const { id, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy, name, onBlur, disabled } = rest;
return (
<Select
items={options.map((option) => ({ label: option.label, value: option.value }))}
items={options.map((option) => ({ label: t(option.label), value: option.value }))}
name={name}
disabled={disabled}
value={typeof value === "string" && value !== "" ? value : null}
@ -83,12 +85,12 @@ const CacheFormField: React.FC<CacheFormFieldProps> = ({ field, embeddingModels,
onBlur={onBlur}
className="w-full"
>
<SelectValue placeholder="Select an option" />
<SelectValue placeholder={t("caching.settings.field.selectPlaceholder")} />
</SelectTrigger>
<SelectContent>
{options.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
{t(option.label)}
</SelectItem>
))}
</SelectContent>
@ -107,11 +109,15 @@ const CacheFormField: React.FC<CacheFormFieldProps> = ({ field, embeddingModels,
model.value === other.value
}
>
<ComboboxInput {...rest} placeholder="Search and select a model..." className="w-full">
<ComboboxInput
{...rest}
placeholder={t("caching.settings.field.modelSearchPlaceholder")}
className="w-full"
>
<ComboboxClear />
</ComboboxInput>
<ComboboxContent>
<ComboboxEmpty>No models found</ComboboxEmpty>
<ComboboxEmpty>{t("caching.settings.field.noModels")}</ComboboxEmpty>
<ComboboxList>
{(model: EmbeddingModelOption) => (
<ComboboxItem key={model.value} value={model} title={model.label}>

View file

@ -1,37 +1,45 @@
import type { ParseKeys } from "i18next";
import React from "react";
import { useTranslation } from "react-i18next";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
interface RedisTypeSelectorProps {
redisType: string;
redisTypeDescriptions: Readonly<Record<string, string>>;
redisTypeDescriptions: Readonly<Record<string, ParseKeys>>;
onTypeChange: (type: string) => void;
}
const REDIS_TYPE_LABELS: Readonly<Record<string, string>> = {
node: "Node (Single Instance)",
cluster: "Cluster",
sentinel: "Sentinel",
semantic: "Semantic",
const REDIS_TYPE_LABELS: Readonly<Record<string, ParseKeys>> = {
node: "caching.settings.redisType.options.node",
cluster: "caching.settings.redisType.options.cluster",
sentinel: "caching.settings.redisType.options.sentinel",
semantic: "caching.settings.redisType.options.semantic",
};
const RedisTypeSelector: React.FC<RedisTypeSelectorProps> = ({ redisType, redisTypeDescriptions, onTypeChange }) => {
const { t } = useTranslation();
return (
<div className="space-y-2">
<label className="text-sm font-medium">Redis Type</label>
<label className="text-sm font-medium">{t("caching.settings.redisType.label")}</label>
<Select value={redisType} onValueChange={(value) => value !== null && onTypeChange(value)}>
<SelectTrigger className="w-full">
<SelectValue>{REDIS_TYPE_LABELS[redisType] ?? redisType}</SelectValue>
<SelectValue>
{Object.hasOwn(REDIS_TYPE_LABELS, redisType) ? t(REDIS_TYPE_LABELS[redisType]) : redisType}
</SelectValue>
</SelectTrigger>
<SelectContent>
{Object.entries(REDIS_TYPE_LABELS).map(([value, label]) => (
<SelectItem key={value} value={value}>
{label}
{t(label)}
</SelectItem>
))}
</SelectContent>
</Select>
<p className="text-xs text-muted-foreground">
{redisTypeDescriptions[redisType] || "Select the type of Redis deployment you're using"}
{t(
(Object.hasOwn(redisTypeDescriptions, redisType) && redisTypeDescriptions[redisType]) ||
"caching.settings.redisType.fallbackDescription",
)}
</p>
</div>
);

View file

@ -1,3 +1,5 @@
import type { ParseKeys } from "i18next";
export type CacheFieldType =
| "string"
| "password"
@ -10,14 +12,14 @@ export type CacheFieldType =
export interface CacheFieldOption {
readonly value: string;
readonly label: string;
readonly label: ParseKeys;
}
export type RedisType = "node" | "cluster" | "sentinel" | "semantic";
export type CacheSection = "connection" | "cluster" | "sentinel" | "semantic" | "ssl" | "cacheManagement" | "gcp";
export type CacheFieldRule = (value: unknown) => string | null;
export type CacheFieldRule = (value: unknown) => ParseKeys | null;
// Marker the backend returns for a configured credential and maps back to the
// stored secret on save, so the plaintext never round-trips through the form.
@ -25,10 +27,10 @@ export const REDACTED_VALUE = "***REDACTED***";
export interface CacheField {
readonly name: string;
readonly label: string;
readonly label: ParseKeys;
readonly type: CacheFieldType;
readonly section: CacheSection;
readonly helpText: string;
readonly helpText: ParseKeys;
readonly redisType: RedisType | null;
readonly defaultValue?: string | number | boolean;
readonly options?: readonly CacheFieldOption[];
@ -40,11 +42,11 @@ export interface CacheField {
export const REDIS_TYPES: readonly RedisType[] = ["node", "cluster", "sentinel", "semantic"];
export const REDIS_TYPE_DESCRIPTIONS: Readonly<Record<RedisType, string>> = {
node: "Standard Redis node/single instance",
cluster: "Redis Cluster mode for high availability and horizontal scaling",
sentinel: "Redis Sentinel mode for high availability with automatic failover",
semantic: "Semantic caching that reuses responses for similar prompts",
export const REDIS_TYPE_DESCRIPTIONS: Readonly<Record<RedisType, ParseKeys>> = {
node: "caching.settings.redisType.descriptions.node",
cluster: "caching.settings.redisType.descriptions.cluster",
sentinel: "caching.settings.redisType.descriptions.sentinel",
semantic: "caching.settings.redisType.descriptions.semantic",
};
const isBlank = (value: unknown): boolean => value === undefined || value === null || String(value).trim() === "";
@ -54,7 +56,7 @@ const portRule: CacheFieldRule = (value) => {
return null;
}
const port = Number(value);
return Number.isInteger(port) && port >= 1 && port <= 65535 ? null : "Port must be an integer between 1 and 65535";
return Number.isInteger(port) && port >= 1 && port <= 65535 ? null : "caching.settings.validation.port";
};
const jsonListRule: CacheFieldRule = (value) => {
@ -65,9 +67,9 @@ const jsonListRule: CacheFieldRule = (value) => {
try {
parsed = JSON.parse(String(value));
} catch {
return "Must be a valid JSON array (use double quotes)";
return "caching.settings.validation.jsonArrayInvalid";
}
return Array.isArray(parsed) ? null : "Must be a JSON array";
return Array.isArray(parsed) ? null : "caching.settings.validation.jsonArrayRequired";
};
const nonNegativeIntegerRule: CacheFieldRule = (value) => {
@ -75,205 +77,202 @@ const nonNegativeIntegerRule: CacheFieldRule = (value) => {
return null;
}
const parsed = Number(value);
return Number.isInteger(parsed) && parsed >= 0 ? null : "Must be a non-negative integer";
return Number.isInteger(parsed) && parsed >= 0 ? null : "caching.settings.validation.nonNegativeInteger";
};
const numberRule: CacheFieldRule = (value) => {
if (isBlank(value)) {
return null;
}
return Number.isNaN(Number(value)) ? "Must be a number" : null;
return Number.isNaN(Number(value)) ? "caching.settings.validation.number" : null;
};
export const CACHE_FIELDS: readonly CacheField[] = [
{
name: "url",
label: "Redis URL",
label: "caching.settings.fields.url.label",
type: "string",
section: "connection",
helpText:
"Full Redis/Valkey connection URL (e.g. redis://:password@host:6379/1). When set, it takes precedence over Host, Port, Password, and Database Index.",
helpText: "caching.settings.fields.url.helpText",
redisType: null,
secret: true,
},
{
name: "host",
label: "Host",
label: "caching.settings.fields.host.label",
type: "string",
section: "connection",
helpText: "Redis server hostname or IP address",
helpText: "caching.settings.fields.host.helpText",
redisType: null,
},
{
name: "port",
label: "Port",
label: "caching.settings.fields.port.label",
type: "string",
section: "connection",
helpText: "Redis server port number",
helpText: "caching.settings.fields.port.helpText",
redisType: null,
defaultValue: "6379",
rules: [portRule],
},
{
name: "db",
label: "Database Index",
label: "caching.settings.fields.db.label",
type: "integer",
section: "connection",
helpText: "Logical database index to isolate the cache (e.g. 1 for redis://host:6379/1)",
helpText: "caching.settings.fields.db.helpText",
redisType: null,
rules: [nonNegativeIntegerRule],
},
{
name: "password",
label: "Password",
label: "caching.settings.fields.password.label",
type: "password",
section: "connection",
helpText: "Redis server password",
helpText: "caching.settings.fields.password.helpText",
redisType: null,
secret: true,
},
{
name: "username",
label: "Username",
label: "caching.settings.fields.username.label",
type: "string",
section: "connection",
helpText: "Redis server username (if required)",
helpText: "caching.settings.fields.username.helpText",
redisType: null,
},
{
name: "redis_startup_nodes",
label: "Startup Nodes",
label: "caching.settings.fields.redisStartupNodes.label",
type: "list",
section: "cluster",
helpText: 'List of startup nodes for Redis Cluster (e.g., [{"host": "127.0.0.1", "port": "7001"}])',
helpText: "caching.settings.fields.redisStartupNodes.helpText",
redisType: "cluster",
rules: [jsonListRule],
},
{
name: "sentinel_nodes",
label: "Sentinel Nodes",
label: "caching.settings.fields.sentinelNodes.label",
type: "list",
section: "sentinel",
helpText: 'List of Sentinel nodes (e.g., [["localhost", 26379]])',
helpText: "caching.settings.fields.sentinelNodes.helpText",
redisType: "sentinel",
rules: [jsonListRule],
},
{
name: "service_name",
label: "Service Name",
label: "caching.settings.fields.serviceName.label",
type: "string",
section: "sentinel",
helpText: "Master service name for Redis Sentinel",
helpText: "caching.settings.fields.serviceName.helpText",
redisType: "sentinel",
},
{
name: "sentinel_password",
label: "Sentinel Password",
label: "caching.settings.fields.sentinelPassword.label",
type: "password",
section: "sentinel",
helpText: "Password for Redis Sentinel authentication",
helpText: "caching.settings.fields.sentinelPassword.helpText",
redisType: "sentinel",
secret: true,
},
{
name: "similarity_threshold",
label: "Similarity Threshold",
label: "caching.settings.fields.similarityThreshold.label",
type: "float",
section: "semantic",
helpText: "Similarity threshold for semantic cache",
helpText: "caching.settings.fields.similarityThreshold.helpText",
redisType: "semantic",
defaultValue: 0.8,
rules: [numberRule],
},
{
name: "redis_semantic_cache_embedding_model",
label: "Embedding Model",
label: "caching.settings.fields.embeddingModel.label",
type: "model-select",
section: "semantic",
helpText: "Embedding model for semantic cache",
helpText: "caching.settings.fields.embeddingModel.helpText",
redisType: "semantic",
},
{
name: "semantic_cache_scope",
label: "Semantic Cache Scope",
label: "caching.settings.fields.semanticCacheScope.label",
type: "select",
section: "semantic",
helpText:
"Who can share a semantic cache hit. Key shares hits between all end users of a key/team/org. End user also isolates per end user; requests without an end user fall back to the key scope.",
helpText: "caching.settings.fields.semanticCacheScope.helpText",
redisType: "semantic",
defaultValue: "key",
options: [
{ value: "key", label: "Key (shared by all end users of the key/team/org)" },
{ value: "end_user", label: "End user (isolated per end user)" },
{ value: "key", label: "caching.settings.scopeOptions.key" },
{ value: "end_user", label: "caching.settings.scopeOptions.endUser" },
],
},
{
name: "ssl",
label: "SSL",
label: "caching.settings.fields.ssl.label",
type: "boolean",
section: "ssl",
helpText: "Enable SSL/TLS connection",
helpText: "caching.settings.fields.ssl.helpText",
redisType: null,
defaultValue: false,
},
{
name: "ssl_cert_reqs",
label: "SSL Cert Reqs",
label: "caching.settings.fields.sslCertReqs.label",
type: "string",
section: "ssl",
helpText: "SSL certificate requirements (None, CERT_REQUIRED, CERT_OPTIONAL)",
helpText: "caching.settings.fields.sslCertReqs.helpText",
redisType: null,
},
{
name: "ssl_check_hostname",
label: "SSL Check Hostname",
label: "caching.settings.fields.sslCheckHostname.label",
type: "boolean",
section: "ssl",
helpText: "Enable SSL hostname verification",
helpText: "caching.settings.fields.sslCheckHostname.helpText",
redisType: null,
defaultValue: false,
},
{
name: "namespace",
label: "Namespace",
label: "caching.settings.fields.namespace.label",
type: "string",
section: "cacheManagement",
helpText: "Namespace prefix for cache keys",
helpText: "caching.settings.fields.namespace.helpText",
redisType: null,
},
{
name: "ttl",
label: "TTL (seconds)",
label: "caching.settings.fields.ttl.label",
type: "float",
section: "cacheManagement",
helpText: "Time-to-live for cached items in seconds",
helpText: "caching.settings.fields.ttl.helpText",
redisType: null,
rules: [numberRule],
},
{
name: "max_connections",
label: "Max Connections",
label: "caching.settings.fields.maxConnections.label",
type: "integer",
section: "cacheManagement",
helpText: "Maximum number of connections in the connection pool",
helpText: "caching.settings.fields.maxConnections.helpText",
redisType: null,
rules: [nonNegativeIntegerRule],
},
{
name: "gcp_service_account",
label: "GCP Service Account",
label: "caching.settings.fields.gcpServiceAccount.label",
type: "string",
section: "gcp",
helpText:
"GCP service account for IAM authentication (e.g., projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com)",
helpText: "caching.settings.fields.gcpServiceAccount.helpText",
redisType: null,
},
{
name: "gcp_ssl_ca_certs",
label: "GCP SSL CA Certs",
label: "caching.settings.fields.gcpSslCaCerts.label",
type: "string",
section: "gcp",
helpText: "Path to SSL CA certificate file for GCP Memorystore Redis",
helpText: "caching.settings.fields.gcpSslCaCerts.helpText",
redisType: null,
},
];

View file

@ -1,6 +1,7 @@
import React, { useState, useEffect, useCallback } from "react";
import { ChevronRight } from "lucide-react";
import { FormProvider, useForm } from "react-hook-form";
import { useTranslation } from "react-i18next";
import { Button } from "@/components/ui/button";
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
import { getCacheSettingsCall, testCacheConnectionCall, updateCacheSettingsCall } from "@/components/networking";
@ -30,6 +31,7 @@ const toRedisType = (value: unknown): RedisType =>
REDIS_TYPES.includes(value as RedisType) ? (value as RedisType) : "node";
const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
const { t } = useTranslation();
const form = useForm<CacheFormValues>({ defaultValues: buildInitialValues({}) });
const [redisType, setRedisType] = useState<RedisType>("node");
const [advancedOpen, setAdvancedOpen] = useState(false);
@ -50,9 +52,9 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
setRedisType(toRedisType(currentValues.redis_type));
} catch (error) {
console.error("Failed to load cache settings:", error);
toast.fromError("Failed to load cache settings");
toast.fromError(t("caching.settings.toasts.loadFailed"));
}
}, [accessToken, form]);
}, [accessToken, form, t]);
useEffect(() => {
loadCacheSettings();
@ -85,7 +87,7 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
});
form.clearErrors();
failures.forEach(([name, message]) => form.setError(name, { message }));
failures.forEach(([name, message]) => form.setError(name, { message: t(message) }));
return failures.length > 0 ? null : values;
};
@ -105,13 +107,17 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
buildCachePayload(redisType, values, { forTesting: true }),
);
if (result.status === "success") {
toast.success("Cache connection test successful!");
toast.success(t("caching.settings.toasts.testSucceeded"));
} else {
toast.fromError(`Connection test failed: ${result.message || result.error}`);
toast.fromError(t("caching.settings.toasts.testFailed", { error: String(result.message || result.error) }));
}
} catch (error) {
console.error("Test connection error:", error);
toast.fromError(`Connection test failed: ${error instanceof Error ? error.message : "Unknown error"}`);
toast.fromError(
t("caching.settings.toasts.testFailed", {
error: error instanceof Error ? error.message : t("caching.settings.toasts.unknownError"),
}),
);
} finally {
setIsTesting(false);
}
@ -129,11 +135,11 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
setIsSaving(true);
try {
await updateCacheSettingsCall(accessToken, buildCachePayload(redisType, values, { forTesting: false }));
toast.success("Cache settings updated successfully");
toast.success(t("caching.settings.toasts.saved"));
await loadCacheSettings();
} catch (error) {
console.error("Failed to save cache settings:", error);
toast.fromError("Failed to update cache settings");
toast.fromError(t("caching.settings.toasts.saveFailed"));
} finally {
setIsSaving(false);
}
@ -148,8 +154,8 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
<FormProvider {...form}>
<form onSubmit={(event) => event.preventDefault()} className="space-y-6">
<div className="max-w-3xl">
<h3 className="text-sm font-medium text-foreground">Cache Settings</h3>
<p className="text-xs text-muted-foreground mt-1">Configure Redis cache for LiteLLM</p>
<h3 className="text-sm font-medium text-foreground">{t("caching.settings.title")}</h3>
<p className="text-xs text-muted-foreground mt-1">{t("caching.settings.subtitle")}</p>
</div>
<RedisTypeSelector
@ -160,7 +166,7 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
<div className="pt-4 border-t border-border">
<CacheFieldSection
title="Connection Settings"
title={t("caching.settings.sections.connection")}
section="connection"
redisType={redisType}
embeddingModels={embeddingModels}
@ -171,7 +177,7 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
{redisType === "cluster" && (
<div className="pt-4 border-t border-border">
<CacheFieldSection
title="Cluster Configuration"
title={t("caching.settings.sections.cluster")}
section="cluster"
redisType={redisType}
embeddingModels={embeddingModels}
@ -183,7 +189,7 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
{redisType === "sentinel" && (
<div className="pt-4 border-t border-border">
<CacheFieldSection
title="Sentinel Configuration"
title={t("caching.settings.sections.sentinel")}
section="sentinel"
redisType={redisType}
embeddingModels={embeddingModels}
@ -195,7 +201,7 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
{redisType === "semantic" && (
<div className="pt-4 border-t border-border">
<CacheFieldSection
title="Semantic Configuration"
title={t("caching.settings.sections.semantic")}
section="semantic"
redisType={redisType}
embeddingModels={embeddingModels}
@ -205,27 +211,27 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
<Collapsible open={advancedOpen} onOpenChange={setAdvancedOpen} className="mt-4">
<CollapsibleTrigger className="group flex w-full items-center justify-between py-2 text-left">
<span className="text-sm font-medium text-foreground">Advanced Settings</span>
<span className="text-sm font-medium text-foreground">{t("caching.settings.advanced")}</span>
<ChevronRight className="size-4 text-muted-foreground transition-transform group-data-panel-open:rotate-90" />
</CollapsibleTrigger>
<CollapsibleContent>
<div className="space-y-6">
<CacheFieldSection
title="SSL Settings"
title={t("caching.settings.sections.ssl")}
section="ssl"
redisType={redisType}
embeddingModels={embeddingModels}
headingLevel="h5"
/>
<CacheFieldSection
title="Cache Management"
title={t("caching.settings.sections.cacheManagement")}
section="cacheManagement"
redisType={redisType}
embeddingModels={embeddingModels}
headingLevel="h5"
/>
<CacheFieldSection
title="GCP Authentication"
title={t("caching.settings.sections.gcp")}
section="gcp"
redisType={redisType}
embeddingModels={embeddingModels}
@ -239,10 +245,10 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
<div className="border-t border-border pt-6 flex justify-end gap-3">
<Button variant="secondary" size="sm" onClick={handleTestConnection} disabled={isTesting} className="text-sm">
{isTesting ? "Testing..." : "Test Connection"}
{isTesting ? t("caching.settings.testing") : t("caching.settings.testConnection")}
</Button>
<Button size="sm" onClick={handleSaveChanges} disabled={isSaving} className="text-sm font-medium">
{isSaving ? "Saving..." : "Save Changes"}
{isSaving ? t("caching.settings.saving") : t("caching.settings.saveChanges")}
</Button>
</div>
</div>

View file

@ -1,5 +1,6 @@
import React from "react";
import { useFormContext } from "react-hook-form";
import { useTranslation } from "react-i18next";
import { FormField } from "@/components/shared/form/FormField";
import { PasswordInput } from "@/components/shared/PasswordInput";
import { Input } from "@/components/ui/input";
@ -8,7 +9,7 @@ import { Textarea } from "@/components/ui/textarea";
import { CoordinationField } from "./coordinationRedisFields";
import type { CoordinationFormValues } from "./coordinationRedisUtils";
export const SECRET_ALREADY_SET_PLACEHOLDER = "Already set. Enter a new value to replace it.";
export const SECRET_ALREADY_SET_PLACEHOLDER = "caching.coordinationRedis.field.secretAlreadySet";
interface CoordinationRedisFormFieldProps {
field: CoordinationField;
@ -16,11 +17,12 @@ interface CoordinationRedisFormFieldProps {
}
const CoordinationRedisFormField: React.FC<CoordinationRedisFormFieldProps> = ({ field, isSecretConfigured }) => {
const { t } = useTranslation();
const form = useFormContext<CoordinationFormValues>();
const placeholder = isSecretConfigured ? SECRET_ALREADY_SET_PLACEHOLDER : field.helpText;
const placeholder = isSecretConfigured ? t(SECRET_ALREADY_SET_PLACEHOLDER) : t(field.helpText);
return (
<FormField control={form.control} name={field.name} label={field.label} description={field.helpText}>
<FormField control={form.control} name={field.name} label={t(field.label)} description={t(field.helpText)}>
{({ ref, value, onChange, ...rest }) => {
if (field.type === "boolean") {
return <Switch {...rest} checked={value === true} onCheckedChange={(checked) => onChange(checked)} />;

View file

@ -1,4 +1,5 @@
import React from "react";
import { useTranslation } from "react-i18next";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import {
COORDINATION_REDIS_TYPES,
@ -12,25 +13,32 @@ interface CoordinationRedisTypeSelectorProps {
onTypeChange: (type: CoordinationRedisType) => void;
}
const CoordinationRedisTypeSelector: React.FC<CoordinationRedisTypeSelectorProps> = ({ redisType, onTypeChange }) => (
<div className="space-y-2">
<label htmlFor="coordination-redis-type" className="text-sm font-medium">
Redis Type
</label>
<Select value={redisType} onValueChange={(value) => value !== null && onTypeChange(value)}>
<SelectTrigger id="coordination-redis-type" className="w-full">
<SelectValue>{COORDINATION_REDIS_TYPE_LABELS[redisType]}</SelectValue>
</SelectTrigger>
<SelectContent>
{COORDINATION_REDIS_TYPES.map((type) => (
<SelectItem key={type} value={type}>
{COORDINATION_REDIS_TYPE_LABELS[type]}
</SelectItem>
))}
</SelectContent>
</Select>
<p className="text-xs text-muted-foreground">{COORDINATION_REDIS_TYPE_DESCRIPTIONS[redisType]}</p>
</div>
);
const CoordinationRedisTypeSelector: React.FC<CoordinationRedisTypeSelectorProps> = ({ redisType, onTypeChange }) => {
const { t } = useTranslation();
return (
<div className="space-y-2">
<label htmlFor="coordination-redis-type" className="text-sm font-medium">
{t("caching.coordinationRedis.redisType.label")}
</label>
<Select value={redisType} onValueChange={(value) => value !== null && onTypeChange(value)}>
<SelectTrigger id="coordination-redis-type" className="w-full">
<SelectValue>
{Object.hasOwn(COORDINATION_REDIS_TYPE_LABELS, redisType)
? t(COORDINATION_REDIS_TYPE_LABELS[redisType])
: ""}
</SelectValue>
</SelectTrigger>
<SelectContent>
{COORDINATION_REDIS_TYPES.map((type) => (
<SelectItem key={type} value={type}>
{t(COORDINATION_REDIS_TYPE_LABELS[type])}
</SelectItem>
))}
</SelectContent>
</Select>
<p className="text-xs text-muted-foreground">{COORDINATION_REDIS_TYPE_DESCRIPTIONS[redisType]}</p>
</div>
);
};
export default CoordinationRedisTypeSelector;

Some files were not shown because too many files have changed in this diff Show more