Merge remote-tracking branch 'origin/main' into litellm_agentic_hook_followup_error

# Conflicts:
#	litellm/litellm_core_utils/litellm_logging.py
This commit is contained in:
mateo-berri 2026-11-03 13:32:10 -08:00
commit 699b1c959e
116 changed files with 7919 additions and 2253 deletions

View file

@ -93,6 +93,8 @@ attributes! {
search_free_requests: SearchFreeRequests,
/// URL of the provider pricing/model page this entry was taken from.
source: String,
/// Model the provider recommends migrating to before `deprecation_date`, as a catalog key when one exists.
successor_model: String,
/// Audio container formats the model can return.
supported_audio_formats: Vec<AudioFormat>,
/// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.
@ -303,6 +305,8 @@ mod tests {
#[case::known_mode("mode", json!("chat"), true)]
#[case::unknown_mode("mode", json!("chatt"), false)]
#[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)]
#[case::successor_model("successor_model", json!("gpt-6-luna"), true)]
#[case::successor_model_not_a_name("successor_model", json!(["gpt-6-luna"]), false)]
#[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)]
#[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)]
#[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)]

View file

@ -1102,20 +1102,19 @@ def _cost_map_model_name(model: str, custom_llm_provider: str | None, region_nam
return _strip_unregistered_leading_segments(f"{provider_prefix}/{model}", region_name)
def _rule_declares_rates(model: str, custom_llm_provider: str | None) -> bool:
rule_info: Final = unmapped_model_generalization(model, custom_llm_provider)
return rule_info is not None and _cost_map_entry_prices_anything(rule_info)
def declared_cost_map_pricing(key: str, model: str, custom_llm_provider: str | None) -> Mapping[str, object]:
entry: Final = _raw_cost_map_entry(key)
if entry is None:
return unmapped_model_generalization(model, custom_llm_provider) or MappingProxyType({})
fill_missing: Final = mapped_model_fill_missing(key, entry)
return entry if fill_missing is None else MappingProxyType({**fill_missing, **entry})
def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool:
key: Final = cached_cost_map_key(model, custom_llm_provider)
if key is None:
return False
entry: Final = _raw_cost_map_entry(key)
if entry is None:
return _rule_declares_rates(model, custom_llm_provider)
fill_missing: Final = mapped_model_fill_missing(key, entry)
return _cost_map_entry_prices_anything(entry if fill_missing is None else {**fill_missing, **entry})
return key is not None and _cost_map_entry_prices_anything(
declared_cost_map_pricing(key, model, custom_llm_provider)
)
def _served_or_priced_requested_model(
@ -2420,18 +2419,20 @@ def completion_cost(
raise e
def get_response_cost_from_hidden_params(
hidden_params: dict | BaseModel,
) -> float | None:
if isinstance(hidden_params, BaseModel):
_hidden_params_dict = hidden_params.model_dump()
else:
_hidden_params_dict = hidden_params
_ADDITIONAL_HEADERS: Final = TypeAdapter(Mapping[str, object])
additional_headers: Final = _hidden_params_dict.get("additional_headers", {})
if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers:
return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"])
return None
def get_response_cost_from_hidden_params(
hidden_params: Mapping[str, object] | BaseModel,
) -> float | None:
params: Final[Mapping[str, object]] = (
hidden_params.model_dump() if isinstance(hidden_params, BaseModel) else hidden_params
)
additional_headers: Final = params.get("additional_headers")
if not isinstance(additional_headers, Mapping):
return None
headers: Final = _ADDITIONAL_HEADERS.validate_python(additional_headers)
return _billable_header_cost(headers.get("llm_provider-x-litellm-response-cost"))
def _billable_header_cost(value: object) -> float | None:

View file

@ -9,6 +9,7 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and
"""
import bisect
import copy
import itertools
import os
@ -21,12 +22,16 @@ from urllib.parse import urlparse
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import PromptManagementClient
from litellm.litellm_core_utils.prompt_templates.common_utils import (
PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT,
cache_control_as_prompt_cache_breakpoint,
prompt_cache_breakpoint_as_cache_control,
with_prompt_cache_breakpoint,
without_prompt_cache_breakpoints,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
tool_call_ids_answered_by_client,
@ -327,6 +332,12 @@ def _part_with_cache_control_renamed(part: object) -> object:
return cache_control_as_prompt_cache_breakpoint(_CONTENT_PART_ADAPTER.validate_python(part))
def _part_with_prompt_cache_breakpoint_renamed(part: object, drop_params: bool) -> object:
if not _carries_prompt_cache_breakpoint(part):
return part
return prompt_cache_breakpoint_as_cache_control(_CONTENT_PART_ADAPTER.validate_python(part), drop_params)
def configured_injection_points(value: object) -> Sequence[CacheControlInjectionPoint]:
if not isinstance(value, (list, tuple)):
return ()
@ -551,6 +562,10 @@ def _carries_cache_breakpoint(block: object) -> bool:
return any(_attribute_or_key(block, key) is not None for key in CACHE_BREAKPOINT_KEYS)
def _carries_cache_control(block: object) -> bool:
return _attribute_or_key(block, "cache_control") is not None
def _attribute_or_key(value: object, key: str) -> object | None:
if hasattr(value, key):
return _ATTRIBUTE_VALUE.validate_python(getattr(value, key))
@ -630,6 +645,12 @@ def _builds_anthropic_messages(model: str, custom_llm_provider: str | None) -> b
return isinstance(chat_config, AnthropicConfig) and not isinstance(chat_config, OpenAIGPTConfig)
def _translates_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None) -> bool:
return (custom_llm_provider or _served_provider_for_model(model)) == "bedrock" or _builds_anthropic_messages(
model, custom_llm_provider
)
def _custom_llm_provider(non_default_params: Mapping[str, object], point: Mapping[str, object]) -> str | None:
provider: Final = non_default_params.get("custom_llm_provider") or point.get(CUSTOM_LLM_PROVIDER_STAMP)
return provider if isinstance(provider, str) else None
@ -673,10 +694,79 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool:
return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES
def _carries_prompt_cache_breakpoint(block: object) -> bool:
return _accepts_prompt_cache_breakpoint(block) and _attribute_or_key(block, "prompt_cache_breakpoint") is not None
def _may_carry_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool:
return message.get("role") != "assistant"
def _tool_result_carries_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool:
return message.get("role") == "tool" and any(
_carries_prompt_cache_breakpoint(part) for part in _as_object_list(message.get("content")) or ()
)
def _part_without_prompt_cache_breakpoint(part: object) -> object:
if not _carries_prompt_cache_breakpoint(part):
return part
return {
key: value
for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items()
if key != "prompt_cache_breakpoint"
}
def _tool_result_without_prompt_cache_breakpoints(message: AllMessageValues) -> AllMessageValues:
if not _tool_result_carries_prompt_cache_breakpoint(message):
return message
parts: Final = _as_object_list(_attribute_or_key(message, "content")) or []
return cast( # cast-ok: the same tool message with its part marks removed
AllMessageValues, {**message, "content": [_part_without_prompt_cache_breakpoint(part) for part in parts]}
)
def _created_cache_control(original: object, translated: object) -> bool:
return (
_attribute_or_key(original, "cache_control") is None
and _attribute_or_key(translated, "cache_control") is not None
)
def _created_cache_controls(original: object, translated: object) -> tuple[bool, ...]:
if translated is original:
return ()
original_parts: Final = _as_object_list(_attribute_or_key(original, "content")) or []
translated_parts: Final = _as_object_list(_attribute_or_key(translated, "content")) or []
return tuple(_created_cache_control(before, after) for before, after in zip(original_parts, translated_parts))
def _part_without_cache_control(part: object) -> object:
return {key: value for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items() if key != "cache_control"}
def _message_dropping_created_cache_controls_between(
message: AllMessageValues, created: Sequence[bool], marks_before: int, first_dropped: int, first_kept_after: int
) -> AllMessageValues:
if not any(created):
return message
parts: Final = _as_object_list(_attribute_or_key(message, "content")) or []
ordinals: Final = itertools.accumulate((int(flag) for flag in created), initial=marks_before)
return cast( # cast-ok: the same message with its middle translated marks removed
AllMessageValues,
{
**message,
"content": [
_part_without_cache_control(part)
if is_created and first_dropped <= ordinal < first_kept_after
else part
for part, is_created, ordinal in zip(parts, created, ordinals)
],
},
)
# Set by a caller whose message list is not the one that goes upstream -- today the
# Responses API layer, whose `instructions` only becomes a system message further down.
# Tells this hook to hand role-targeted points to the pass holding the final messages
@ -762,9 +852,6 @@ class AnthropicCacheControlHook(CustomPromptManagement):
if not injection_points:
return model, messages, non_default_params
# Create a deep copy of messages to avoid modifying the original list
processed_messages = copy.deepcopy(messages)
message_points: Final = tuple(
cast(CacheControlMessageInjectionPoint, point)
for point in injection_points
@ -775,6 +862,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
request_params: Final = cast(Mapping[str, object], non_default_params) # cast-ok: base hook's bare dict
custom_llm_provider: Final = _custom_llm_provider(request_params, injection_points[0])
stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect")
model_info: Final = routed_model_info(request_params.get("model_info"))
openai_dialect: Final = (
stamped_dialect
if isinstance(stamped_dialect, bool)
@ -783,9 +871,14 @@ class AnthropicCacheControlHook(CustomPromptManagement):
custom_llm_provider,
request_params.get("api_base") or request_params.get("base_url"),
request_params.get("prompt_cache_options"),
routed_model_info(request_params.get("model_info")),
model_info,
)
)
processed_messages = copy.deepcopy(
[without_prompt_cache_breakpoints(message) for message in messages]
if not openai_dialect and message_points and _translates_prompt_cache_breakpoint(model, custom_llm_provider)
else messages
)
# A provisional message list defers every role-targeted point to the pass holding
# the final one: a role with no message here may have one there, and settling all
# of them in one pass is what lets config order decide the shared breakpoint
@ -876,7 +969,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
)
if hosted_flag is not None:
return hosted_flag
return hosted_flag and not _translates_prompt_cache_breakpoint(model, custom_llm_provider)
if not supports_openai_prompt_cache_breakpoint(model, model_info=model_info):
return False
if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai":
@ -929,6 +1022,19 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
return int(wire_cache_control is not None) + tool_blocks + envelope_blocks
@staticmethod
def tools_keeping_cache_control_on_chat(tools: object) -> tuple[object, ...]:
return tuple(
tool for tool in _as_object_list(tools) or () if not _chat_transform_drops_tool_cache_control(tool)
)
@staticmethod
def count_chat_external_cache_breakpoints(optional_params: object) -> int:
params: Final = _validated_object_mapping(optional_params) or {}
return AnthropicCacheControlHook.count_external_cache_breakpoints(
_as_object_list(params.get("tools")), params.get("cache_control")
)
@staticmethod
def count_external_cache_breakpoints_on_messages_route(
tools: Iterable[object] | None, cache_control: object, request_kwargs: object
@ -1084,6 +1190,16 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
return message_count + content_count + tool_call_count
@staticmethod
def count_converse_cache_points(message: AllMessageValues) -> int:
content: Final = _as_object_list(_attribute_or_key(message, "content"))
tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls")) or []
marked_parts: Final = sum(1 for block in content or () if _carries_cache_control(block))
marked_tool_calls: Final = sum(1 for tool_call in tool_calls if _carries_cache_control(tool_call))
if message.get("role") == "tool":
return int(_carries_cache_control(message) or marked_parts > 0)
return (int(_carries_cache_control(message)) if content is None else marked_parts) + marked_tool_calls
@staticmethod
def _message_has_cache_control(message: AllMessageValues, answered_tool_ids: Container[str]) -> bool:
"""Return True if the message already carries any cache_control."""
@ -1142,6 +1258,92 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return renamed
return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed)
@staticmethod
def prompt_cache_breakpoints_as_cache_control(message: AllMessageValues, drop_params: bool) -> AllMessageValues:
"""Every OpenAI ``prompt_cache_breakpoint`` on a part OpenAI accepts one on moved to Anthropic's
``cache_control``. Marks with no OpenAI slot are left for the caller to strip."""
if not _may_carry_prompt_cache_breakpoint(message):
return message
parts: Final = _as_object_list(_attribute_or_key(message, "content"))
if parts is None or not any(_carries_prompt_cache_breakpoint(part) for part in parts):
return message
return cast( # cast-ok: the same message with its part marks renamed
AllMessageValues,
{**message, "content": [_part_with_prompt_cache_breakpoint_renamed(part, drop_params) for part in parts]},
)
@staticmethod
def prompt_cache_breakpoints_dropped_from_tool_results(
messages: Sequence[AllMessageValues], drop_params: bool, model: str
) -> tuple[AllMessageValues, ...]:
if not any(_tool_result_carries_prompt_cache_breakpoint(message) for message in messages):
return tuple(messages)
if not drop_params:
raise UnsupportedParamsError(
message=(
f"{model} takes no cache breakpoint on a tool result, so prompt_cache_breakpoint on a tool "
f"message has no equivalent. Remove it, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it."
),
status_code=400,
)
verbose_logger.debug("Dropping prompt_cache_breakpoint from tool results for %s under drop_params", model)
return tuple(_tool_result_without_prompt_cache_breakpoints(message) for message in messages)
@staticmethod
def prompt_cache_breakpoints_as_capped_cache_control(
messages: Sequence[AllMessageValues],
drop_params: bool,
sent_breakpoints: Callable[[AllMessageValues], int],
external_breakpoints: Callable[[], int],
) -> tuple[AllMessageValues, ...]:
"""``prompt_cache_breakpoints_as_cache_control`` over the whole request, held to the provider's
breakpoint cap, counting each message by the breakpoints ``sent_breakpoints`` says its provider sends.
Under ``drop_params`` the first and last translated marks fill the slots the client's own
``cache_control`` marks leave, the first half taking the odd slot, so the stable prefix and the newest
turns stay cached, and without it a request over the cap is rejected."""
translated: Final = tuple(
AnthropicCacheControlHook.prompt_cache_breakpoints_as_cache_control(message, drop_params)
for message in messages
)
if all(after is before for before, after in zip(messages, translated)):
return translated
created: Final = tuple(_created_cache_controls(before, after) for before, after in zip(messages, translated))
created_marks: Final = sum(sum(flags) for flags in created)
if created_marks == 0:
return translated
external: Final = external_breakpoints()
breakpoints: Final = sum(map(sent_breakpoints, translated)) + external
excess: Final = breakpoints - MAX_CACHE_CONTROL_BLOCKS
if excess <= 0:
return translated
if not drop_params:
raise UnsupportedParamsError(
message=(
f"This model takes at most {MAX_CACHE_CONTROL_BLOCKS} cache breakpoints per request, and this "
f"request has {breakpoints} once its prompt_cache_breakpoint marks are translated. Remove {excess} "
f"of them, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to keep the first and last of them."
),
status_code=400,
)
verbose_logger.debug("Dropping the middle prompt_cache_breakpoint marks past the cap under drop_params")
marks_before: Final = tuple(itertools.accumulate((sum(flags) for flags in created), initial=0))
def capped_request(kept_marks: int) -> tuple[AllMessageValues, ...]:
first_dropped: Final = (kept_marks + 1) // 2
first_kept_after: Final = created_marks - kept_marks // 2
return tuple(
_message_dropping_created_cache_controls_between(
message, flags, before, first_dropped, first_kept_after
)
for message, flags, before in zip(translated, created, marks_before)
)
def over_cap(kept_marks: int) -> bool:
return sum(map(sent_breakpoints, capped_request(kept_marks))) + external > MAX_CACHE_CONTROL_BLOCKS
first_over_cap: Final = bisect.bisect_left(range(max(0, created_marks - excess) + 1), True, key=over_cap)
return capped_request(max(0, first_over_cap - 1))
@staticmethod
def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues:
if not _may_carry_prompt_cache_breakpoint(message):
@ -1487,9 +1689,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
configured: Final = configured_injection_points(non_default_params.get("cache_control_injection_points"))
if configured:
tools_keeping_marks: Final = tuple(
tool for tool in tools or () if not _chat_transform_drops_tool_cache_control(tool)
)
tools_keeping_marks: Final = AnthropicCacheControlHook.tools_keeping_cache_control_on_chat(tools)
stamped: Final = AnthropicCacheControlHook._stamped_for_prompt_hook(
configured,
AnthropicCacheControlHook.count_external_cache_breakpoints(
@ -1782,9 +1982,17 @@ def messages_within_cache_control_cap(
Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
"""
reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
_validated_object_list(tools), cache_control, request_kwargs
return messages_within_reserved_cache_control_cap(
messages,
AnthropicCacheControlHook.count_external_cache_breakpoints(
_validated_object_list(tools), cache_control, request_kwargs
),
)
def messages_within_reserved_cache_control_cap(
messages: Sequence[_MessageT], reserved_marks: int
) -> Sequence[_MessageT]:
kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
if dropped > 0:

View file

@ -57,6 +57,8 @@ from litellm.cost_calculator import (
BaseTokenUsageProcessor,
RealtimeAPITokenUsageProcessor,
ResponsesWebSocketTokenUsageProcessor,
declared_cost_map_pricing,
get_response_cost_from_hidden_params,
get_usage_object,
pricing_entry_for_cost_calc,
select_model_name_for_cost_calc,
@ -75,6 +77,7 @@ from litellm.integrations.deepeval.deepeval import DeepEvalLogger
from litellm.integrations.langtrace import langtrace_trace_endpoint
from litellm.integrations.mlflow import MlflowLogger
from litellm.integrations.sqs import SQSLogger
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.litellm_core_utils.agentic_followup_failure import ServedRound
from litellm.litellm_core_utils.classifier_logging import (
classifier_audit_fields,
@ -217,7 +220,10 @@ from litellm.types.utils import (
batch_chat_usage,
)
from litellm.types.videos.main import VideoObject
from litellm.utils import get_base_model_from_metadata, print_verbose
from litellm.utils import (
get_base_model_from_metadata,
print_verbose,
)
from ..integrations.argilla import ArgillaLogger
from ..integrations.arize.arize_phoenix import ArizePhoenixLogger
@ -2596,6 +2602,33 @@ class Logging(LiteLLMLoggingBaseClass):
**kwargs,
)
def _response_price_is_known(self, result: object, cost: float, payload: StandardLoggingPayload) -> bool:
if (
cost != 0
or self._provider_reported_cost == 0
or get_response_cost_from_hidden_params(get_hidden_params_dict(result)) == 0
):
return True
if self.model_call_details.get("cache_hit") is True or _served_from_cache_at_no_charge(result):
return True
params: Final = _LITELLM_PARAMS.validate_python(self._model_call_details_view.get("litellm_params") or {})
if is_unbilled_non_inference_call(
payload["call_type"], StandardLoggingPayloadSetup.merge_litellm_metadata(params), result
):
return True
if self.custom_cost_per_token is not None or self.custom_cost_per_second is not None:
return True
passthrough: Final = _LITELLM_PARAMS.validate_python(
self._model_call_details_view.get("passthrough_logging_payload") or {}
)
if passthrough.get("cost_per_request") is not None:
return True
model_info: Final = payload["model_map_information"]["model_map_value"]
if model_info is None:
return False
prices: Final = declared_cost_map_pricing(model_info["key"], model_info["key"], payload["custom_llm_provider"])
return any(value is not None and ("cost" in key or "pricing" in key) for key, value in prices.items())
def prepare_response_cost(self, response: object, now: datetime.datetime) -> PreparedCost | None:
if self.prepared_cost is not None:
return self.prepared_cost
@ -2648,6 +2681,7 @@ class Logging(LiteLLMLoggingBaseClass):
"cost_breakdown": copy.deepcopy(payload["cost_breakdown"]),
},
usage,
price_known=self._response_price_is_known(prepared[2], raw_cost, payload),
)
self.record_partial_usage_for_failure(
usage, raw_cost, observation=self.model_call_details["_accounting_usage_observed"]

View file

@ -17,6 +17,7 @@ class PreparedCost:
amount: float
payload: StandardLoggingPayload
usage: Usage
price_known: bool = True
@dataclass(frozen=True, slots=True)

View file

@ -21,12 +21,14 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.batch_utils import InMemoryFile
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
ChatCompletionAudioObject,
ChatCompletionCachedContent,
ChatCompletionFileObject,
ChatCompletionImageObject,
ChatCompletionPromptCacheBreakpoint,
@ -2183,6 +2185,9 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool:
_MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object])
PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT: Final = (
"set `drop_params: true` on the request or in litellm_settings (`litellm.drop_params = True` in the SDK)"
)
def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
@ -2262,6 +2267,51 @@ def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
)
def prompt_cache_breakpoint_as_cache_control(block: _MarkedT, drop_params: bool) -> _MarkedT:
"""OpenAI's ``prompt_cache_breakpoint`` mark rewritten as Anthropic's ``cache_control`` on the same block.
A ``cache_control`` the client already wrote on the block wins, TTL included."""
marker: Final = block.get("prompt_cache_breakpoint")
if marker is None:
return block
unmarked: Final = {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"}
control: Final = (
None
if block.get("cache_control") is not None
else _cache_control_for_prompt_cache_breakpoint(marker, drop_params)
)
marked: Final = unmarked if control is None else {**unmarked, "cache_control": control}
return cast(_MarkedT, marked) # cast-ok: same block shape as the input with the marker key renamed
def _cache_control_for_prompt_cache_breakpoint(marker: object, drop_params: bool) -> ChatCompletionCachedContent | None:
try:
breakpoint_mark: Final = _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
except ValidationError:
if not drop_params:
raise UnsupportedParamsError(
message=(
f"prompt_cache_breakpoint must be {{'mode': 'explicit'}} with an optional ttl of '30m' or '12h', got "
f"{marker!r}. Fix the marker, or "
f"{PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it."
),
status_code=400,
)
verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
return None
ttl: Final = breakpoint_mark.get("ttl")
if ttl is not None and not drop_params:
raise UnsupportedParamsError(
message=(
f"This model's prompt cache lasts 5m, or 1h where it offers one, so prompt_cache_breakpoint "
f"ttl={ttl!r} has no equivalent. Remove the ttl, {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to send "
"a 5m breakpoint instead, or send cache_control {'type': 'ephemeral', 'ttl': '1h'} on the block "
"where the model offers a 1h cache."
),
status_code=400,
)
return ChatCompletionCachedContent(type="ephemeral")
_BREAKPOINT_BLOCK_LIST_KEYS: Final = frozenset({"content", "tool_calls"})
_BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object])
_BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object])

View file

@ -2370,6 +2370,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params
)
def translates_prompt_cache_breakpoint(self, model: str) -> bool:
return True
def transform_request(
self,
model: str,
@ -2383,19 +2386,32 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"""
## VALIDATE REQUEST
"""Anthropic requires ``tools`` when messages include tool blocks; LiteLLM injects a dummy tool if omitted (no ``modify_params`` needed)."""
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, cache_control_count
from litellm.litellm_core_utils.prompt_templates.factory import (
anthropic_messages_pt,
)
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
translated_messages: Final = (
list(
AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control(
messages,
drop_params,
cache_control_count,
lambda: AnthropicCacheControlHook.count_chat_external_cache_breakpoints(optional_params),
)
)
if self.translates_prompt_cache_breakpoint(model)
else messages
)
capped_messages: Final = (
list(
self._messages_within_cache_control_cap(
messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
translated_messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
)
)
if drop_params
else messages
else translated_messages
)
if "tools" not in optional_params and has_tool_call_blocks(capped_messages):

View file

@ -965,6 +965,8 @@ class BaseAWSLLM(SignsRequestsWithAWS):
self,
aws_role_name: str,
ssl_verify: bool | str | None = None,
aws_region_name: str | None = None,
aws_sts_endpoint: str | None = None,
) -> bool:
"""
Check if the current environment is already running as the target IAM role.
@ -998,7 +1000,14 @@ class BaseAWSLLM(SignsRequestsWithAWS):
import boto3
with tracer.trace("boto3.client(sts).get_caller_identity"):
sts_client: Final = boto3.client("sts", verify=self._get_ssl_verify(ssl_verify))
sts_client: Final = boto3.client(
"sts",
**self._build_sts_client_kwargs(
aws_sts_endpoint=aws_sts_endpoint,
ssl_verify=ssl_verify,
aws_region_name=aws_region_name,
),
)
identity: Final = sts_client.get_caller_identity()
caller_arn: Final = identity.get("Arn", "")
@ -1289,7 +1298,12 @@ class BaseAWSLLM(SignsRequestsWithAWS):
caller's key skips both. ``aws_session_name`` defaults inside this fetcher rather than in
``get_credentials`` so the cache key stays stable when the caller does not supply one.
"""
if self._is_already_running_as_role(aws_role_name, ssl_verify=ssl_verify):
if self._is_already_running_as_role(
aws_role_name,
ssl_verify=ssl_verify,
aws_region_name=aws_region_name,
aws_sts_endpoint=aws_sts_endpoint,
):
verbose_logger.debug(
"Already running as target role %s, using ambient credentials",
aws_role_name,
@ -1415,7 +1429,12 @@ class BaseAWSLLM(SignsRequestsWithAWS):
# partition, and role name). This avoids silently using the
# wrong identity when there is a genuine trust-policy or
# permission misconfiguration.
if self._is_already_running_as_role(aws_role_name, ssl_verify=ssl_verify):
if self._is_already_running_as_role(
aws_role_name,
ssl_verify=ssl_verify,
aws_region_name=aws_region_name,
aws_sts_endpoint=aws_sts_endpoint,
):
verbose_logger.warning(
"AssumeRole failed for %s (%s). "
"Caller is already running as this role; "

View file

@ -23,6 +23,11 @@ from litellm.constants import (
BEDROCK_MIN_THINKING_BUDGET_TOKENS,
RESPONSE_FORMAT_TOOL_NAME,
)
from litellm.integrations.anthropic_cache_control_hook import (
MAX_CACHE_CONTROL_BLOCKS,
AnthropicCacheControlHook,
configured_injection_points,
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
drops_unsupported_params,
@ -131,6 +136,7 @@ from ..common_utils import (
bedrock_capability_model,
bedrock_converse_supports_parallel_tool_use_config,
bedrock_model_accepts_cache_points,
bedrock_model_known_to_accept_cache_points,
bedrock_reasoning_effort_disabled,
bedrock_rejects_stop_sequences,
get_anthropic_beta_from_headers,
@ -204,6 +210,11 @@ def _has_native_text_format(output_config: object) -> bool:
return False
def _cache_mark_kept(block: Mapping[str, object], keeps_cache_marks: bool) -> Mapping[str, object]:
cache_control: Final = block.get("cache_control")
return {"cache_control": cache_control} if keeps_cache_marks and cache_control is not None else {}
class AmazonConverseConfig(BaseConfig):
"""
Reference - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html
@ -243,7 +254,7 @@ class AmazonConverseConfig(BaseConfig):
@staticmethod
def _convert_consecutive_user_messages_to_guarded_text(
messages: list[AllMessageValues], optional_params: dict
messages: list[AllMessageValues], optional_params: dict, keeps_cache_marks: bool = False
) -> list[AllMessageValues]:
"""
Convert consecutive user messages at the end to guarded_text type if guardrailConfig is present
@ -281,7 +292,11 @@ class AmazonConverseConfig(BaseConfig):
new_content = []
for item in content:
if isinstance(item, dict) and item.get("type") == "text":
new_item = {"type": "guarded_text", "text": item["text"]}
new_item = {
"type": "guarded_text",
"text": item["text"],
**_cache_mark_kept(_JSON_OBJECT.validate_python(item), keeps_cache_marks),
}
new_content.append(new_item)
else:
new_content.append(item)
@ -289,7 +304,13 @@ class AmazonConverseConfig(BaseConfig):
messages_copy[user_message_index]["content"] = new_content
elif isinstance(content, str):
# If content is a string, convert it to guarded_text
messages_copy[user_message_index]["content"] = [{"type": "guarded_text", "text": content}]
messages_copy[user_message_index]["content"] = [
{
"type": "guarded_text",
"text": content,
**_cache_mark_kept(_JSON_OBJECT.validate_python(user_message), keeps_cache_marks),
}
]
return messages_copy
@ -1595,6 +1616,74 @@ class AmazonConverseConfig(BaseConfig):
return ChatCompletionTextObject(type="text", text=text)
return ChatCompletionTextObject(type="text", text=text, cache_control=cache_control)
def _messages_with_cache_marks(
self,
messages: Sequence[AllMessageValues],
model: str,
deployment_model: str,
drop_params: bool,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
model_info: Mapping[str, object] | None,
) -> Sequence[AllMessageValues]:
if not bedrock_model_known_to_accept_cache_points(model, deployment_model, model_info):
return messages
markable: Final = (
AnthropicCacheControlHook.prompt_cache_breakpoints_dropped_from_tool_results(messages, drop_params, model)
if "nova" in model.lower()
else messages
)
return AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control(
markable,
drop_params,
AnthropicCacheControlHook.count_converse_cache_points,
lambda: self._tool_cache_points_on_the_wire(
model, deployment_model, optional_params, litellm_params, model_info
),
)
def _tool_cache_points_on_the_wire(
self,
model: str,
deployment_model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
model_info: Mapping[str, object] | None,
) -> int:
tools: Final = optional_params.get("tools")
bedrock_tools, _ = self._process_tools_and_beta(
tools if isinstance(tools, list) else [], model, None, {}, litellm_params, deployment_model, model_info
)
injects_tool_config_point: Final = bool(bedrock_tools) and any(
point.get("location") == "tool_config"
for point in configured_injection_points(optional_params.get("cache_control_injection_points"))
)
return sum(1 for tool in bedrock_tools if "cachePoint" in tool) + int(injects_tool_config_point)
def _guarded_text_keeps_cache_marks(
self,
messages: Sequence[AllMessageValues],
system_content_blocks: Sequence[SystemContentBlock],
model: str,
deployment_model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
model_info: Mapping[str, object] | None,
) -> bool:
if "guardrailConfig" not in optional_params or not bedrock_model_known_to_accept_cache_points(
model, deployment_model, model_info
):
return False
message_cache_points: Final = sum(map(AnthropicCacheControlHook.count_converse_cache_points, messages))
if message_cache_points == 0:
return False
cache_points: Final = (
sum(1 for block in system_content_blocks if "cachePoint" in block)
+ message_cache_points
+ self._tool_cache_points_on_the_wire(model, deployment_model, optional_params, litellm_params, model_info)
)
return cache_points <= MAX_CACHE_CONTROL_BLOCKS
def _transform_system_message(
self,
messages: list[AllMessageValues],
@ -2152,22 +2241,27 @@ class AmazonConverseConfig(BaseConfig):
return data
@staticmethod
def _claude_messages_within_cache_control_cap(
self,
messages: Sequence[AllMessageValues],
capability_model: str,
deployment_model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
model_info: Mapping[str, object] | None,
) -> Sequence[AllMessageValues]:
if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower():
return messages
from litellm.integrations.anthropic_cache_control_hook import (
messages_within_cache_control_cap,
messages_within_reserved_cache_control_cap,
tool_message_with_single_cache_mark,
)
return messages_within_cache_control_cap(
tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools")
return messages_within_reserved_cache_control_cap(
tuple(map(tool_message_with_single_cache_mark, messages)),
self._tool_cache_points_on_the_wire(
capability_model, deployment_model, optional_params, litellm_params, model_info
),
)
async def _async_transform_request(
@ -2183,10 +2277,20 @@ class AmazonConverseConfig(BaseConfig):
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
model_info: Final = routed_model_info(litellm_params.get("model_info"))
json_optional_params: Final = _JSON_OBJECT.validate_python(optional_params)
marked_messages: Final = self._messages_with_cache_marks(
messages,
capability_model,
model,
drops_unsupported_params(litellm_params.get("drop_params")),
json_optional_params,
litellm_params,
model_info,
)
messages, system_content_blocks = self._transform_system_message(
list(
self._claude_messages_within_cache_control_cap(
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
marked_messages, capability_model, model, json_optional_params, litellm_params, model_info
)
),
model=capability_model,
@ -2195,8 +2299,19 @@ class AmazonConverseConfig(BaseConfig):
)
# Convert last user message to guarded_text if guardrailConfig is present
guard_input: Final = self._handle_orphaned_tool_blocks(messages, optional_params)
messages = self._convert_consecutive_user_messages_to_guarded_text(
self._handle_orphaned_tool_blocks(messages, optional_params), optional_params
guard_input,
optional_params,
keeps_cache_marks=self._guarded_text_keeps_cache_marks(
guard_input,
system_content_blocks,
capability_model,
model,
json_optional_params,
litellm_params,
model_info,
),
)
## TRANSFORMATION ##
@ -2276,10 +2391,20 @@ class AmazonConverseConfig(BaseConfig):
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
model_info: Final = routed_model_info(litellm_params.get("model_info"))
json_optional_params: Final = _JSON_OBJECT.validate_python(optional_params)
marked_messages: Final = self._messages_with_cache_marks(
messages,
capability_model,
model,
drops_unsupported_params(litellm_params.get("drop_params")),
json_optional_params,
litellm_params,
model_info,
)
messages, system_content_blocks = self._transform_system_message(
list(
self._claude_messages_within_cache_control_cap(
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
marked_messages, capability_model, model, json_optional_params, litellm_params, model_info
)
),
model=capability_model,
@ -2288,8 +2413,19 @@ class AmazonConverseConfig(BaseConfig):
)
# Convert last user message to guarded_text if guardrailConfig is present
guard_input: Final = self._handle_orphaned_tool_blocks(messages, optional_params)
messages = self._convert_consecutive_user_messages_to_guarded_text(
self._handle_orphaned_tool_blocks(messages, optional_params), optional_params
guard_input,
optional_params,
keeps_cache_marks=self._guarded_text_keeps_cache_marks(
guard_input,
system_content_blocks,
capability_model,
model,
json_optional_params,
litellm_params,
model_info,
),
)
_data: Final[CommonRequestObject] = self._transform_request_helper(

View file

@ -1559,22 +1559,63 @@ def bedrock_model_accepts_cache_points(
def bedrock_declared_cache_point_support(model: str, model_info: Mapping[str, object] | None = None) -> bool | None:
if _OPENAI_FAMILY_MODEL_RE.search(model):
return False
explicit_marker_support: Final = _explicit_cache_point_support(model, model_info)
if explicit_marker_support is not None:
return explicit_marker_support
return _mapped_prompt_caching_support(model, model_info)
def bedrock_model_known_to_accept_cache_points(
model: str, deployment_model: str | None = None, model_info: Mapping[str, object] | None = None
) -> bool:
"""Like ``bedrock_model_accepts_cache_points``, but only for the Claude and Nova families Bedrock documents
as taking cache checkpoints, unless the entry sets ``supports_prompt_cache_breakpoint``: an OpenAI
``prompt_cache_breakpoint`` reaches every model of a mixed fleet, not only the ones a client marked for
Bedrock caching, and other families price implicit cache reads yet may reject the cachePoint field."""
deployment_declared: Final = (
None if deployment_model is None else _declared_marker_support(deployment_model, model_info)
)
declared: Final = (
deployment_declared if deployment_declared is not None else _declared_marker_support(model, model_info)
)
return declared is True
def _declared_marker_support(model: str, model_info: Mapping[str, object] | None) -> bool | None:
if _OPENAI_FAMILY_MODEL_RE.search(model):
return False
explicit_marker_support: Final = _explicit_cache_point_support(model, model_info)
if explicit_marker_support is not None:
return explicit_marker_support
mapped: Final = _mapped_prompt_caching_support(model, model_info)
if not _documents_cache_checkpoints(model):
return None if mapped is None else False
return mapped is not False
def _documents_cache_checkpoints(model: str) -> bool:
return _is_claude(model) or get_bedrock_base_model(model).startswith("amazon.nova")
def _explicit_cache_point_support(model: str, model_info: Mapping[str, object] | None) -> bool | None:
deployment_flag: Final = None if model_info is None else model_info.get("supports_prompt_cache_breakpoint")
if deployment_flag is not None:
return deployment_flag is True
catalog_entry: Final = _catalog_entry_reader(model_info)
map_keys: Final = (model, get_bedrock_base_model(model))
registered_keys: Final = tuple(f"bedrock/{route}{model}" for route in ("", "converse/", "converse_like/"))
explicit_marker_support: Final = next(
return next(
(
entry.get("supports_prompt_cache_breakpoint") is True
for key in (*registered_keys, *map_keys)
for key in (*registered_keys, model, get_bedrock_base_model(model))
if (entry := catalog_entry(key)) is not None and entry.get("supports_prompt_cache_breakpoint") is not None
),
None,
)
if explicit_marker_support is not None:
return explicit_marker_support
def _mapped_prompt_caching_support(model: str, model_info: Mapping[str, object] | None) -> bool | None:
catalog_entry: Final = _catalog_entry_reader(model_info)
map_keys: Final = (model, get_bedrock_base_model(model))
if not any(catalog_entry(key) is not None for key in map_keys):
return None
deployment_caching: Final = None if model_info is None else lax_bool(model_info.get("supports_prompt_caching"))

View file

@ -112,6 +112,9 @@ class OpenCodeGoMessagesChatConfig(_GoReportedCostMixin, AnthropicConfig):
def custom_llm_provider(self) -> str:
return "opencode_go"
def translates_prompt_cache_breakpoint(self, model: str) -> bool:
return "claude" in model.lower()
def get_complete_url(
self,
api_base: str | None,

View file

@ -2135,12 +2135,14 @@
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json",
"token_count_model": {
"model": "anthropic.claude-opus-5",
"litellm_provider": "bedrock_mantle",
"region": "us-east-1"
}
},
"input_cost_per_token_batches": 2.5e-06,
"output_cost_per_token_batches": 1.25e-05
},
"anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -2221,12 +2223,14 @@
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/",
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json",
"token_count_model": {
"model": "global.anthropic.claude-opus-5",
"litellm_provider": "bedrock_mantle",
"region": "us-east-1"
}
},
"input_cost_per_token_batches": 2.5e-06,
"output_cost_per_token_batches": 1.25e-05
},
"global.anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -2307,12 +2311,14 @@
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/",
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json",
"token_count_model": {
"model": "anthropic.claude-opus-5",
"litellm_provider": "bedrock_mantle",
"region": "us-east-1"
}
},
"input_cost_per_token_batches": 2.75e-06,
"output_cost_per_token_batches": 1.375e-05
},
"us.anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -2393,12 +2399,14 @@
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/",
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/eu-west-1/index.json",
"token_count_model": {
"model": "anthropic.claude-opus-5",
"litellm_provider": "bedrock_mantle",
"region": "eu-west-1"
}
},
"input_cost_per_token_batches": 2.75e-06,
"output_cost_per_token_batches": 1.375e-05
},
"eu.anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -2474,12 +2482,14 @@
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/",
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-southeast-2/index.json",
"token_count_model": {
"model": "anthropic.claude-opus-5",
"litellm_provider": "bedrock_mantle",
"region": "ap-southeast-4"
}
},
"input_cost_per_token_batches": 2.75e-06,
"output_cost_per_token_batches": 1.375e-05
},
"au.anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -2560,7 +2570,9 @@
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/"
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-northeast-1/index.json",
"input_cost_per_token_batches": 2.75e-06,
"output_cost_per_token_batches": 1.375e-05
},
"jp.anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -3949,7 +3961,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true,
"supports_max_reasoning_effort": false,
"prompt_cache_min_tokens": 512
},
"azure_ai/claude-fable-5-1": {
@ -3983,10 +3995,40 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true,
"supports_max_reasoning_effort": false,
"prompt_cache_min_tokens": 512,
"deprecation_date": "2027-12-05"
},
"azure_ai/claude-mythos-5-1": {
"input_cost_per_token": 1e-05,
"output_cost_per_token": 5e-05,
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 2.5e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"mode": "chat",
"supports_adaptive_thinking": true,
"thinking_always_on": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_forced_tool_use": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": false,
"prompt_cache_min_tokens": 512,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models"
},
"azure_ai/claude-opus-5": {
"deprecation_date": "2027-07-08",
"supports_mid_conversation_system": true,
@ -22269,7 +22311,7 @@
},
"mode": "chat",
"output_cost_per_token": 3.74997e-06,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
"supported_modalities": [
"text",
"image",
@ -22287,7 +22329,9 @@
"input_cost_per_token_priority": 1.35002e-06,
"output_cost_per_token_priority": 6.75003e-06,
"cache_read_input_token_cost_priority": 1.3503e-07,
"cache_creation_input_token_cost_priority": 1.35002e-06
"cache_creation_input_token_cost_priority": 1.35002e-06,
"supports_audio_input": true,
"supports_video_input": true
},
"databricks/databricks-gemini-3-7-flash": {
"cache_creation_input_token_cost": 7.4998e-07,
@ -22301,7 +22345,7 @@
},
"mode": "chat",
"output_cost_per_token": 3.74997e-06,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
"supported_modalities": [
"text",
"image",
@ -22319,7 +22363,9 @@
"input_cost_per_token_priority": 1.35002e-06,
"output_cost_per_token_priority": 6.75003e-06,
"cache_read_input_token_cost_priority": 1.3503e-07,
"cache_creation_input_token_cost_priority": 1.35002e-06
"cache_creation_input_token_cost_priority": 1.35002e-06,
"supports_audio_input": true,
"supports_video_input": true
},
"databricks/databricks-gemini-3-6-flash": {
"cache_creation_input_token_cost": 7.4998e-07,
@ -22335,7 +22381,7 @@
"mode": "chat",
"output_cost_per_token": 3.74997e-06,
"output_dbu_cost_per_token": 5.3571e-05,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
"supported_modalities": [
"text",
"image",
@ -22354,7 +22400,9 @@
"input_cost_per_token_priority": 1.35002e-06,
"output_cost_per_token_priority": 6.75003e-06,
"cache_read_input_token_cost_priority": 1.3503e-07,
"cache_creation_input_token_cost_priority": 1.35002e-06
"cache_creation_input_token_cost_priority": 1.35002e-06,
"supports_audio_input": true,
"supports_video_input": true
},
"databricks/databricks-gemini-3-5-flash": {
"cache_creation_input_token_cost": 1.87502e-06,
@ -22370,7 +22418,7 @@
"mode": "chat",
"output_cost_per_token": 1.124998e-05,
"output_dbu_cost_per_token": 0.000160714,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
"supported_modalities": [
"text",
"image",
@ -22388,7 +22436,9 @@
"input_cost_per_token_priority": 3.37498e-06,
"output_cost_per_token_priority": 2.025002e-05,
"cache_read_input_token_cost_priority": 3.3747e-07,
"cache_creation_input_token_cost_priority": 3.37498e-06
"cache_creation_input_token_cost_priority": 3.37498e-06,
"supports_audio_input": true,
"supports_video_input": true
},
"databricks/databricks-gemini-3-5-flash-lite": {
"cache_creation_input_token_cost": 3.7499e-07,
@ -22404,7 +22454,7 @@
"mode": "chat",
"output_cost_per_token": 3.12501e-06,
"output_dbu_cost_per_token": 4.4643e-05,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
"supported_modalities": [
"text",
"image",
@ -22424,7 +22474,9 @@
"input_cost_per_token_priority": 6.7501e-07,
"output_cost_per_token_priority": 5.62499e-06,
"cache_read_input_token_cost_priority": 6.748e-08,
"cache_creation_input_token_cost_priority": 6.7501e-07
"cache_creation_input_token_cost_priority": 6.7501e-07,
"supports_audio_input": true,
"supports_video_input": true
},
"databricks/databricks-gemma-3-12b": {
"cache_creation_input_token_cost": 1.5001e-07,
@ -24167,7 +24219,7 @@
"max_output_tokens": 163840,
"input_cost_per_token": 5e-07,
"output_cost_per_token": 2.15e-06,
"cache_read_input_token_cost": 4e-07,
"cache_read_input_token_cost": 3.5e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
@ -24241,7 +24293,7 @@
"max_output_tokens": 163840,
"input_cost_per_token": 2.5e-07,
"output_cost_per_token": 9.5e-07,
"cache_read_input_token_cost": 2.16e-07,
"cache_read_input_token_cost": 1.3e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
@ -29639,6 +29691,7 @@
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-lite-image": {
"deprecation_date": "2027-06-28",
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_batches": 1.25e-08,
"cache_read_input_token_cost_flex": 1.25e-08,
@ -30638,6 +30691,7 @@
"google_maps_grounding_cost_per_query": 0.014
},
"vertex_ai/gemini-3.8-flash": {
"deprecation_date": "2027-09-02",
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_batches": 3.75e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
@ -30744,6 +30798,7 @@
"supports_native_streaming": true
},
"vertex_ai/gemini-3.8-live": {
"deprecation_date": "2027-09-25",
"gemini_audio_only_live": true,
"input_cost_per_audio_token": 3e-06,
"input_cost_per_image_token": 1e-06,
@ -31421,6 +31476,7 @@
"cache_read_input_token_cost_batches": 3e-08,
"cache_read_input_token_cost_flex": 3e-08,
"cache_read_input_token_cost_priority": 5.4e-08,
"deprecation_date": "2026-12-01",
"input_cost_per_audio_token": 1e-06,
"input_cost_per_audio_token_flex": 5e-07,
"input_cost_per_token": 3e-07,
@ -32004,6 +32060,7 @@
"output_cost_per_token_flex": 5e-06,
"supports_url_context": true,
"cache_read_input_token_cost_above_200k_tokens_batches": 2.5e-07,
"deprecation_date": "2026-12-01",
"input_cost_per_token_above_200k_tokens_batches": 1.25e-06,
"output_cost_per_token_above_200k_tokens_batches": 7.5e-06
},
@ -32107,7 +32164,7 @@
},
"gemini/gemini-3.5-flash-lite": {
"cache_read_input_token_cost": 3e-08,
"cache_read_input_token_cost_batches": 2e-08,
"cache_read_input_token_cost_batches": 1.5e-08,
"cache_read_input_token_cost_flex": 2e-08,
"cache_read_input_token_cost_priority": 5e-08,
"input_cost_per_token": 3e-07,
@ -32704,13 +32761,13 @@
"input_cost_per_audio_token": 1.5e-06,
"input_cost_per_token": 1.5e-06,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_input_tokens": 131072,
"max_output_tokens": 57920,
"mode": "chat",
"output_cost_per_reasoning_token": 9e-06,
"output_cost_per_token": 9e-06,
"output_cost_per_video_token": 1.75e-05,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-flash-preview",
"supported_endpoints": [
"/v1/chat/completions"
],
@ -32728,7 +32785,8 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_video_input": true,
"supports_vision": true
"supports_vision": true,
"deprecation_date": "2026-11-06"
},
"gemini-3.5-flash": {
"prompt_cache_min_tokens": 4096,
@ -32916,6 +32974,7 @@
"google_maps_grounding_cost_per_query": 0.014
},
"gemini-3.8-flash": {
"deprecation_date": "2027-09-02",
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_batches": 3.75e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
@ -44263,7 +44322,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/google%2Fgemma-3-27b-it"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 4e-08
},
"nebius/meta-llama/Llama-3.3-70B-Instruct": {
"max_input_tokens": 131072,
@ -44489,7 +44549,7 @@
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash"
},
"nebius/deepseek-ai/DeepSeek-V4-Flash-0731": {
"max_input_tokens": 1024000,
"max_input_tokens": 1048576,
"max_output_tokens": 1024000,
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 2.8e-07,
@ -44497,7 +44557,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash-0731"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1e-08
},
"nebius/deepseek-ai/DeepSeek-V4-Pro": {
"max_input_tokens": 1048576,
@ -44508,29 +44569,32 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1.4e-07
},
"nebius/deepseek-ai/DeepSeek-V4-Pro-0813": {
"input_cost_per_token": 1.32e-06,
"litellm_provider": "nebius",
"max_input_tokens": 979000,
"max_input_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 3.96e-06,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro-0813",
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"supports_function_calling": true,
"supports_reasoning": true
"supports_reasoning": true,
"cache_read_input_token_cost": 4e-08
},
"nebius/deepseek-ai/DeepSeek-V4.1-Flash": {
"input_cost_per_token": 3e-07,
"litellm_provider": "nebius",
"max_input_tokens": 1048000,
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://tokenfactory.nebius.com/endpoints?modals=endpoint-details&model-id=deepseek-ai/DeepSeek-V4.1-Flash",
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true
"supports_vision": true,
"cache_read_input_token_cost": 1e-08
},
"nebius/MiniMaxAI/MiniMax-M2.5": {
"max_input_tokens": 196608,
@ -44553,7 +44617,8 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M3"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 6e-08
},
"nebius/moonshotai/Kimi-K2.6": {
"max_input_tokens": 262144,
@ -44565,7 +44630,8 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K2.6"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1.6e-07
},
"nebius/moonshotai/Kimi-K2.7-Code": {
"max_input_tokens": 262144,
@ -44577,10 +44643,11 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/moonshotai%2FKimi-K2.7-Code"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1.5e-07
},
"nebius/moonshotai/Kimi-K3": {
"max_input_tokens": 1024000,
"max_input_tokens": 1048576,
"max_output_tokens": 1024000,
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
@ -44589,7 +44656,8 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K3"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 3e-07
},
"nebius/NousResearch/Hermes-4-405B": {
"max_input_tokens": 131072,
@ -44600,7 +44668,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-405B"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 2e-07
},
"nebius/NousResearch/Hermes-4-70B": {
"max_input_tokens": 131072,
@ -44644,7 +44713,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNVIDIA-Nemotron-3-Nano-30B-A3B"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1.2e-08
},
"nebius/nvidia/Nemotron-3-Nano-Omni": {
"max_input_tokens": 262144,
@ -44667,7 +44737,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2Fnemotron-3-super-120b-a12b"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 6e-08
},
"nebius/nvidia/Nemotron-3-Ultra-550b-a55b": {
"max_input_tokens": 1048576,
@ -44678,7 +44749,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Ultra-550b-a55b"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1.2e-07
},
"nebius/nvidia/Nemotron-3_5-Lightning": {
"max_input_tokens": 1048576,
@ -44689,7 +44761,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3_5-Lightning"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 3e-08
},
"nebius/openai/gpt-oss-120b": {
"max_input_tokens": 131072,
@ -44700,7 +44773,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/openai%2Fgpt-oss-120b"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1e-07
},
"nebius/openbmb/MiniCPM-V-4_5": {
"max_input_tokens": 32000,
@ -44710,7 +44784,8 @@
"litellm_provider": "nebius",
"mode": "chat",
"supports_vision": true,
"source": "https://tokenfactory.nebius.com/models/catalog/image2text/openbmb%2FMiniCPM-V-4_5"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1.3e-07
},
"nebius/Qwen/Qwen3-235B-A22B-Instruct-2507": {
"max_input_tokens": 262144,
@ -44720,7 +44795,8 @@
"litellm_provider": "nebius",
"mode": "chat",
"supports_function_calling": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-235B-A22B-Instruct-2507"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 5e-08
},
"nebius/Qwen/Qwen3-30B-A3B-Instruct-2507": {
"max_input_tokens": 262144,
@ -44730,7 +44806,8 @@
"litellm_provider": "nebius",
"mode": "chat",
"supports_function_calling": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-30B-A3B-Instruct-2507"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1e-07
},
"nebius/Qwen/Qwen3-Next-80B-A3B-Thinking": {
"max_input_tokens": 128000,
@ -44752,7 +44829,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.5-397B-A17B"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1.2e-07
},
"nebius/Qwen/Qwen3.8-27B": {
"input_cost_per_token": 4.5e-07,
@ -44760,9 +44838,10 @@
"max_input_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3e-06,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.8-27B",
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"supports_function_calling": true,
"supports_reasoning": true
"supports_reasoning": true,
"cache_read_input_token_cost": 1e-07
},
"nebius/zai-org/GLM-5.1": {
"max_input_tokens": 202752,
@ -44773,7 +44852,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.1"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 2.6e-07
},
"nebius/zai-org/GLM-5.2": {
"max_input_tokens": 1048576,
@ -44784,21 +44864,23 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.2"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1.4e-07
},
"nebius/zai-org/GLM-5.3": {
"input_cost_per_token": 1.4e-06,
"litellm_provider": "nebius",
"max_input_tokens": 1024000,
"max_input_tokens": 1048576,
"max_output_tokens": 1024000,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3",
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"supports_function_calling": true,
"supports_reasoning": true
"supports_reasoning": true,
"cache_read_input_token_cost": 2.6e-07
},
"nebius/zai-org/GLM-5.3-Flash": {
"max_input_tokens": 1024000,
"max_input_tokens": 1048576,
"max_output_tokens": 1024000,
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 5e-07,
@ -44806,8 +44888,9 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3-Flash",
"supports_vision": true
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"supports_vision": true,
"cache_read_input_token_cost": 3e-08
},
"nebius/BAAI/bge-en-icl": {
"max_input_tokens": 32768,
@ -49811,7 +49894,7 @@
"mode": "chat",
"output_cost_per_token": 3e-05,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/",
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json",
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -49833,7 +49916,9 @@
"model": "anthropic.claude-opus-5",
"litellm_provider": "bedrock_mantle",
"region": "us-gov-west-1"
}
},
"input_cost_per_token_batches": 3e-06,
"output_cost_per_token_batches": 1.5e-05
},
"us-gov.anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -52920,6 +53005,7 @@
},
"vertex_ai/gemini-3.1-flash-lite-image": {
"deprecation_date": "2027-06-28",
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_batches": 1.25e-08,
"cache_read_input_token_cost_flex": 1.25e-08,
@ -54148,7 +54234,7 @@
"voyage/voyage-lite-01": {
"input_cost_per_token": 1e-07,
"litellm_provider": "voyage",
"max_input_tokens": 4096,
"max_input_tokens": 4000,
"mode": "embedding",
"output_cost_per_token": 0.0
},
@ -64807,7 +64893,7 @@
"supports_vision": false
},
"novita/deepseek/deepseek-v4-pro-0813": {
"cache_read_input_token_cost": 1.3200000000000002e-07,
"cache_read_input_token_cost": 4.4e-08,
"input_cost_per_token": 1.32e-06,
"litellm_provider": "novita",
"max_input_tokens": 1048576,
@ -64919,13 +65005,13 @@
"supports_vision": false
},
"novita/mindai/macaron-v1-venti": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 1.5e-06,
"cache_read_input_token_cost": 1.65e-07,
"input_cost_per_token": 8.25e-07,
"litellm_provider": "novita",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.5e-06,
"output_cost_per_token": 2.475e-06,
"source": "https://novita.ai/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -65028,13 +65114,13 @@
"supports_vision": false
},
"novita/mindai/macaron-v1-tall": {
"cache_read_input_token_cost": 8e-08,
"input_cost_per_token": 4.5000000000000003e-07,
"cache_read_input_token_cost": 4.4e-08,
"input_cost_per_token": 2.475e-07,
"litellm_provider": "novita",
"max_input_tokens": 262144,
"max_output_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 2.6e-06,
"output_cost_per_token": 1.43e-06,
"source": "https://novita.ai/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -66459,9 +66545,9 @@
"deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": {
"max_output_tokens": 1048576,
"max_input_tokens": 1048576,
"input_cost_per_token": 8e-08,
"input_cost_per_token": 6e-08,
"output_cost_per_token": 1.8e-07,
"cache_read_input_token_cost": 1.6e-08,
"cache_read_input_token_cost": 1.5e-08,
"supports_prompt_caching": true,
"litellm_provider": "deepinfra",
"mode": "chat",
@ -66840,8 +66926,8 @@
"deepinfra/google/gemma-4-31B-it": {
"max_output_tokens": 262144,
"max_input_tokens": 262144,
"input_cost_per_token": 1.3e-07,
"output_cost_per_token": 3.8e-07,
"input_cost_per_token": 2e-07,
"output_cost_per_token": 4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_function_calling": true,
@ -68552,7 +68638,10 @@
"model": "anthropic.claude-opus-5",
"litellm_provider": "bedrock_mantle",
"region": "us-gov-west-1"
}
},
"input_cost_per_token_batches": 3e-06,
"output_cost_per_token_batches": 1.5e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json"
},
"bedrock/us-gov-west-1/anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -68827,7 +68916,10 @@
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
"supports_xhigh_reasoning_effort": true,
"input_cost_per_token_batches": 3e-06,
"output_cost_per_token_batches": 1.5e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json"
},
"bedrock/us-gov-east-1/anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -69850,6 +69942,14 @@
"cache_read_input_token_cost": 3e-08,
"source": "https://docs.perplexity.ai/docs/agent-api/models"
},
"perplexity/google/gemini-4-argon": {
"litellm_provider": "perplexity",
"mode": "responses",
"input_cost_per_token": 2e-06,
"output_cost_per_token": 1e-05,
"cache_read_input_token_cost": 1e-07,
"source": "https://docs.perplexity.ai/docs/agent-api/models"
},
"perplexity/google/gemini-3.6-flash": {
"litellm_provider": "perplexity",
"mode": "responses",
@ -69977,6 +70077,14 @@
"cache_read_input_token_cost": 1.15e-09,
"source": "https://docs.perplexity.ai/docs/agent-api/models"
},
"perplexity/perplexity/nemotron-3.5-super-120b-a12b": {
"litellm_provider": "perplexity",
"mode": "responses",
"input_cost_per_token": 2e-07,
"output_cost_per_token": 5e-07,
"cache_read_input_token_cost": 4e-08,
"source": "https://docs.perplexity.ai/docs/agent-api/models"
},
"perplexity/perplexity/nemotron-3-ultra-550b-a55b": {
"litellm_provider": "perplexity",
"mode": "responses",
@ -70102,12 +70210,12 @@
"input_cost_per_token": 1.5e-06,
"litellm_provider": "vertex_ai",
"max_output_tokens": 57920,
"max_input_tokens": 57920,
"max_input_tokens": 131072,
"mode": "chat",
"output_cost_per_reasoning_token": 9e-06,
"output_cost_per_token": 9e-06,
"output_cost_per_video_token": 1.75e-05,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-1-1-flash",
"supported_endpoints": [
"/v1beta/interactions"
],
@ -74503,12 +74611,14 @@
"input_cost_per_token": 5.5e-06,
"cache_read_input_token_cost": 5.5e-07,
"output_cost_per_token": 2.75e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-northeast-1/index.json",
"token_count_model": {
"model": "anthropic.claude-opus-5",
"litellm_provider": "bedrock_mantle",
"region": "ap-southeast-4"
}
},
"input_cost_per_token_batches": 2.75e-06,
"output_cost_per_token_batches": 1.375e-05
},
"apac.anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -75105,7 +75215,7 @@
"supports_function_calling": true,
"input_cost_per_token": 2e-06,
"output_cost_per_token": 1e-05,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost": 1e-07,
"source": "https://docs.perplexity.ai/docs/agent-api/models"
},
"perplexity/anthropic/claude-fable-5-1": {
@ -75153,6 +75263,9 @@
"output_cost_per_token": 5e-05,
"cache_read_input_token_cost": 1e-06,
"cache_creation_input_token_cost": 1.25e-05,
"input_cost_per_token_above_272k_tokens": 2e-05,
"output_cost_per_token_above_272k_tokens": 7.5e-05,
"cache_read_input_token_cost_above_272k_tokens": 2e-06,
"source": "https://docs.perplexity.ai/docs/agent-api/models"
},
"perplexity/openai/gpt-6-luna": {
@ -76620,6 +76733,9 @@
"input_cost_per_token": 1e-07,
"output_cost_per_token": 5e-07,
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
"source": "https://docs.perplexity.ai/docs/agent-api/models"
},
"us-gov.anthropic.claude-haiku-5-5": {
@ -76954,23 +77070,19 @@
},
"vertex_ai/gemini-robotics-er-2-preview": {
"cache_read_input_token_cost": 1e-07,
"cache_read_input_token_cost_batches": 5e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 1e-06,
"input_cost_per_token_batches": 5e-07,
"litellm_provider": "vertex_ai",
"max_input_tokens": 131072,
"max_output_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 5e-06,
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
"/v1/completions"
],
"supported_modalities": [
"text",
@ -77000,23 +77112,19 @@
},
"gemini-robotics-er-2-preview": {
"cache_read_input_token_cost": 1e-07,
"cache_read_input_token_cost_batches": 5e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 1e-06,
"input_cost_per_token_batches": 5e-07,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 131072,
"max_output_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 5e-06,
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
"/v1/completions"
],
"supported_modalities": [
"text",
@ -77102,6 +77210,7 @@
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-4-argon": {
"deprecation_date": "2027-10-13",
"cache_read_input_token_cost": 1e-07,
"cache_read_input_token_cost_batches": 5e-08,
"cache_read_input_token_cost_flex": 5e-08,
@ -77510,9 +77619,10 @@
"max_output_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 1e-06,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/reflection%2Fbeam",
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"supports_function_calling": true,
"supports_reasoning": true
"supports_reasoning": true,
"cache_read_input_token_cost": 5e-08
},
"baseten/deepseek-ai/DeepSeek-V4.1-Flash-1017": {
"cache_read_input_token_cost": 7e-09,
@ -81783,5 +81893,55 @@
"supports_response_schema": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
},
"nebius/deepseek-ai/DeepSeek-V4.1-Flash-1017": {
"max_input_tokens": 1048576,
"input_cost_per_token": 3e-07,
"cache_read_input_token_cost": 6e-09,
"output_cost_per_token": 1.2e-06,
"litellm_provider": "nebius",
"mode": "chat",
"supports_vision": true,
"supports_reasoning": true,
"supports_function_calling": true,
"source": "https://tokenfactory.nebius.com/api/public/models_info"
},
"nebius/meta-models/muse-spark-open": {
"max_input_tokens": 1048576,
"input_cost_per_token": 1.25e-06,
"cache_read_input_token_cost": 1.5e-07,
"output_cost_per_token": 4.25e-06,
"litellm_provider": "nebius",
"mode": "chat",
"supports_vision": true,
"supports_video_input": true,
"supports_reasoning": true,
"supports_function_calling": true,
"source": "https://tokenfactory.nebius.com/api/public/models_info"
},
"nebius/MiniMaxAI/MiniMax-M3.1-Flash": {
"max_input_tokens": 1048576,
"input_cost_per_token": 1.5e-07,
"cache_read_input_token_cost": 1.5e-08,
"output_cost_per_token": 6e-07,
"litellm_provider": "nebius",
"mode": "chat",
"supports_vision": true,
"supports_reasoning": true,
"supports_function_calling": true,
"source": "https://tokenfactory.nebius.com/api/public/models_info"
},
"nebius/nvidia/Nemotron-3.5-Super-VL": {
"max_input_tokens": 1048576,
"input_cost_per_token": 3e-07,
"cache_read_input_token_cost": 6e-08,
"output_cost_per_token": 9e-07,
"litellm_provider": "nebius",
"mode": "chat",
"supports_vision": true,
"supports_video_input": true,
"supports_reasoning": true,
"supports_function_calling": true,
"source": "https://tokenfactory.nebius.com/api/public/models_info"
}
}

View file

@ -23,7 +23,12 @@ from litellm.router import Router
from litellm.router_api import fallback_chain
from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager, get_valid_models, potential_model_names_in_lookup_order
from litellm.utils import (
AZURE_MODEL_PLACEHOLDER,
ProviderConfigManager,
get_valid_models,
potential_model_names_in_lookup_order,
)
LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders)
_PROVIDERS_SENDING_THE_DEPLOYMENT_PATH_AS_THE_MODEL_ID: Final = frozenset({LlmProviders.TOGETHER_AI.value})
@ -375,6 +380,10 @@ def _ids_routing_to_catalog_models(
catalog_ids: Sequence[str],
routes_to_model: Callable[[str, str], bool],
) -> tuple[str, ...]:
if name_pattern.count("*") > deployment_pattern.count("*"):
return _catalog_ids_a_wider_name_matches(
name_pattern, deployment_pattern, provider, catalog_ids, routes_to_model
)
if name_pattern.count("*") != deployment_pattern.count("*") or _lists_name_fragments(
name_pattern, deployment_pattern
):
@ -419,6 +428,23 @@ def _ids_routing_to_catalog_models(
)
def _catalog_ids_a_wider_name_matches(
name_pattern: str,
deployment_pattern: str,
provider: str,
catalog_ids: Sequence[str],
routes_to_model: Callable[[str, str], bool],
) -> tuple[str, ...]:
name_regex: Final = _wildcard_name_regex(name_pattern)
return tuple(
catalog_id
for catalog_id in _ids_routing_to_catalog_models(
deployment_pattern, deployment_pattern, provider, catalog_ids, routes_to_model
)
if name_regex.fullmatch(catalog_id)
)
def _model_info_field(deployment: Mapping[str, object], field: str) -> object:
model_info: Final = deployment.get("model_info")
return model_info.get(field) if is_str_keyed_mapping(model_info) else None
@ -456,11 +482,12 @@ def _routes_to_model(
return lambda listed_id, model: routed_models is None or model in routed_models(listed_id, team_id)
def _deployment_pattern_naming_listed_ids(wildcard_suffix: str, litellm_params: LiteLLM_Params | None) -> str | None:
def _deployment_pattern_naming_listed_ids(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> str | None:
if litellm_params is None or "/" not in litellm_params.model or not _check_wildcard_routing(litellm_params.model):
return None
name_suffix: Final = wildcard_model.split("/", 1)[-1]
deployment_suffix: Final = litellm_params.model.partition("/")[2]
return litellm_params.model if wildcard_suffix != "*" or deployment_suffix != "*" else None
return litellm_params.model if name_suffix != "*" or deployment_suffix != "*" else None
def get_known_models_from_wildcard(
@ -468,6 +495,43 @@ def get_known_models_from_wildcard(
litellm_params: LiteLLM_Params | None = None,
routes_to_model: Callable[[str, str], bool] = lambda _listed_id, _model: True,
) -> list[str]:
provider_id_prefix: Final = _custom_llm_provider_prefix(wildcard_model, litellm_params)
if litellm_params is None or provider_id_prefix is None:
return list(_known_models_from_wildcard(wildcard_model, litellm_params, routes_to_model))
return [
listed_id.removeprefix(provider_id_prefix)
for listed_id in _known_models_from_wildcard(
wildcard_model,
litellm_params.model_copy(update={"model": f"{provider_id_prefix}{litellm_params.model}"}),
lambda listed_id, model: routes_to_model(
listed_id.removeprefix(provider_id_prefix), model.removeprefix(provider_id_prefix)
),
)
]
def _custom_llm_provider_prefix(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> str | None:
if (
litellm_params is None
or not litellm_params.custom_llm_provider
or "/" in wildcard_model
or wildcard_model == "*"
or "/" in litellm_params.model
or not _check_wildcard_routing(litellm_params.model)
):
return None
return f"{litellm_params.custom_llm_provider}/"
def _without_azure_placeholder(catalog_ids: Sequence[str]) -> tuple[str, ...]:
return tuple(catalog_id for catalog_id in catalog_ids if catalog_id != AZURE_MODEL_PLACEHOLDER)
def _known_models_from_wildcard(
wildcard_model: str,
litellm_params: LiteLLM_Params | None,
routes_to_model: Callable[[str, str], bool],
) -> tuple[str, ...]:
wildcard_model_to_expand: Final = (
litellm_params.model
if wildcard_model == "*"
@ -476,15 +540,14 @@ def get_known_models_from_wildcard(
and "/" in litellm_params.model
else wildcard_model
)
try:
wildcard_provider_prefix, wildcard_suffix = wildcard_model_to_expand.split("/", 1)
except ValueError: # safely fail
return []
deployment_pattern_naming_listed_ids: Final = (
wildcard_model_to_expand
if wildcard_model_to_expand != wildcard_model
else _deployment_pattern_naming_listed_ids(wildcard_suffix, litellm_params)
else _deployment_pattern_naming_listed_ids(wildcard_model, litellm_params)
)
wildcard_provider_prefix, has_provider_prefix, wildcard_suffix = wildcard_model_to_expand.partition("/")
if not has_provider_prefix and deployment_pattern_naming_listed_ids is None:
return ()
# Use provider from litellm_params when available, otherwise from wildcard prefix
# (e.g., "openai" from "openai/*" - needed for BYOK where wildcard isn't in router)
@ -501,16 +564,14 @@ def get_known_models_from_wildcard(
wildcard_models = get_provider_models(provider=provider, litellm_params=litellm_params)
if wildcard_models is None:
return []
return ()
if deployment_pattern_naming_listed_ids is not None:
return list(
_ids_routing_to_catalog_models(
name_pattern=wildcard_model,
deployment_pattern=deployment_pattern_naming_listed_ids,
provider=provider,
catalog_ids=wildcard_models,
routes_to_model=routes_to_model,
)
return _ids_routing_to_catalog_models(
name_pattern=wildcard_model,
deployment_pattern=deployment_pattern_naming_listed_ids,
provider=provider,
catalog_ids=wildcard_models if has_provider_prefix else _without_azure_placeholder(wildcard_models),
routes_to_model=routes_to_model,
)
if wildcard_suffix != "*":
model_prefix: Final = wildcard_suffix.replace("*", "")
@ -526,7 +587,7 @@ def get_known_models_from_wildcard(
# add model prefix to wildcard models
wildcard_models = [f"{model_prefix}{model.removeprefix(provider_id_prefix)}" for model in wildcard_models]
return list(
return tuple(
dict.fromkeys(_under_wildcard_prefix(model, wildcard_provider_prefix, provider) for model in wildcard_models)
)
@ -564,16 +625,6 @@ def _names_no_deployment_serves_by_name(
return tuple(name for name in names if not deployed_by_name(name, team_id))
def _wildcard_pattern_to_expand(model_name: str, litellm_model: str) -> str | None:
if not _check_wildcard_routing(model_name):
return None
if "/" in model_name or model_name == "*":
return model_name
if _check_wildcard_routing(litellm_model):
return litellm_model
return model_name
def _can_expand_to_an_allowed_name(wildcard_pattern: str, allowed_model_names: Collection[str] | None) -> bool:
if allowed_model_names is None:
return True
@ -601,10 +652,10 @@ def expand_wildcard_deployments_for_model_info(
raw_params = deployment.get("litellm_params")
litellm_params_dict: dict[str, object] = raw_params if isinstance(raw_params, dict) else {}
litellm_model = str(litellm_params_dict.get("model") or "")
if (wildcard_pattern := _wildcard_pattern_to_expand(model_name, litellm_model)) is None:
if not _check_wildcard_routing(model_name):
expanded.append(deployment)
continue
if not _can_expand_to_an_allowed_name(wildcard_pattern, allowed_model_names):
if not _can_expand_to_an_allowed_name(model_name, allowed_model_names):
continue
try:
@ -613,7 +664,7 @@ def expand_wildcard_deployments_for_model_info(
expanded.append(deployment)
continue
expanded_names = get_known_models_from_wildcard(
wildcard_model=wildcard_pattern,
wildcard_model=model_name,
litellm_params=litellm_params,
routes_to_model=_routes_to_model(routed_models, _deployment_team_id(deployment)),
)

View file

@ -1889,6 +1889,13 @@ def _response_cost_header_value(hidden_params: Mapping[str, object]) -> float |
def cost_headers(
response_cost: float | str | None, litellm_logging_obj: LiteLLMLoggingObj | None
) -> Mapping[str, str | None]:
if (
response_cost == 0
and litellm_logging_obj is not None
and litellm_logging_obj.prepared_cost is not None
and not litellm_logging_obj.prepared_cost.price_known
):
return {}
cost_breakdown: Final = _get_cost_breakdown_from_logging_obj(
litellm_logging_obj=litellm_logging_obj, response_cost=response_cost
)

View file

@ -0,0 +1,221 @@
"""Which sessions a logs page shows and how many match, read from the newest rows instead of every row. Reads grow
until one settles both, so the result is what grouping every row would give."""
import math
from collections.abc import Awaitable, Callable, Sequence
from dataclasses import dataclass
from typing import Final, Protocol, TypeAlias
from typing_extensions import ReadOnly, TypedDict
MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')"
AGENT_CALL_TYPE_SQL: Final = "'asend_message'"
SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)"
SESSION_REPRESENTATIVE_ORDER_SQL: Final = (
f"(call_type = {AGENT_CALL_TYPE_SQL}) DESC, "
f'CASE WHEN call_type = {AGENT_CALL_TYPE_SQL} THEN "endTime" END DESC NULLS LAST, '
f'call_type IN {MCP_CALL_TYPES_SQL}, "startTime" DESC, request_id'
)
class _ReadRow(TypedDict):
session_key: ReadOnly[str | None]
api_key: ReadOnly[str | None]
last_activity: ReadOnly[str | None]
settled: ReadOnly[bool | None]
rows_read: ReadOnly[int]
session_count: ReadOnly[int]
class _SupportsQueryRaw(Protocol):
async def query_raw(self, query: str, *args: object) -> Sequence[_ReadRow]: ...
@dataclass(frozen=True, slots=True)
class Session:
key: str
api_key: str
last_activity: str
@dataclass(frozen=True, slots=True)
class SessionPage:
sessions: tuple[Session, ...]
has_more: bool
next_cursor: str | None
total: int
total_is_capped: bool
@dataclass(frozen=True, slots=True)
class _Request:
cursor: tuple[str, str, str] | None
offset: int
page_limit: int
sort_desc: bool
status: str | None
cap: int
def first_read_size(self) -> int | None:
"""Every row when oldest first, or when filtered by outcome, which any of a session's rows can decide. Otherwise
the fewest that could settle a page: a session past the cap and an older row."""
return self.cap + 2 if self.sort_desc and self.status is None else None
@dataclass(frozen=True, slots=True)
class _Read:
candidates: tuple[Session, ...]
last_settled: bool
rows_read: int
session_count: int
saw_every_row: bool
def settles(self, request: _Request) -> bool:
page_settled: Final = (
request.page_limit <= 0
or self.saw_every_row
or (len(self.candidates) > request.page_limit and self.last_settled)
)
return page_settled and (self.saw_every_row or self.session_count > request.cap)
def next_size(self, request: _Request) -> int:
rows_per_session: Final = self.rows_read / self.session_count
return max(2 * self.rows_read, math.ceil(1.25 * (request.cap + 1) * rows_per_session))
_Reader: TypeAlias = Callable[[int | None], Awaitable[_Read]]
def _parse_session_cursor(session_cursor: str | None) -> tuple[str, str, str] | None:
if session_cursor is None or session_cursor.count("|") < 2:
return None
last_activity, _, rest = session_cursor.partition("|")
api_key, _, session_key = rest.partition("|")
if not last_activity or not session_key:
return None
return (last_activity, session_key, api_key)
async def read_session_page(
db: _SupportsQueryRaw,
*,
where_clause: str,
params: Sequence[object],
status: str | None,
page: int,
page_size: int,
sort_desc: bool,
session_cursor: str | None,
cap: int,
) -> SessionPage:
"""One page of sessions after ``session_cursor``, or after ``(page - 1) * page_size`` sessions, keeping only those
whose outcome is ``status`` when given. ``where_clause`` reads ``params`` as ``$1`` onward."""
cursor: Final = _parse_session_cursor(session_cursor)
offset: Final = (page - 1) * page_size if cursor is None else 0
request: Final = _Request(cursor, offset, min(page_size, cap - offset), sort_desc, status, cap)
return await _session_page(_sql_reader(db, where_clause, params, request), request)
async def _session_page(read: _Reader, request: _Request) -> SessionPage:
last = await read(request.first_read_size()) # rebind-ok: each read replaces the last until one settles the page
while not last.settles(request):
last = await read(last.next_size(request))
has_more: Final = len(last.candidates) > request.page_limit
sessions: Final = last.candidates[: request.page_limit]
final: Final = sessions[-1] if has_more and sessions else None
return SessionPage(
sessions=sessions,
has_more=has_more,
next_cursor=f"{final.last_activity}|{final.api_key}|{final.key}" if final else None,
total=min(last.session_count, request.cap),
total_is_capped=last.session_count > request.cap,
)
def _sessions_sql(request: _Request, status_index: int) -> str:
"""Each session's last activity, from its rows in ``scanned``, keeping only sessions whose outcome is the status."""
if request.status is None:
return (
'SELECT session_key, api_key, MAX("startTime") AS last_activity FROM scanned GROUP BY session_key, api_key'
)
return f"""
SELECT session_key, api_key, last_activity
FROM (
SELECT DISTINCT ON (session_key, api_key)
session_key,
api_key,
status,
MAX("startTime") OVER (PARTITION BY session_key, api_key) AS last_activity
FROM scanned
ORDER BY session_key, api_key, {SESSION_REPRESENTATIVE_ORDER_SQL}
) AS outcomes
WHERE COALESCE(status, 'success') = ${status_index}
"""
def _sql_reader(db: _SupportsQueryRaw, where_clause: str, params: Sequence[object], request: _Request) -> _Reader:
status_index: Final = len(params) + 1
status_args: Final = () if request.status is None else (request.status,)
cursor_index: Final = status_index + len(status_args)
after_cursor: Final = (
f"WHERE (last_activity, session_key, api_key) {'<' if request.sort_desc else '>'} "
f"(${cursor_index}::timestamp, ${cursor_index + 1}, ${cursor_index + 2})"
if request.cursor
else ""
)
outcome_columns: Final = ', status, call_type, "endTime", request_id' if request.status is not None else ""
direction: Final = "DESC" if request.sort_desc else "ASC"
page_rows: Final = request.page_limit + 1 if request.page_limit > 0 else 0
skip: Final = f"OFFSET {request.offset}" if page_rows and request.offset else ""
async def read(size: int | None) -> _Read:
# LEFT JOIN keeps the summary row when no session is on the page.
rows: Final = await db.query_raw(
f"""
WITH scanned AS (
SELECT {SESSION_KEY_EXPR} AS session_key, api_key, "startTime"{outcome_columns}
FROM "LiteLLM_SpendLogs"
WHERE {where_clause}
{f'ORDER BY "startTime" DESC LIMIT {size}' if size is not None else ""}
),
sessions AS ({_sessions_sql(request, status_index)}),
summary AS (
SELECT COUNT(*) AS rows_read,
MIN("startTime") AS oldest,
(SELECT COUNT(*) FROM sessions) AS session_count
FROM scanned
)
SELECT page.session_key,
page.api_key,
page.last_activity::text AS last_activity,
page.last_activity > summary.oldest AS settled,
summary.rows_read,
summary.session_count
FROM summary
LEFT JOIN LATERAL (
SELECT session_key, api_key, last_activity
FROM sessions
{after_cursor}
ORDER BY last_activity {direction}, session_key {direction}, api_key {direction}
LIMIT {page_rows} {skip}
) AS page ON TRUE
ORDER BY page.last_activity {direction}, page.session_key {direction}, page.api_key {direction}
""",
*params,
*status_args,
*(request.cursor or ()),
)
rows_read: Final = int(rows[0]["rows_read"])
return _Read(
candidates=tuple(
Session(str(row["session_key"]), str(row["api_key"]), str(row["last_activity"]))
for row in rows
if row["session_key"] is not None
),
last_settled=bool(rows[-1]["settled"]),
rows_read=rows_read,
session_count=int(rows[0]["session_count"]),
saw_every_row=size is None or rows_read < size,
)
return read

View file

@ -52,6 +52,13 @@ from litellm.proxy.auth.authorization_dependencies import (
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.openai_error_payload import error_status_code, openai_error_param, openai_error_type
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.spend_tracking.session_page import (
AGENT_CALL_TYPE_SQL,
MCP_CALL_TYPES_SQL,
SESSION_KEY_EXPR,
SESSION_REPRESENTATIVE_ORDER_SQL,
read_session_page,
)
from litellm.proxy.spend_tracking.spend_capture_rate import (
ProviderBillingCredentialMissing,
ProviderBillingRequestFailed,
@ -87,23 +94,15 @@ router: Final = APIRouter()
SPEND_LOGS_PAGINATION_COUNT_CAP: Final = 10000
_DECODED_JSON: Final = TypeAdapter(object)
_SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)"
_SESSION_GROUP_KEY_SQL: Final = f"{_SESSION_KEY_EXPR}, api_key"
_MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')"
_AGENT_CALL_TYPE_SQL: Final = "'asend_message'"
_SESSION_REPRESENTATIVE_ORDER_SQL: Final = (
f"(call_type = {_AGENT_CALL_TYPE_SQL}) DESC, "
f'CASE WHEN call_type = {_AGENT_CALL_TYPE_SQL} THEN "endTime" END DESC NULLS LAST, '
f'call_type IN {_MCP_CALL_TYPES_SQL}, "startTime" DESC, request_id'
)
_SESSION_GROUP_KEY_SQL: Final = f"{SESSION_KEY_EXPR}, api_key"
_BATCH_CALL_TYPES_SQL: Final = "('acreate_batch', 'create_batch', 'aretrieve_batch', 'retrieve_batch')"
_SPAN_TYPE_SQL_CONDITIONS: Final[Mapping[str, str]] = MappingProxyType(
{
"mcp": f"call_type IN {_MCP_CALL_TYPES_SQL}",
"agent": f"call_type = {_AGENT_CALL_TYPE_SQL}",
"mcp": f"call_type IN {MCP_CALL_TYPES_SQL}",
"agent": f"call_type = {AGENT_CALL_TYPE_SQL}",
"batch": f"call_type IN {_BATCH_CALL_TYPES_SQL}",
"llm": (
f"(call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL} "
f"(call_type NOT IN {MCP_CALL_TYPES_SQL} AND call_type != {AGENT_CALL_TYPE_SQL} "
f"AND call_type NOT IN {_BATCH_CALL_TYPES_SQL})"
),
}
@ -2948,23 +2947,6 @@ async def ui_view_spend_logs(
sql_params.append(json.dumps(used_client_oauth_token))
p += 1
if status_filter is not None and group_by_session is True and not is_search_lookup:
session_filter_conditions: Final = " AND ".join(sql_conditions) or "TRUE"
sql_conditions.append(
f"""({_SESSION_GROUP_KEY_SQL}) IN (
SELECT session_key, api_key FROM (
SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL})
{_SESSION_KEY_EXPR} AS session_key, api_key, status
FROM "LiteLLM_SpendLogs"
WHERE {session_filter_conditions}
ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL}
) AS session_outcomes
WHERE COALESCE(status, 'success') = ${p}
)"""
)
sql_params.append(status_filter)
p += 1
if (
group_by_session is True
and not is_v2
@ -2977,12 +2959,30 @@ async def ui_view_spend_logs(
sql_conditions=sql_conditions,
sql_params=sql_params,
next_param_index=p,
status_filter=status_filter,
page=page,
page_size=page_size,
sort_desc=order_direction != "asc",
session_cursor=session_cursor,
)
if status_filter is not None and group_by_session is True and not is_search_lookup:
session_filter_conditions: Final = " AND ".join(sql_conditions) or "TRUE"
sql_conditions.append(
f"""({_SESSION_GROUP_KEY_SQL}) IN (
SELECT session_key, api_key FROM (
SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL})
{SESSION_KEY_EXPR} AS session_key, api_key, status
FROM "LiteLLM_SpendLogs"
WHERE {session_filter_conditions}
ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL}
) AS session_outcomes
WHERE COALESCE(status, 'success') = ${p}
)"""
)
sql_params.append(status_filter)
p += 1
# Build the ORDER BY expression. ttft_ms is computed from
# completionStartTime - startTime; non-streaming rows (where
# completionStartTime is null or equals endTime) yield NULL, so we
@ -3031,7 +3031,7 @@ async def ui_view_spend_logs(
{_SPEND_LOG_LIST_COLUMNS}
FROM "LiteLLM_SpendLogs"
WHERE {joined_conditions}
ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL}
ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL}
) AS session_representatives
ORDER BY {exact_request_id_first}{_order_expr} {_sql_dir}{_nulls_clause}, request_id
LIMIT ${p} OFFSET ${p + 1}
@ -3080,22 +3080,6 @@ async def ui_view_spend_logs(
raise handle_exception_on_proxy(e)
class _SessionPageRow(TypedDict):
session_key: ReadOnly[str]
api_key: ReadOnly[str]
last_activity: ReadOnly[str]
def _parse_session_cursor(session_cursor: str | None) -> tuple[str, str, str] | None:
if session_cursor is None or session_cursor.count("|") < 2:
return None
last_activity, _, rest = session_cursor.partition("|")
api_key, _, session_key = rest.partition("|")
if not last_activity or not session_key:
return None
return (last_activity, session_key, api_key)
async def _fetch_session_representatives(
prisma_client: "PrismaClient",
where_clause: str,
@ -3104,16 +3088,18 @@ async def _fetch_session_representatives(
session_keys: Sequence[tuple[str, str]],
) -> list[dict[str, object]]: # mutable-ok: _build_ui_spend_logs_response writes session counts onto each row
"""Fetch the final agent outcome, or newest non-MCP row, of each ``(session_key, api_key)`` session, in ``session_keys`` order."""
# Lets Postgres use the session_id and request_id indexes.
rep_query: Final = f"""
SELECT * FROM (
SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL})
{_SPEND_LOG_LIST_COLUMNS}
FROM "LiteLLM_SpendLogs"
WHERE {where_clause}
AND (session_id = ANY(${next_param_index}::text[]) OR request_id = ANY(${next_param_index}::text[]))
AND ({_SESSION_GROUP_KEY_SQL}) IN (
SELECT * FROM unnest(${next_param_index}::text[], ${next_param_index + 1}::text[])
)
ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL}
ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL}
) AS session_representatives
"""
rep_rows: Final[Sequence[dict[str, object]]] = await _query_raw( # mutable-ok: rows are enriched in place
@ -3129,117 +3115,32 @@ async def _fetch_session_representatives(
return [rep_by_key[key] for key in session_keys if key in rep_by_key]
async def _count_grouped_sessions(
prisma_client: "PrismaClient",
where_clause: str,
sql_params: Sequence[object],
next_param_index: int,
) -> tuple[int, bool]:
"""Count the sessions matching the filter, returning ``(total, total_is_capped)`` bounded by the count cap."""
count_query: Final = f"""
SELECT COUNT(*) AS total_count
FROM (
SELECT 1
FROM "LiteLLM_SpendLogs"
WHERE {where_clause}
GROUP BY {_SESSION_GROUP_KEY_SQL}
LIMIT ${next_param_index}
) AS bounded_sessions
"""
count_rows: Final[Sequence[_SpendLogsCountRow]] = await _query_raw(
prisma_client, count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1
)
raw_total: Final = int(count_rows[0]["total_count"]) if count_rows else 0
return (
(SPEND_LOGS_PAGINATION_COUNT_CAP, True) if raw_total > SPEND_LOGS_PAGINATION_COUNT_CAP else (raw_total, False)
)
async def _ui_session_grouped_spend_logs(
prisma_client: "PrismaClient",
sql_conditions: Sequence[str],
sql_params: Sequence[object],
next_param_index: int,
status_filter: str | None,
page: int,
page_size: int,
sort_desc: bool,
session_cursor: str | None,
) -> Mapping[str, object]:
"""
One row per session, keyset-paginated by session last activity.
Sessions are derived on the fly from ``LiteLLM_SpendLogs`` (no extra
table): rows sharing a ``session_id`` and ``api_key`` form a session, rows
without a session id are singletons keyed by ``request_id``. A page is the
next ``page_size`` sessions ordered by ``(MAX(startTime), session_key,
api_key)``, resumed from the ``session_cursor`` keyset
``'<last_activity>|<api_key>|<session_key>'`` instead of an OFFSET, so
page depth does not degrade the query plan. A request for ``page > 1``
without a cursor (the UI jumping straight to the last page, or back to a
page it never walked through) falls back to ``OFFSET (page - 1) *
page_size``, trimmed to the end of the ``SPEND_LOGS_PAGINATION_COUNT_CAP``
window the capped ``total`` promises, so a page never runs past that total
and one starting at or past it returns no rows without a query. Each session is represented
by its final agent outcome (or newest non-MCP row), enriched by ``_build_ui_spend_logs_response``
exactly like the flat listing, and the response carries
``next_session_cursor`` / ``has_more`` while ``total`` counts sessions
(capped like the flat total). A page that runs out of sessions while still
holding some is itself the end of the list, so its ``total`` is
``offset + len(page)`` and the grouped count query is skipped; a page that
starts past the end says nothing about the total, so that one is counted.
"""
"""One row per session, keyset-paginated by last activity. ``read_session_page`` picks and counts the sessions,
and each is shown by its final agent outcome or newest non-MCP row."""
where_clause: Final = " AND ".join(sql_conditions) if sql_conditions else "TRUE"
cmp_op: Final = "<" if sort_desc else ">"
direction: Final = "DESC" if sort_desc else "ASC"
cursor: Final = _parse_session_cursor(session_cursor)
having_clause: Final = (
f'HAVING (MAX("startTime"), {_SESSION_GROUP_KEY_SQL}) {cmp_op} '
f"(${next_param_index}::timestamp, ${next_param_index + 1}, ${next_param_index + 2})"
if cursor
else ""
session_page: Final = await read_session_page(
prisma_client.db, # pyright: ignore[reportArgumentType] # PrismaWrapper forwards raw queries via __getattr__
where_clause=where_clause,
params=sql_params,
status=status_filter,
page=page,
page_size=page_size,
sort_desc=sort_desc,
session_cursor=session_cursor,
cap=SPEND_LOGS_PAGINATION_COUNT_CAP,
)
cursor_params: Final[tuple[object, ...]] = cursor if cursor else ()
limit_index: Final = next_param_index + len(cursor_params)
offset: Final = (page - 1) * page_size if cursor is None else 0
page_limit: Final = min(page_size, SPEND_LOGS_PAGINATION_COUNT_CAP - offset)
offset_params: Final[tuple[int, ...]] = (offset,) if offset and page_limit > 0 else ()
offset_clause: Final = f"OFFSET ${limit_index + 1}" if offset_params else ""
page_query: Final = f"""
SELECT {_SESSION_KEY_EXPR} AS session_key,
api_key,
MAX("startTime")::text AS last_activity
FROM "LiteLLM_SpendLogs"
WHERE {where_clause}
GROUP BY {_SESSION_GROUP_KEY_SQL}
{having_clause}
ORDER BY MAX("startTime") {direction}, {_SESSION_KEY_EXPR} {direction}, api_key {direction}
LIMIT ${limit_index} {offset_clause}
"""
page_rows: Final[Sequence[_SessionPageRow]] = (
()
if page_limit <= 0
else await _query_raw(prisma_client, page_query, *sql_params, *cursor_params, page_limit + 1, *offset_params)
)
has_more: Final = len(page_rows) > page_limit
visible_rows: Final = page_rows[:page_limit]
next_cursor: Final = (
f"{visible_rows[-1]['last_activity']}|{visible_rows[-1]['api_key']}|{visible_rows[-1]['session_key']}"
if has_more and visible_rows
else None
)
page_starts_inside_the_list: Final = offset == 0 or len(page_rows) > 0
page_ends_the_list: Final = cursor is None and page_limit > 0 and not has_more and page_starts_inside_the_list
total_records, total_is_capped = (
(offset + len(page_rows), False)
if page_ends_the_list
else await _count_grouped_sessions(prisma_client, where_clause, sql_params, next_param_index)
)
session_keys: Final = tuple((row["session_key"], row["api_key"]) for row in visible_rows)
session_keys: Final = tuple((session.key, session.api_key) for session in session_page.sessions)
data: Final[list[dict[str, object]]] = ( # mutable-ok: _build_ui_spend_logs_response writes onto each row
await _fetch_session_representatives(
prisma_client=prisma_client,
@ -3253,18 +3154,17 @@ async def _ui_session_grouped_spend_logs(
)
_hydrate_spend_log_metadata(data)
total_pages: Final = (total_records + page_size - 1) // page_size
response: Final[Mapping[str, object]] = await _build_ui_spend_logs_response(
prisma_client,
data,
total_records,
session_page.total,
page,
page_size,
total_pages,
(session_page.total + page_size - 1) // page_size,
enrich_session_counts=True,
total_is_capped=total_is_capped,
total_is_capped=session_page.total_is_capped,
)
return {**response, "next_session_cursor": next_cursor, "has_more": has_more}
return {**response, "next_session_cursor": session_page.next_cursor, "has_more": session_page.has_more}
class RequestResponsePayload(NamedTuple):
@ -4720,16 +4620,16 @@ async def _build_ui_spend_logs_response(
)
), 0)::bigint AS session_total_duration_ms,
COUNT(*) FILTER (
WHERE call_type IN {_MCP_CALL_TYPES_SQL}
WHERE call_type IN {MCP_CALL_TYPES_SQL}
)::int AS mcp_tool_call_count,
COALESCE(SUM(spend) FILTER (
WHERE call_type IN {_MCP_CALL_TYPES_SQL}
WHERE call_type IN {MCP_CALL_TYPES_SQL}
), 0)::double precision AS mcp_tool_call_spend,
COUNT(*) FILTER (WHERE LOWER(cache_hit) = 'true')::int AS session_cache_hit_count,
COUNT(*) FILTER (
WHERE call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL}
WHERE call_type NOT IN {MCP_CALL_TYPES_SQL} AND call_type != {AGENT_CALL_TYPE_SQL}
)::int AS session_llm_count,
COUNT(*) FILTER (WHERE call_type = {_AGENT_CALL_TYPE_SQL})::int AS session_agent_count,
COUNT(*) FILTER (WHERE call_type = {AGENT_CALL_TYPE_SQL})::int AS session_agent_count,
COALESCE(SUM(prompt_tokens), 0)::bigint AS session_total_prompt_tokens,
COALESCE(SUM(completion_tokens), 0)::bigint AS session_total_completion_tokens,
COALESCE(SUM(total_tokens), 0)::bigint AS session_total_tokens

View file

@ -6,6 +6,7 @@ from collections import Counter
from collections.abc import Mapping, MutableMapping, Sequence
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
Annotated,
Final,
Literal,
@ -16,7 +17,6 @@ from typing import (
from urllib.parse import urlparse
from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile
from prisma.errors import PrismaError
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError, create_model, field_validator
from pydantic.fields import FieldInfo, PydanticUndefined
from typing_extensions import NotRequired, ReadOnly, TypedDict
@ -63,6 +63,9 @@ from litellm.types.proxy.management_endpoints.ui_sso import (
SSOConfig,
)
if TYPE_CHECKING:
from prisma.errors import PrismaError
router: Final = APIRouter()
_DECODED_JSON: Final = TypeAdapter(object)
@ -1260,7 +1263,9 @@ def _parse_environment_variables_param(param_value: str | Mapping[str, object] |
async def _load_environment_variables_param(
repo: ConfigRepository,
) -> Mapping[str, object] | None | PrismaError | ValueError:
) -> "Mapping[str, object] | None | PrismaError | ValueError":
from prisma.errors import PrismaError
try:
entry: Final = await _config_param_db(repo).find_unique(where={"param_name": "environment_variables"})
param_value: Final = None if entry is None else _parse_environment_variables_param(entry.param_value)
@ -1314,6 +1319,8 @@ async def update_sso_settings(
detail={"error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature."},
)
from prisma.errors import PrismaError
# Read the existing SSO row first so the audit log captures a real
# before/after diff. Stored values are encrypted; decrypt them so the
# before-snapshot has the same shape as after_value, and rely on

View file

@ -55,6 +55,17 @@ def _sorted_pairs(pairs: Iterable[tuple[str, JsonValue]]) -> tuple[tuple[str, Js
return tuple(sorted(pairs, key=lambda pair: pair[0]))
_CACHE_MARKER_KEYS: Final = frozenset(("cache_control", "prompt_cache_breakpoint"))
def _is_cache_marked(block: Mapping[str, object]) -> bool:
match block:
case {"cache_control": {"type": "ephemeral"}}:
return True
case _:
return block.get("prompt_cache_breakpoint") is not None
def _canonical_bytes(value: object) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
@ -66,7 +77,7 @@ def _block_unit(
return _canonical_bytes((envelope, block)), message_run_type
block_type: Final = block.get("type")
block_run_type: Final = block_type if isinstance(block_type, str) and block_type in _TOOL_RUN_BLOCK_TYPES else None
stripped: Final = _sorted_pairs(item for item in block.items() if item[0] != "cache_control")
stripped: Final = _sorted_pairs(item for item in block.items() if item[0] not in _CACHE_MARKER_KEYS)
return _canonical_bytes((envelope, stripped)), message_run_type or block_run_type
@ -264,15 +275,9 @@ class PromptCachingCache:
continue
for content_idx, content_block in enumerate(content):
if isinstance(content_block, dict):
cache_control = content_block.get("cache_control")
if (
cache_control is not None
and isinstance(cache_control, dict)
and cache_control.get("type") == "ephemeral"
):
last_cacheable_message_idx = msg_idx
last_cacheable_content_idx = content_idx
if isinstance(content_block, dict) and _is_cache_marked(content_block):
last_cacheable_message_idx = msg_idx
last_cacheable_content_idx = content_idx
# If no cacheable block found, return empty list (no cacheable prefix)
if last_cacheable_message_idx is None:

View file

@ -9076,6 +9076,9 @@ def bundled_provider_models(provider: str) -> tuple[str, ...]:
return tuple(model for model in bundled if not is_hidden_as_retired(cost_map.get(model, {})))
AZURE_MODEL_PLACEHOLDER: Final = "Azure-LLM"
def get_valid_models(
check_provider_endpoint: bool | None = None,
custom_llm_provider: str | None = None,
@ -9131,7 +9134,7 @@ def get_valid_models(
continue
if provider == "azure":
valid_models.append("Azure-LLM")
valid_models.append(AZURE_MODEL_PLACEHOLDER)
elif provider_config is not None and check_provider_endpoint and provider is not None:
valid_models.extend(
_get_valid_models_from_provider_api(

View file

@ -2135,12 +2135,14 @@
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json",
"token_count_model": {
"model": "anthropic.claude-opus-5",
"litellm_provider": "bedrock_mantle",
"region": "us-east-1"
}
},
"input_cost_per_token_batches": 2.5e-06,
"output_cost_per_token_batches": 1.25e-05
},
"anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -2221,12 +2223,14 @@
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/",
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json",
"token_count_model": {
"model": "global.anthropic.claude-opus-5",
"litellm_provider": "bedrock_mantle",
"region": "us-east-1"
}
},
"input_cost_per_token_batches": 2.5e-06,
"output_cost_per_token_batches": 1.25e-05
},
"global.anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -2307,12 +2311,14 @@
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/",
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json",
"token_count_model": {
"model": "anthropic.claude-opus-5",
"litellm_provider": "bedrock_mantle",
"region": "us-east-1"
}
},
"input_cost_per_token_batches": 2.75e-06,
"output_cost_per_token_batches": 1.375e-05
},
"us.anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -2393,12 +2399,14 @@
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/",
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/eu-west-1/index.json",
"token_count_model": {
"model": "anthropic.claude-opus-5",
"litellm_provider": "bedrock_mantle",
"region": "eu-west-1"
}
},
"input_cost_per_token_batches": 2.75e-06,
"output_cost_per_token_batches": 1.375e-05
},
"eu.anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -2474,12 +2482,14 @@
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/",
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-southeast-2/index.json",
"token_count_model": {
"model": "anthropic.claude-opus-5",
"litellm_provider": "bedrock_mantle",
"region": "ap-southeast-4"
}
},
"input_cost_per_token_batches": 2.75e-06,
"output_cost_per_token_batches": 1.375e-05
},
"au.anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -2560,7 +2570,9 @@
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/"
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-northeast-1/index.json",
"input_cost_per_token_batches": 2.75e-06,
"output_cost_per_token_batches": 1.375e-05
},
"jp.anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -3949,7 +3961,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true,
"supports_max_reasoning_effort": false,
"prompt_cache_min_tokens": 512
},
"azure_ai/claude-fable-5-1": {
@ -3983,10 +3995,40 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true,
"supports_max_reasoning_effort": false,
"prompt_cache_min_tokens": 512,
"deprecation_date": "2027-12-05"
},
"azure_ai/claude-mythos-5-1": {
"input_cost_per_token": 1e-05,
"output_cost_per_token": 5e-05,
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 2.5e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"mode": "chat",
"supports_adaptive_thinking": true,
"thinking_always_on": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_forced_tool_use": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": false,
"prompt_cache_min_tokens": 512,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models"
},
"azure_ai/claude-opus-5": {
"deprecation_date": "2027-07-08",
"supports_mid_conversation_system": true,
@ -22269,7 +22311,7 @@
},
"mode": "chat",
"output_cost_per_token": 3.74997e-06,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
"supported_modalities": [
"text",
"image",
@ -22287,7 +22329,9 @@
"input_cost_per_token_priority": 1.35002e-06,
"output_cost_per_token_priority": 6.75003e-06,
"cache_read_input_token_cost_priority": 1.3503e-07,
"cache_creation_input_token_cost_priority": 1.35002e-06
"cache_creation_input_token_cost_priority": 1.35002e-06,
"supports_audio_input": true,
"supports_video_input": true
},
"databricks/databricks-gemini-3-7-flash": {
"cache_creation_input_token_cost": 7.4998e-07,
@ -22301,7 +22345,7 @@
},
"mode": "chat",
"output_cost_per_token": 3.74997e-06,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
"supported_modalities": [
"text",
"image",
@ -22319,7 +22363,9 @@
"input_cost_per_token_priority": 1.35002e-06,
"output_cost_per_token_priority": 6.75003e-06,
"cache_read_input_token_cost_priority": 1.3503e-07,
"cache_creation_input_token_cost_priority": 1.35002e-06
"cache_creation_input_token_cost_priority": 1.35002e-06,
"supports_audio_input": true,
"supports_video_input": true
},
"databricks/databricks-gemini-3-6-flash": {
"cache_creation_input_token_cost": 7.4998e-07,
@ -22335,7 +22381,7 @@
"mode": "chat",
"output_cost_per_token": 3.74997e-06,
"output_dbu_cost_per_token": 5.3571e-05,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
"supported_modalities": [
"text",
"image",
@ -22354,7 +22400,9 @@
"input_cost_per_token_priority": 1.35002e-06,
"output_cost_per_token_priority": 6.75003e-06,
"cache_read_input_token_cost_priority": 1.3503e-07,
"cache_creation_input_token_cost_priority": 1.35002e-06
"cache_creation_input_token_cost_priority": 1.35002e-06,
"supports_audio_input": true,
"supports_video_input": true
},
"databricks/databricks-gemini-3-5-flash": {
"cache_creation_input_token_cost": 1.87502e-06,
@ -22370,7 +22418,7 @@
"mode": "chat",
"output_cost_per_token": 1.124998e-05,
"output_dbu_cost_per_token": 0.000160714,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
"supported_modalities": [
"text",
"image",
@ -22388,7 +22436,9 @@
"input_cost_per_token_priority": 3.37498e-06,
"output_cost_per_token_priority": 2.025002e-05,
"cache_read_input_token_cost_priority": 3.3747e-07,
"cache_creation_input_token_cost_priority": 3.37498e-06
"cache_creation_input_token_cost_priority": 3.37498e-06,
"supports_audio_input": true,
"supports_video_input": true
},
"databricks/databricks-gemini-3-5-flash-lite": {
"cache_creation_input_token_cost": 3.7499e-07,
@ -22404,7 +22454,7 @@
"mode": "chat",
"output_cost_per_token": 3.12501e-06,
"output_dbu_cost_per_token": 4.4643e-05,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
"supported_modalities": [
"text",
"image",
@ -22424,7 +22474,9 @@
"input_cost_per_token_priority": 6.7501e-07,
"output_cost_per_token_priority": 5.62499e-06,
"cache_read_input_token_cost_priority": 6.748e-08,
"cache_creation_input_token_cost_priority": 6.7501e-07
"cache_creation_input_token_cost_priority": 6.7501e-07,
"supports_audio_input": true,
"supports_video_input": true
},
"databricks/databricks-gemma-3-12b": {
"cache_creation_input_token_cost": 1.5001e-07,
@ -24167,7 +24219,7 @@
"max_output_tokens": 163840,
"input_cost_per_token": 5e-07,
"output_cost_per_token": 2.15e-06,
"cache_read_input_token_cost": 4e-07,
"cache_read_input_token_cost": 3.5e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
@ -24241,7 +24293,7 @@
"max_output_tokens": 163840,
"input_cost_per_token": 2.5e-07,
"output_cost_per_token": 9.5e-07,
"cache_read_input_token_cost": 2.16e-07,
"cache_read_input_token_cost": 1.3e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
@ -29639,6 +29691,7 @@
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-lite-image": {
"deprecation_date": "2027-06-28",
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_batches": 1.25e-08,
"cache_read_input_token_cost_flex": 1.25e-08,
@ -30638,6 +30691,7 @@
"google_maps_grounding_cost_per_query": 0.014
},
"vertex_ai/gemini-3.8-flash": {
"deprecation_date": "2027-09-02",
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_batches": 3.75e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
@ -30744,6 +30798,7 @@
"supports_native_streaming": true
},
"vertex_ai/gemini-3.8-live": {
"deprecation_date": "2027-09-25",
"gemini_audio_only_live": true,
"input_cost_per_audio_token": 3e-06,
"input_cost_per_image_token": 1e-06,
@ -31421,6 +31476,7 @@
"cache_read_input_token_cost_batches": 3e-08,
"cache_read_input_token_cost_flex": 3e-08,
"cache_read_input_token_cost_priority": 5.4e-08,
"deprecation_date": "2026-12-01",
"input_cost_per_audio_token": 1e-06,
"input_cost_per_audio_token_flex": 5e-07,
"input_cost_per_token": 3e-07,
@ -32004,6 +32060,7 @@
"output_cost_per_token_flex": 5e-06,
"supports_url_context": true,
"cache_read_input_token_cost_above_200k_tokens_batches": 2.5e-07,
"deprecation_date": "2026-12-01",
"input_cost_per_token_above_200k_tokens_batches": 1.25e-06,
"output_cost_per_token_above_200k_tokens_batches": 7.5e-06
},
@ -32107,7 +32164,7 @@
},
"gemini/gemini-3.5-flash-lite": {
"cache_read_input_token_cost": 3e-08,
"cache_read_input_token_cost_batches": 2e-08,
"cache_read_input_token_cost_batches": 1.5e-08,
"cache_read_input_token_cost_flex": 2e-08,
"cache_read_input_token_cost_priority": 5e-08,
"input_cost_per_token": 3e-07,
@ -32704,13 +32761,13 @@
"input_cost_per_audio_token": 1.5e-06,
"input_cost_per_token": 1.5e-06,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_input_tokens": 131072,
"max_output_tokens": 57920,
"mode": "chat",
"output_cost_per_reasoning_token": 9e-06,
"output_cost_per_token": 9e-06,
"output_cost_per_video_token": 1.75e-05,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-flash-preview",
"supported_endpoints": [
"/v1/chat/completions"
],
@ -32728,7 +32785,8 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_video_input": true,
"supports_vision": true
"supports_vision": true,
"deprecation_date": "2026-11-06"
},
"gemini-3.5-flash": {
"prompt_cache_min_tokens": 4096,
@ -32916,6 +32974,7 @@
"google_maps_grounding_cost_per_query": 0.014
},
"gemini-3.8-flash": {
"deprecation_date": "2027-09-02",
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_batches": 3.75e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
@ -44263,7 +44322,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/google%2Fgemma-3-27b-it"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 4e-08
},
"nebius/meta-llama/Llama-3.3-70B-Instruct": {
"max_input_tokens": 131072,
@ -44489,7 +44549,7 @@
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash"
},
"nebius/deepseek-ai/DeepSeek-V4-Flash-0731": {
"max_input_tokens": 1024000,
"max_input_tokens": 1048576,
"max_output_tokens": 1024000,
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 2.8e-07,
@ -44497,7 +44557,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash-0731"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1e-08
},
"nebius/deepseek-ai/DeepSeek-V4-Pro": {
"max_input_tokens": 1048576,
@ -44508,29 +44569,32 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1.4e-07
},
"nebius/deepseek-ai/DeepSeek-V4-Pro-0813": {
"input_cost_per_token": 1.32e-06,
"litellm_provider": "nebius",
"max_input_tokens": 979000,
"max_input_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 3.96e-06,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro-0813",
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"supports_function_calling": true,
"supports_reasoning": true
"supports_reasoning": true,
"cache_read_input_token_cost": 4e-08
},
"nebius/deepseek-ai/DeepSeek-V4.1-Flash": {
"input_cost_per_token": 3e-07,
"litellm_provider": "nebius",
"max_input_tokens": 1048000,
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://tokenfactory.nebius.com/endpoints?modals=endpoint-details&model-id=deepseek-ai/DeepSeek-V4.1-Flash",
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true
"supports_vision": true,
"cache_read_input_token_cost": 1e-08
},
"nebius/MiniMaxAI/MiniMax-M2.5": {
"max_input_tokens": 196608,
@ -44553,7 +44617,8 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M3"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 6e-08
},
"nebius/moonshotai/Kimi-K2.6": {
"max_input_tokens": 262144,
@ -44565,7 +44630,8 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K2.6"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1.6e-07
},
"nebius/moonshotai/Kimi-K2.7-Code": {
"max_input_tokens": 262144,
@ -44577,10 +44643,11 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/moonshotai%2FKimi-K2.7-Code"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1.5e-07
},
"nebius/moonshotai/Kimi-K3": {
"max_input_tokens": 1024000,
"max_input_tokens": 1048576,
"max_output_tokens": 1024000,
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
@ -44589,7 +44656,8 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K3"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 3e-07
},
"nebius/NousResearch/Hermes-4-405B": {
"max_input_tokens": 131072,
@ -44600,7 +44668,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-405B"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 2e-07
},
"nebius/NousResearch/Hermes-4-70B": {
"max_input_tokens": 131072,
@ -44644,7 +44713,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNVIDIA-Nemotron-3-Nano-30B-A3B"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1.2e-08
},
"nebius/nvidia/Nemotron-3-Nano-Omni": {
"max_input_tokens": 262144,
@ -44667,7 +44737,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2Fnemotron-3-super-120b-a12b"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 6e-08
},
"nebius/nvidia/Nemotron-3-Ultra-550b-a55b": {
"max_input_tokens": 1048576,
@ -44678,7 +44749,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Ultra-550b-a55b"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1.2e-07
},
"nebius/nvidia/Nemotron-3_5-Lightning": {
"max_input_tokens": 1048576,
@ -44689,7 +44761,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3_5-Lightning"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 3e-08
},
"nebius/openai/gpt-oss-120b": {
"max_input_tokens": 131072,
@ -44700,7 +44773,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/openai%2Fgpt-oss-120b"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1e-07
},
"nebius/openbmb/MiniCPM-V-4_5": {
"max_input_tokens": 32000,
@ -44710,7 +44784,8 @@
"litellm_provider": "nebius",
"mode": "chat",
"supports_vision": true,
"source": "https://tokenfactory.nebius.com/models/catalog/image2text/openbmb%2FMiniCPM-V-4_5"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1.3e-07
},
"nebius/Qwen/Qwen3-235B-A22B-Instruct-2507": {
"max_input_tokens": 262144,
@ -44720,7 +44795,8 @@
"litellm_provider": "nebius",
"mode": "chat",
"supports_function_calling": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-235B-A22B-Instruct-2507"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 5e-08
},
"nebius/Qwen/Qwen3-30B-A3B-Instruct-2507": {
"max_input_tokens": 262144,
@ -44730,7 +44806,8 @@
"litellm_provider": "nebius",
"mode": "chat",
"supports_function_calling": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-30B-A3B-Instruct-2507"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1e-07
},
"nebius/Qwen/Qwen3-Next-80B-A3B-Thinking": {
"max_input_tokens": 128000,
@ -44752,7 +44829,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.5-397B-A17B"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1.2e-07
},
"nebius/Qwen/Qwen3.8-27B": {
"input_cost_per_token": 4.5e-07,
@ -44760,9 +44838,10 @@
"max_input_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3e-06,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.8-27B",
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"supports_function_calling": true,
"supports_reasoning": true
"supports_reasoning": true,
"cache_read_input_token_cost": 1e-07
},
"nebius/zai-org/GLM-5.1": {
"max_input_tokens": 202752,
@ -44773,7 +44852,8 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.1"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 2.6e-07
},
"nebius/zai-org/GLM-5.2": {
"max_input_tokens": 1048576,
@ -44784,21 +44864,23 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.2"
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"cache_read_input_token_cost": 1.4e-07
},
"nebius/zai-org/GLM-5.3": {
"input_cost_per_token": 1.4e-06,
"litellm_provider": "nebius",
"max_input_tokens": 1024000,
"max_input_tokens": 1048576,
"max_output_tokens": 1024000,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3",
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"supports_function_calling": true,
"supports_reasoning": true
"supports_reasoning": true,
"cache_read_input_token_cost": 2.6e-07
},
"nebius/zai-org/GLM-5.3-Flash": {
"max_input_tokens": 1024000,
"max_input_tokens": 1048576,
"max_output_tokens": 1024000,
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 5e-07,
@ -44806,8 +44888,9 @@
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3-Flash",
"supports_vision": true
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"supports_vision": true,
"cache_read_input_token_cost": 3e-08
},
"nebius/BAAI/bge-en-icl": {
"max_input_tokens": 32768,
@ -49811,7 +49894,7 @@
"mode": "chat",
"output_cost_per_token": 3e-05,
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/",
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json",
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -49833,7 +49916,9 @@
"model": "anthropic.claude-opus-5",
"litellm_provider": "bedrock_mantle",
"region": "us-gov-west-1"
}
},
"input_cost_per_token_batches": 3e-06,
"output_cost_per_token_batches": 1.5e-05
},
"us-gov.anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -52920,6 +53005,7 @@
},
"vertex_ai/gemini-3.1-flash-lite-image": {
"deprecation_date": "2027-06-28",
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_batches": 1.25e-08,
"cache_read_input_token_cost_flex": 1.25e-08,
@ -54148,7 +54234,7 @@
"voyage/voyage-lite-01": {
"input_cost_per_token": 1e-07,
"litellm_provider": "voyage",
"max_input_tokens": 4096,
"max_input_tokens": 4000,
"mode": "embedding",
"output_cost_per_token": 0.0
},
@ -64807,7 +64893,7 @@
"supports_vision": false
},
"novita/deepseek/deepseek-v4-pro-0813": {
"cache_read_input_token_cost": 1.3200000000000002e-07,
"cache_read_input_token_cost": 4.4e-08,
"input_cost_per_token": 1.32e-06,
"litellm_provider": "novita",
"max_input_tokens": 1048576,
@ -64919,13 +65005,13 @@
"supports_vision": false
},
"novita/mindai/macaron-v1-venti": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 1.5e-06,
"cache_read_input_token_cost": 1.65e-07,
"input_cost_per_token": 8.25e-07,
"litellm_provider": "novita",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.5e-06,
"output_cost_per_token": 2.475e-06,
"source": "https://novita.ai/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -65028,13 +65114,13 @@
"supports_vision": false
},
"novita/mindai/macaron-v1-tall": {
"cache_read_input_token_cost": 8e-08,
"input_cost_per_token": 4.5000000000000003e-07,
"cache_read_input_token_cost": 4.4e-08,
"input_cost_per_token": 2.475e-07,
"litellm_provider": "novita",
"max_input_tokens": 262144,
"max_output_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 2.6e-06,
"output_cost_per_token": 1.43e-06,
"source": "https://novita.ai/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -66459,9 +66545,9 @@
"deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": {
"max_output_tokens": 1048576,
"max_input_tokens": 1048576,
"input_cost_per_token": 8e-08,
"input_cost_per_token": 6e-08,
"output_cost_per_token": 1.8e-07,
"cache_read_input_token_cost": 1.6e-08,
"cache_read_input_token_cost": 1.5e-08,
"supports_prompt_caching": true,
"litellm_provider": "deepinfra",
"mode": "chat",
@ -66840,8 +66926,8 @@
"deepinfra/google/gemma-4-31B-it": {
"max_output_tokens": 262144,
"max_input_tokens": 262144,
"input_cost_per_token": 1.3e-07,
"output_cost_per_token": 3.8e-07,
"input_cost_per_token": 2e-07,
"output_cost_per_token": 4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_function_calling": true,
@ -68552,7 +68638,10 @@
"model": "anthropic.claude-opus-5",
"litellm_provider": "bedrock_mantle",
"region": "us-gov-west-1"
}
},
"input_cost_per_token_batches": 3e-06,
"output_cost_per_token_batches": 1.5e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json"
},
"bedrock/us-gov-west-1/anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -68827,7 +68916,10 @@
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
"supports_xhigh_reasoning_effort": true,
"input_cost_per_token_batches": 3e-06,
"output_cost_per_token_batches": 1.5e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json"
},
"bedrock/us-gov-east-1/anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -69850,6 +69942,14 @@
"cache_read_input_token_cost": 3e-08,
"source": "https://docs.perplexity.ai/docs/agent-api/models"
},
"perplexity/google/gemini-4-argon": {
"litellm_provider": "perplexity",
"mode": "responses",
"input_cost_per_token": 2e-06,
"output_cost_per_token": 1e-05,
"cache_read_input_token_cost": 1e-07,
"source": "https://docs.perplexity.ai/docs/agent-api/models"
},
"perplexity/google/gemini-3.6-flash": {
"litellm_provider": "perplexity",
"mode": "responses",
@ -69977,6 +70077,14 @@
"cache_read_input_token_cost": 1.15e-09,
"source": "https://docs.perplexity.ai/docs/agent-api/models"
},
"perplexity/perplexity/nemotron-3.5-super-120b-a12b": {
"litellm_provider": "perplexity",
"mode": "responses",
"input_cost_per_token": 2e-07,
"output_cost_per_token": 5e-07,
"cache_read_input_token_cost": 4e-08,
"source": "https://docs.perplexity.ai/docs/agent-api/models"
},
"perplexity/perplexity/nemotron-3-ultra-550b-a55b": {
"litellm_provider": "perplexity",
"mode": "responses",
@ -70102,12 +70210,12 @@
"input_cost_per_token": 1.5e-06,
"litellm_provider": "vertex_ai",
"max_output_tokens": 57920,
"max_input_tokens": 57920,
"max_input_tokens": 131072,
"mode": "chat",
"output_cost_per_reasoning_token": 9e-06,
"output_cost_per_token": 9e-06,
"output_cost_per_video_token": 1.75e-05,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-1-1-flash",
"supported_endpoints": [
"/v1beta/interactions"
],
@ -74503,12 +74611,14 @@
"input_cost_per_token": 5.5e-06,
"cache_read_input_token_cost": 5.5e-07,
"output_cost_per_token": 2.75e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-northeast-1/index.json",
"token_count_model": {
"model": "anthropic.claude-opus-5",
"litellm_provider": "bedrock_mantle",
"region": "ap-southeast-4"
}
},
"input_cost_per_token_batches": 2.75e-06,
"output_cost_per_token_batches": 1.375e-05
},
"apac.anthropic.claude-opus-5-5": {
"bedrock_converse_supports_strict_tools": false,
@ -75105,7 +75215,7 @@
"supports_function_calling": true,
"input_cost_per_token": 2e-06,
"output_cost_per_token": 1e-05,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost": 1e-07,
"source": "https://docs.perplexity.ai/docs/agent-api/models"
},
"perplexity/anthropic/claude-fable-5-1": {
@ -75153,6 +75263,9 @@
"output_cost_per_token": 5e-05,
"cache_read_input_token_cost": 1e-06,
"cache_creation_input_token_cost": 1.25e-05,
"input_cost_per_token_above_272k_tokens": 2e-05,
"output_cost_per_token_above_272k_tokens": 7.5e-05,
"cache_read_input_token_cost_above_272k_tokens": 2e-06,
"source": "https://docs.perplexity.ai/docs/agent-api/models"
},
"perplexity/openai/gpt-6-luna": {
@ -76620,6 +76733,9 @@
"input_cost_per_token": 1e-07,
"output_cost_per_token": 5e-07,
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
"source": "https://docs.perplexity.ai/docs/agent-api/models"
},
"us-gov.anthropic.claude-haiku-5-5": {
@ -76954,23 +77070,19 @@
},
"vertex_ai/gemini-robotics-er-2-preview": {
"cache_read_input_token_cost": 1e-07,
"cache_read_input_token_cost_batches": 5e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 1e-06,
"input_cost_per_token_batches": 5e-07,
"litellm_provider": "vertex_ai",
"max_input_tokens": 131072,
"max_output_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 5e-06,
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
"/v1/completions"
],
"supported_modalities": [
"text",
@ -77000,23 +77112,19 @@
},
"gemini-robotics-er-2-preview": {
"cache_read_input_token_cost": 1e-07,
"cache_read_input_token_cost_batches": 5e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 1e-06,
"input_cost_per_token_batches": 5e-07,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 131072,
"max_output_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 5e-06,
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
"/v1/completions"
],
"supported_modalities": [
"text",
@ -77102,6 +77210,7 @@
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-4-argon": {
"deprecation_date": "2027-10-13",
"cache_read_input_token_cost": 1e-07,
"cache_read_input_token_cost_batches": 5e-08,
"cache_read_input_token_cost_flex": 5e-08,
@ -77510,9 +77619,10 @@
"max_output_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 1e-06,
"source": "https://tokenfactory.nebius.com/models/catalog/text2text/reflection%2Fbeam",
"source": "https://tokenfactory.nebius.com/api/public/models_info",
"supports_function_calling": true,
"supports_reasoning": true
"supports_reasoning": true,
"cache_read_input_token_cost": 5e-08
},
"baseten/deepseek-ai/DeepSeek-V4.1-Flash-1017": {
"cache_read_input_token_cost": 7e-09,
@ -81783,5 +81893,55 @@
"supports_response_schema": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
},
"nebius/deepseek-ai/DeepSeek-V4.1-Flash-1017": {
"max_input_tokens": 1048576,
"input_cost_per_token": 3e-07,
"cache_read_input_token_cost": 6e-09,
"output_cost_per_token": 1.2e-06,
"litellm_provider": "nebius",
"mode": "chat",
"supports_vision": true,
"supports_reasoning": true,
"supports_function_calling": true,
"source": "https://tokenfactory.nebius.com/api/public/models_info"
},
"nebius/meta-models/muse-spark-open": {
"max_input_tokens": 1048576,
"input_cost_per_token": 1.25e-06,
"cache_read_input_token_cost": 1.5e-07,
"output_cost_per_token": 4.25e-06,
"litellm_provider": "nebius",
"mode": "chat",
"supports_vision": true,
"supports_video_input": true,
"supports_reasoning": true,
"supports_function_calling": true,
"source": "https://tokenfactory.nebius.com/api/public/models_info"
},
"nebius/MiniMaxAI/MiniMax-M3.1-Flash": {
"max_input_tokens": 1048576,
"input_cost_per_token": 1.5e-07,
"cache_read_input_token_cost": 1.5e-08,
"output_cost_per_token": 6e-07,
"litellm_provider": "nebius",
"mode": "chat",
"supports_vision": true,
"supports_reasoning": true,
"supports_function_calling": true,
"source": "https://tokenfactory.nebius.com/api/public/models_info"
},
"nebius/nvidia/Nemotron-3.5-Super-VL": {
"max_input_tokens": 1048576,
"input_cost_per_token": 3e-07,
"cache_read_input_token_cost": 6e-08,
"output_cost_per_token": 9e-07,
"litellm_provider": "nebius",
"mode": "chat",
"supports_vision": true,
"supports_video_input": true,
"supports_reasoning": true,
"supports_function_calling": true,
"source": "https://tokenfactory.nebius.com/api/public/models_info"
}
}

View file

@ -13,6 +13,7 @@ IMPORTED_MODULES: Final[tuple[str, ...]] = (
"litellm.proxy.mcp_server.server",
"litellm.proxy.mcp_server.mcp_server_manager",
"litellm.proxy.mcp_server.rest_endpoints",
"litellm.proxy.proxy_server",
)

View file

@ -377,7 +377,11 @@ def test_stream_completion_controls_publication(store: RedisResponseStore, provi
assert len(provider.hits) == (2 if truncated else 1)
def test_store_outage_preserves_provider_success(provider: Provider) -> None:
@pytest.mark.parametrize("cluster_mode", ("0", "1"))
def test_store_outage_preserves_provider_success(
provider: Provider, monkeypatch: pytest.MonkeyPatch, cluster_mode: str,
) -> None:
monkeypatch.setenv("E2E_PROVIDER_CACHE_REDIS_CLUSTER", cluster_mode)
with socket.socket() as probe:
probe.bind(("127.0.0.1", 0))
port: Final = probe.getsockname()[1]
@ -401,6 +405,20 @@ def test_old_lease_cannot_overwrite_new_owner(store: RedisResponseStore) -> None
assert isinstance(hit, CacheHit) and hit.payload == b"new"
def test_separate_namespaces_keep_their_own_response(store: RedisResponseStore) -> None:
other: Final = replace(store, namespace=store.namespace + "-other")
first_lease: Final = store.lookup("shared-key")
assert isinstance(first_lease, CaptureLease)
assert store.publish("shared-key", first_lease, b"first namespace")
other_lease: Final = other.lookup("shared-key")
assert isinstance(other_lease, CaptureLease)
assert other.publish("shared-key", other_lease, b"other namespace")
first_hit: Final = store.lookup("shared-key")
other_hit: Final = other.lookup("shared-key")
assert isinstance(first_hit, CacheHit) and first_hit.payload == b"first namespace"
assert isinstance(other_hit, CacheHit) and other_hit.payload == b"other namespace"
def test_identity_preserves_values_and_never_contains_credentials() -> None:
variants: Final = (b'{}', b'{"a":null}', b'{"a":false}', b'{"a":0}', b'{"a":0.0}', b'{"a":"0"}', b' { }', None, b'')
url: Final = "https://example.invalid/v1/chat/completions"

View file

@ -32,10 +32,10 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
- `migrations/` - isolated Docker startup, concurrent migration, crash recovery, and legacy database compatibility. The CircleCI migration workflow enables `LITELLM_MIGRATION_TESTS=1`. These tests own their proxy containers and databases, so they do not use the shared proxy preflight or shared database cleanup
- `llm_translation/` - LLM endpoint and provider-translation behavior: passthrough, custom pricing, OCR, and the non-chat inference endpoints (`/v1/responses`, `/v1/messages`, `/embeddings`, `/v1/rerank`, `/v1/audio/speech`, `/v1/images/generations`), each against a deployment the test creates via `/model/new` and deletes on teardown
- `llm_translation/` - Azure realtime translation is explicitly skipped pending investigation of intermittent missing transcripts and remains unqualified. OpenAI translation and other Azure realtime cases remain enabled. LLM endpoint and provider-translation behavior: passthrough, custom pricing, OCR, and the non-chat inference endpoints (`/v1/responses`, `/v1/messages`, `/embeddings`, `/v1/rerank`, `/v1/audio/speech`, `/v1/images/generations`), each against a deployment the test creates via `/model/new` and deletes on teardown
- `access_control/` - the gateway's authorization and error-shape contract: per-key model allow-lists, route-group permissions (`allowed_routes`), and unknown-model validation
- `embeddings/` - the `/embeddings` endpoint across providers
- `batches/` - the `/batches` endpoint (placeholder until the first test lands)
- `batches/` - the `/batches` endpoint. GovCloud batch coverage is explicitly skipped until its infrastructure is provisioned. Other missing credentials and provider errors remain failures
- `realtime/` - realtime websocket sessions, including the pipecat audio path
- `quota_management/` - quota enforcement and accounting, one subfolder per behavior: `ratelimit/` (rpm/tpm blocks, window reset, pacing headers on live traffic), `budgets/` (budget definition, enforcement, and reset windows: key, team, tag, soft, multi-window), and `spend_tracking/` (spend logging and cost attribution on `/spend/*`)
- `management/` - key/team/user/organization management routes: create/update/delete persistence via the info routes, team membership, and llm-only-key route denials (API surface, not Playwright)
@ -50,7 +50,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
- `gateway/` - proxy configuration only (`litellm-config.yml`). No tests
- `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke. `pr_gate_version_resolver.py` only forwards to `cli_harness/npm_version_resolver.py`, because the Buildkite runner images still run that path and install @latest without it
- `cli_harness/` - no tests, only the plumbing every suite that drives a real coding-agent CLI shares: `npm_version_resolver.py` picks the CLI release to install (the newest npm release at least three days old, run by CI as `python tests/e2e/cli_harness/npm_version_resolver.py <npm package>`, stdlib only), `lockdown.py` gives the CLI subprocess an allowlisted env and a fresh empty HOME so it never sees the runner's credentials or dotfiles, and `rate_limiter.py` is the cross-process token bucket per upstream that keeps parallel CLI runs under one request rate. Covered by `tests/e2e_harness/cli_harness/`. Code only one CLI suite needs stays in that suite
- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, and auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(<setup id>, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/`
- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, and auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(<setup id>, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. The Bedrock Runtime setup is explicitly skipped until stage IAM permits `bedrock:InvokeModel` on its default project. The other setups still run, and skipped Runtime cases do not qualify that route. Covered by `tests/e2e_harness/codex/`
- `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`. Its specs follow the dashboard's rule against testing what shadcn and Base UI already guarantee, in `ui/litellm-dashboard/AGENTS.md`
## MCP suite: real Datadog only
@ -329,3 +329,7 @@ other.<area>.<case>.<assertion>
- the `codex/` suite is the one exception: a single setup that a LiteLLM or provider bug breaks may carry `pytest.mark.xfail(strict=True, reason=...)` with a plain one-sentence reason on that setup's `pytest.param`, so it keeps running and goes red once the bug is fixed. Strict only, and never on a flaky setup (one that passes some runs and fails others)
- a cell addresses the stack through its front door (`PROXY_BASE_URL`) the way a customer does, never a gateway pod by address, and proves a cross-replica property with N independent calls through that door, naming the miss odds (2^-N at two pods) in its docstring. `PROXY_REPLICA_URLS` is for read-backs that are per pod by nature (polling until a management write has converged on every replica, `/metrics`, RSS), never for steering the scenario a cell asserts on at a chosen pod
## Provider response cache
The provider response cache can share the stage Valkey instance through a `rediss://` URL. Set `E2E_PROVIDER_CACHE_REDIS_CLUSTER=1` for a cluster-aware client, including ElastiCache Serverless. Keep the dedicated HMAC key and namespace, and restrict test cleanup to keys in the test's own namespace

View file

@ -11,8 +11,9 @@ cost write-back via a cross-run marker baton (design below).
Only supported cells are tested. The capability table in `capabilities.py` holds one
row per supported (provider, scenario) pair, so there are no skipped cells in the
parametrized run. The batches suite never skips: missing provider creds or upstream
failures are hard test failures (see `tests/e2e/AGENTS.md`).
parametrized run. Missing provider creds or upstream failures are hard test failures
(see `tests/e2e/AGENTS.md`). GovCloud is explicitly skipped until its infrastructure
is provisioned and does not count as qualified coverage.
| Provider | create | retrieve | cancel | list | content download | file backing |
|-----------|--------|----------|--------|------|------------------|--------------|

View file

@ -1210,6 +1210,7 @@ def _govcloud_params() -> LiteLLMParamsBody:
)
@pytest.mark.skip(reason="GovCloud E2E infrastructure is not provisioned")
class TestBedrockBatchGovCloud:
"""Bedrock batch lifecycle in the AWS GovCloud partition (us-gov-west-1).

View file

@ -13,7 +13,7 @@ from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from models import LiteLLMParamsBody
GPT_ON_BEDROCK_MANTLE: Final = "bedrock_mantle/openai.gpt-6.1-sol"
GPT_ON_BEDROCK_RUNTIME: Final = "bedrock/us.openai.gpt-6.1-sol"
GPT_ON_BEDROCK_RUNTIME: Final = "bedrock/global.openai.gpt-6.1-sol"
GPT_ON_AZURE_OPENAI: Final = "azure/gpt-6.1-sol"
GPT_ON_OPENAI: Final = "openai/gpt-6.1-sol"
CLAUDE_ON_BEDROCK: Final = "bedrock/us.anthropic.claude-sonnet-5-5"
@ -60,7 +60,10 @@ SETUPS: Final = (
RateBucket.BEDROCK_CONVERSE,
),
id="bedrock_runtime",
marks=_drives(Provider.BEDROCK, GPT_ON_BEDROCK_RUNTIME),
marks=(
_drives(Provider.BEDROCK, GPT_ON_BEDROCK_RUNTIME),
pytest.mark.skip(reason="Stage IAM lacks bedrock:InvokeModel on the default Bedrock Runtime project"),
),
),
pytest.param(
Deployment(

View file

@ -38,7 +38,7 @@ from models import (
from proxy_client import ProxyClient
from pydantic import BaseModel, Field
GUARDRAIL_BACKEND: Final = "gemini/gemini-2.5-flash"
GUARDRAIL_BACKEND: Final = "gemini/gemini-3.8-flash"
GuardrailMode = Literal["pre_call", "post_call", "during_call", "logging_only"]
BlockedWordAction = Literal["BLOCK", "MASK"]

View file

@ -128,7 +128,7 @@ class TestOpenAIModerationGuardrail:
)
allowed = unwrap(
client.messages(scoped_key, model, BENIGN_PROMPT, guardrails=[name], max_tokens=64)
client.messages(scoped_key, model, BENIGN_PROMPT, guardrails=[name], max_tokens=512)
)
assert allowed.content or allowed.choices, (
"the same moderation guardrail must let a benign /v1/messages prompt through, but "

View file

@ -57,8 +57,9 @@ Bedrock and xai (`xai/grok-4-1-fast-non-reasoning`) are supported by the proxy b
kept commented out in `PROVIDERS` until they pass end-to-end here; re-enable them by
uncommenting their entry.
Every provider is provisioned and asserted; the suite never skips a provider. Per
`tests/e2e/AGENTS.md` there is no sanctioned skip: the whole-suite proxy-liveness
Azure realtime translation is explicitly skipped pending investigation of intermittent
missing transcripts and does not count as qualified coverage. OpenAI translation and
other Azure realtime cases remain enabled. Per `tests/e2e/AGENTS.md`, the whole-suite proxy-liveness
probe hard-fails when no proxy answers, and a provider whose credentials or upstream
realtime model are missing on the gateway is likewise a hard failure, not a skip.
Give the gateway each provider's credentials to turn its tests green.

View file

@ -360,7 +360,18 @@ LIVE_TRANSCRIPTION_PROVIDERS: Final = (
)
TRANSLATION_PARAMS: Final = tuple(
pytest.param(p, id=p.id, marks=audio_marks(p, "translation")) for p in TRANSLATION_PROVIDERS
pytest.param(
p,
id=p.id,
marks=(
*audio_marks(p, "translation"),
pytest.mark.skipif(
p.id == "azure",
reason="Azure realtime translation intermittently returns no transcript; qualification deferred",
),
),
)
for p in TRANSLATION_PROVIDERS
)
LIVE_TRANSCRIPTION_PARAMS: Final = tuple(
pytest.param(p, id=p.id, marks=audio_marks(p, "transcription")) for p in LIVE_TRANSCRIPTION_PROVIDERS

View file

@ -45,7 +45,7 @@ from passthrough_client import PassthroughClient
pytestmark = pytest.mark.e2e
COHERE_BACKEND = "cohere/command-r-08-2024"
GEMINI_BACKEND = "gemini/gemini-2.5-flash"
GEMINI_BACKEND = "gemini/gemini-3.8-flash"
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano"
AZURE_OPENAI_API_VERSION: Final = "v1"
@ -243,7 +243,7 @@ _PERSON_SCHEMA: dict[str, object] = {
OPENAI_CHAT_MODEL: Final = "gpt-5.5"
ANTHROPIC_CHAT_MODEL: Final = "claude-haiku-4-5"
GEMINI_FLASH_MODEL: Final = "gemini-2.5-flash"
GEMINI_FLASH_MODEL: Final = "gemini-3.8-flash"
CHAT_MODELS: Final = (
pytest.param(

View file

@ -5,7 +5,7 @@ teardown) instead of relying on a statically configured model, so the check is
self-contained and never inherits pricing another suite or a stale config left on
the shared proxy. custom-priced-flash sets input/output rates deliberately far
above the canonical gemini price; the isolation sibling shares the same
gemini/gemini-2.5-flash backend but sets no override. Three behaviors are checked
gemini/gemini-3.8-flash backend but sets no override. Three behaviors are checked
independently:
- billing: a real call's logged cost breakdown charges input and output tokens at
@ -37,10 +37,8 @@ from models import (
pytestmark = pytest.mark.e2e
BACKEND_MODEL = "gemini/gemini-2.5-flash"
BACKEND_MODEL = "gemini/gemini-3.8-flash"
GEMINI_API_KEY = "os.environ/GEMINI_API_KEY"
# Deliberately ~100x above canonical gemini-2.5-flash (input 3e-7 / output 2.5e-6)
# so an override that is ignored or under-applied bills at the base rate and fails.
CUSTOM_INPUT_RATE = 5e-05
CUSTOM_OUTPUT_RATE = 1e-04
@ -78,7 +76,7 @@ def _provision(
input_cost_per_token: float | None,
output_cost_per_token: float | None,
) -> str:
"""Register a fresh gemini/gemini-2.5-flash deployment (deleted on teardown) and
"""Register a fresh gemini/gemini-3.8-flash deployment (deleted on teardown) and
return its model name. With the cost fields set the deployment carries a custom
pricing override; with them None it is a plain sibling on the same backend. The
marker keeps the name unique so concurrent runs on the shared proxy never
@ -251,7 +249,7 @@ class TestCustomPricing:
assert sibling_entry is not None, f"{sibling} absent from /model/info"
# custom-priced-flash overrides pricing; the sibling shares the same
# gemini/gemini-2.5-flash backend but sets no override, so it must keep its
# gemini/gemini-3.8-flash backend but sets no override, so it must keep its
# own price. Equal rates mean the override leaked into the shared cost map.
assert (
sibling_entry.model_info.input_cost_per_token

View file

@ -21,7 +21,7 @@ from pydantic import BaseModel
pytestmark = pytest.mark.e2e
UPSTREAM_MODEL = "gemini/gemini-2.5-flash"
UPSTREAM_MODEL = "gemini/gemini-3.8-flash"
class _GenerateContentPart(BaseModel):

View file

@ -24,7 +24,7 @@ from sdk_clients import SdkClients
pytestmark = pytest.mark.e2e
IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-1"
IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-2.5-sunburst"
_TEST_PNG = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PMQ0AAAwDoPo3"

View file

@ -41,7 +41,7 @@ from proxy_client import ProxyClient
from pydantic import BaseModel
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
pytestmark = [pytest.mark.e2e, pytest.mark.provider_live]
FLAGGED_INVOKE_MODEL = "bedrock/invoke/us.anthropic.claude-sonnet-5"
UNFLAGGED_INVOKE_MODEL = "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0"

View file

@ -5,7 +5,7 @@ and verifies the proxy still logged a costed SpendLogs row
(call_type="pass_through_endpoint"), correlated by the id the caller was served:
the x-litellm-call-id header on gemini, the `msg_...` message id on anthropic.
Covered: gemini ("gemini-2.5-flash") + anthropic ("claude-haiku-4-5"), streaming +
Covered: gemini ("gemini-3.8-flash") + anthropic ("claude-haiku-4-5"), streaming +
non-streaming, plus native tool calls. See LLM_TRANSLATION_COVERAGE_MATRIX.md.
A passthrough call returning non-2xx fails hard (never a skip); once it returns
@ -32,7 +32,7 @@ from passthrough_client import (
completed_responses_object,
)
GEMINI_MODEL: Final = "gemini-2.5-flash"
GEMINI_MODEL: Final = "gemini-3.8-flash"
ANTHROPIC_PASSTHROUGH_MODEL: Final = "claude-haiku-4-5"
EMBEDDING_MODEL = "text-embedding-3-small"
REALTIME_MODEL = "gpt-realtime-2"
@ -291,7 +291,7 @@ class TestPassthroughModelAllowlist:
result = client.anthropic_message(key, ANTHROPIC_PASSTHROUGH_MODEL, f"say hi {unique_marker()}")
assert result.status_code == 403, (
"a key restricted to gemini-2.5-flash must be denied a claude passthrough call, "
"a key restricted to gemini-3.8-flash must be denied a claude passthrough call, "
f"got {result.status_code}: {result.body[:300]}"
)

View file

@ -57,7 +57,7 @@ OPENAI_VISION_BACKEND: Final = "openai/gpt-4o"
ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5"
BEDROCK_CONVERSE_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
GEMINI_BACKEND: Final = "gemini/gemini-2.5-flash"
GEMINI_BACKEND: Final = "gemini/gemini-3.8-flash"
OPENAI_RESPONSES_BACKEND: Final = "openai/gpt-5.5"
INSTRUCTIONS = "You are a helpful assistant"
CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg"

View file

@ -18,10 +18,11 @@ import json
import os
import time
from dataclasses import dataclass
from typing import Callable, Literal
from datetime import datetime, timedelta, timezone
from typing import Callable, Final, Literal
import pytest
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
from pydantic import AliasChoices, BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError, field_validator
from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, settle_propagation
from e2e_metadata import step
@ -124,23 +125,39 @@ class CreateGuardrailResponse(BaseModel):
guardrail_name: str | None = None
_LANGFUSE_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
class LangfuseObservation(BaseModel):
model_config = ConfigDict(extra="ignore", populate_by_name=True)
model_config = ConfigDict(extra="ignore", populate_by_name=True, frozen=True)
id: str
trace_id: str | None = Field(default=None, alias="traceId")
name: str | None = None
type: str | None = None
calculated_total_cost: float | None = Field(default=None, alias="calculatedTotalCost")
calculated_total_cost: float | None = Field(
default=None, validation_alias=AliasChoices("totalCost", "calculatedTotalCost")
)
level: str | None = None
input: object | None = None
output: object | None = None
metadata: object | None = None
input: JsonValue = None
output: JsonValue = None
metadata: JsonValue = None
usage: object | None = None
usage_details: object | None = Field(default=None, alias="usageDetails")
model: str | None = None
@field_validator("input", "output", "metadata", mode="before")
@classmethod
def decode_json_fields(cls, value: object) -> JsonValue:
if not isinstance(value, str):
return _LANGFUSE_JSON.validate_python(value)
try:
return _LANGFUSE_JSON.validate_json(value)
except ValidationError:
return value
class LangfuseObservationList(BaseModel):
model_config = ConfigDict(extra="ignore")
@ -171,12 +188,15 @@ def is_otel_v2_generation(obs: LangfuseObservation, *, key_alias: str) -> bool:
class LangfuseListParams(BaseModel):
model_config = ConfigDict(populate_by_name=True)
model_config = ConfigDict(populate_by_name=True, frozen=True)
limit: int = 100
trace_id: str | None = Field(default=None, alias="traceId")
name: str | None = None
from_start_time: str | None = Field(default=None, alias="fromStartTime")
from_start_time: str = Field(alias="fromStartTime")
to_start_time: str = Field(alias="toStartTime")
fields: str = "core,basic,io,metadata,model,usage"
expand_metadata: str = Field(default="attributes", alias="expandMetadata")
@dataclass(frozen=True, slots=True)
@ -580,14 +600,16 @@ class LoggingClient:
name: str | None = None,
from_start_time: str | None = None,
) -> list[LangfuseObservation]:
result = get(
URL(f"{creds.host}/api/public/observations"),
now: Final = datetime.now(timezone.utc)
result: Final = get(
URL(f"{creds.host}/api/public/v2/observations"),
headers=creds.auth_headers,
params=LangfuseListParams(
limit=100,
traceId=trace_id,
name=name,
fromStartTime=from_start_time,
fromStartTime=from_start_time or (now - timedelta(minutes=15)).isoformat(),
toStartTime=now.isoformat(),
),
response_type=LangfuseObservationList,
timeout=30.0,

View file

@ -32,6 +32,7 @@ pytestmark = pytest.mark.e2e
class LangsmithHeaders(Headers):
x_api_key: str = Field(serialization_alias="x-api-key")
x_tenant_id: str | None = Field(default=None, serialization_alias="x-tenant-id")
class LangsmithRunExtra(BaseModel):
@ -50,6 +51,7 @@ class LangsmithCreds:
api_key: str
base_url: str
project: str
tenant_id: str | None
def load_langsmith_creds() -> LangsmithCreds:
@ -62,6 +64,7 @@ def load_langsmith_creds() -> LangsmithCreds:
api_key=api_key,
base_url=(os.getenv("LANGSMITH_BASE_URL") or "https://api.smith.langchain.com").rstrip("/"),
project=os.getenv("LANGSMITH_PROJECT") or "litellm-e2e",
tenant_id=os.getenv("LANGSMITH_TENANT_ID"),
)
@ -69,7 +72,7 @@ def _fetch_run(creds: LangsmithCreds, run_id: uuid.UUID) -> LangsmithRun | None:
result = get_external(
f"{creds.base_url}/runs/{run_id}",
response_type=LangsmithRun,
headers=LangsmithHeaders(x_api_key=creds.api_key),
headers=LangsmithHeaders(x_api_key=creds.api_key, x_tenant_id=creds.tenant_id),
)
match result:
case Success(data=run):
@ -102,7 +105,10 @@ class TestLangsmithBatchSerialization:
async def test_non_json_native_metadata_reaches_langsmith(self) -> None:
creds: Final = load_langsmith_creds()
logger: Final = LangsmithLogger(
langsmith_api_key=creds.api_key, langsmith_project=creds.project, langsmith_base_url=creds.base_url
langsmith_api_key=creds.api_key,
langsmith_project=creds.project,
langsmith_base_url=creds.base_url,
langsmith_tenant_id=creds.tenant_id,
)
assert not logger.is_mock_mode, "LangsmithLogger initialised in mock mode; this e2e needs the real API"
marker: Final = unique_marker()

View file

@ -79,7 +79,7 @@ def _chain_reaches(span_id: str, root_id: str, trace: JaegerTrace) -> bool:
return False
def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None:
def assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None:
"""The enforced behavior: the destination holds exactly one call-id-tagged
trace for the call, rooted at the SERVER span, with auth/db children and
the gen-AI span all connected into that one tree - no dangling parent
@ -112,10 +112,9 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -
roots = [span for span in trace.spans if not span.references]
assert len(roots) == 1, f"expected exactly one root span, got {[s.operation_name for s in roots]}; spans: {names}"
root = roots[0]
assert root.operation_name == f"POST {route}", (
f"the root must be the SERVER span 'POST {route}', got {root.operation_name!r}"
)
assert root.kind == "server", f"the root span must have kind=server, got {root.kind!r}"
request_route: Final = root.tag("http.route") or root.tag("url.path") or root.tag("http.target")
assert request_route == route, f"the root must identify route {route!r}, got {request_route!r}"
assert f"auth {route}" in names, f"auth phase span 'auth {route}' missing; spans: {names}"
assert any(name.startswith(DB_SPAN_PREFIX) for name in names), (
@ -133,12 +132,12 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -
def _poll(otel_reader: OtelReader, *, call_id: str, route: str, genai_span: str) -> CallTraces:
return otel_reader.poll_traces_for_call(
call_id=call_id,
settled_names={f"POST {route}", f"auth {route}", genai_span},
settled_names={f"auth {route}", genai_span},
settled_prefixes={DB_SPAN_PREFIX},
)
def _assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None:
def assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None:
"""The enforced behavior: the gen-AI span for the attempt that served the
stream records a TTFT that is a real measurement - present, numeric,
positive, and strictly less than that span's own total duration. A TTFT of
@ -282,7 +281,7 @@ class TestOtelTraceCompleteness:
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
_assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"])
@meta(
@ -315,7 +314,7 @@ class TestOtelTraceCompleteness:
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
hits: Final = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
_assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"])
@meta(
@ -333,7 +332,7 @@ class TestOtelTraceCompleteness:
"""This test verifies that one successful non-streaming /v1/messages request
produces exactly one complete OTEL trace.
The trace must have a single root span named "POST /v1/messages". The
The trace must have a single root SERVER span identifying /v1/messages. The
authentication, database, and model-call spans must all belong to
the same trace and have valid parent relationships leading back to that root.
@ -353,7 +352,7 @@ class TestOtelTraceCompleteness:
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
_assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["responses"])
@meta(
@ -392,7 +391,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
@pytest.mark.covers("logging.otel.stream.exports_metric", exercised_on=["chat_completions"])
@meta(
@ -443,7 +442,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
served = one_served_genai_span(traces.hits[0], genai_span)
assert served.tag("litellm.request.streaming") is True, (
@ -500,7 +499,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
served = one_served_genai_span(traces.hits[0], genai_span)
assert served.tag("litellm.request.streaming") is True, (
@ -557,7 +556,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
one_served_genai_span(traces.hits[0], genai_span)
@ -614,7 +613,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_real_ttft(traces.hits, genai_span=genai_span)
assert_real_ttft(traces.hits, genai_span=genai_span)
@pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["messages"])
@meta(
@ -661,7 +660,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_real_ttft(traces.hits, genai_span=genai_span)
assert_real_ttft(traces.hits, genai_span=genai_span)
@pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["responses"])
@meta(
@ -708,7 +707,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_real_ttft(traces.hits, genai_span=genai_span)
assert_real_ttft(traces.hits, genai_span=genai_span)
@pytest.mark.covers("logging.otel.failure.exports_metric", exercised_on=["chat_completions"])
@meta(
@ -763,7 +762,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {model_name}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
root = next(span for span in traces.hits[0].spans if not span.references)
assert str(root.tag("http.status_code")) == "401", (
@ -825,7 +824,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {model_name}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
root = next(span for span in traces.hits[0].spans if not span.references)
assert str(root.tag("http.status_code")) == "401", (

View file

@ -418,7 +418,7 @@ class TestBudgetListV1:
@pytest.mark.covers("mgmt.budget.list_v1.happy_path")
@meta(Subject(domain=Domain.SPEND_BUDGETS, route=Route.BUDGET_MANAGEMENT))
def test_refuses_a_sort_field_and_a_parameter_it_does_not_support(self, client: ManagementClient) -> None:
assert _list_status(client, BudgetPageParams(sort="budget_duration")) == 400
assert _list_status(client, BudgetPageParams(sort="not_a_sort_field")) == 400
assert _list_status(client, BudgetPageParams(not_a_parameter="b-1")) == 400
@pytest.mark.covers("mgmt.budget.list_v1.admin_only")

View file

@ -28,11 +28,10 @@ from dataclasses import dataclass
from typing import Final
import pytest
from pydantic import BaseModel, ConfigDict, Field
from e2e_config import OTEL_QUERY_URL, POLL_INTERVAL, POLL_TIMEOUT
from e2e_http import URL, NetworkError, NoBody, Result, Success, get
from e2e_metadata import step
from pydantic import BaseModel, ConfigDict, Field
#: OTEL resource service.name the proxy exports under (OTEL_SERVICE_NAME default).
JAEGER_SERVICE = "litellm"
@ -118,9 +117,15 @@ def _ticks() -> Iterator[None]:
time.sleep(POLL_INTERVAL)
def _settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool:
present = set(trace.span_names())
return names.issubset(present) and all(any(name.startswith(prefix) for name in present) for prefix in prefixes)
def trace_settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool:
root: Final = root_span(trace)
present: Final = frozenset(trace.span_names())
return (
root is not None
and root.kind == "server"
and names.issubset(present)
and all(any(name.startswith(prefix) for name in present) for prefix in prefixes)
)
def root_span(trace: JaegerTrace) -> JaegerSpan | None:
@ -172,7 +177,7 @@ class _Observation:
def settled(self, names: set[str], prefixes: set[str]) -> bool:
hits: Final = self.traces.hits
return self.unreachable is None and len(hits) == 1 and not self.missing and _settled(hits[0], names, prefixes)
return self.unreachable is None and len(hits) == 1 and not self.missing and trace_settled(hits[0], names, prefixes)
@dataclass(frozen=True, slots=True)

View file

@ -14,8 +14,8 @@ from typing import Final, Protocol, cast
from provider_cache import LIFETIME_SECONDS, CacheBusy, CacheEdge, CacheHit, CacheLookup, CacheUnavailable, CaptureLease
from pydantic import TypeAdapter, ValidationError
from redis import Redis
from redis.exceptions import RedisError
from redis import Redis, RedisCluster
from redis.exceptions import RedisClusterException, RedisError
REDIS_ARRAY: Final[TypeAdapter[list[bytes]]] = TypeAdapter(list[bytes])
@ -83,7 +83,7 @@ class RedisResponseStore:
started: Final = time.monotonic()
try:
result: Final = self.client.eval(LOOKUP, 2, *self.keys(key), token, self.lifetime_ms, self.lease_ms)
except (RedisError, OSError):
except (RedisError, RedisClusterException, OSError):
return CacheUnavailable()
try:
parts: Final = tuple(REDIS_ARRAY.validate_python(result, strict=True))
@ -102,28 +102,46 @@ class RedisResponseStore:
result: Final = self.client.eval(
PUBLISH, 2, *self.keys(key), lease.token, lease.captured_at_ms, lease.expires_at_ms, payload, self.lifetime_ms,
)
except (RedisError, OSError):
except (RedisError, RedisClusterException, OSError):
return False
return result == 1
def release(self, key: str, lease: CaptureLease) -> bool:
try:
result: Final = self.client.eval(RELEASE, 1, self.keys(key)[1], lease.token)
except (RedisError, OSError):
except (RedisError, RedisClusterException, OSError):
return False
return result == 1
def discard(self, key: str, payload: bytes) -> bool:
try:
result: Final = self.client.eval(DISCARD, 1, self.keys(key)[0], payload)
except (RedisError, OSError):
except (RedisError, RedisClusterException, OSError):
return False
return result == 1
@functools.lru_cache(maxsize=16)
def _redis_commands(url: str, cluster: bool) -> RedisCommands:
if cluster:
return cast(
RedisCommands, RedisCluster(url=url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False)
)
return cast(RedisCommands, Redis.from_url(url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False))
@dataclass(frozen=True, slots=True)
class LazyRedisCommands:
url: str
cluster: bool
def eval(self, script: str, numkeys: int, *args: str | bytes | int) -> object:
return _redis_commands(self.url, self.cluster).eval(script, numkeys, *args)
def redis_store(url: str, namespace: str) -> RedisResponseStore:
client: Final = Redis.from_url(url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False)
return RedisResponseStore(cast(RedisCommands, client), namespace)
client: Final = LazyRedisCommands(url, os.environ.get("E2E_PROVIDER_CACHE_REDIS_CLUSTER") == "1")
return RedisResponseStore(client, namespace)
def write_metrics(cache: CacheEdge) -> None:

View file

@ -55,7 +55,7 @@ from spend_e2e_client import (
pytestmark = pytest.mark.e2e
CHAT_MODEL: Final = "gemini-2.5-flash"
CHAT_MODEL: Final = "gemini-3.8-flash"
MESSAGES_MODEL: Final = "claude-haiku-4-5"
RESPONSES_MODEL: Final = "openai-responses-codex"
EMBED_MODEL: Final = "openai-text-embedding-3-small"

View file

@ -3,7 +3,7 @@
Run against a proxy started with the gateway config. Coverage rationale:
SPEND_TRACKING_COVERAGE_MATRIX.md.
Model names are literals from that config: chat tests hit "gemini-2.5-flash",
Model names are literals from that config: chat tests hit "gemini-3.8-flash",
embedding tests hit "openai-text-embedding-3-small".
Every test: fresh scoped key (isolation) -> real provider call -> unwrap (hard
@ -37,7 +37,7 @@ from spend_reconciliation import BACKEND as TRAFFIC_BACKEND
pytestmark = pytest.mark.e2e
GEMINI_MODEL = "gemini-2.5-flash"
GEMINI_MODEL = "gemini-3.8-flash"
CLAUDE_MODEL = "claude-haiku-4-5"
CODEX_MODEL = "openai-responses-codex"
EMBEDDING_MODEL = "openai-text-embedding-3-small"
@ -780,7 +780,7 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None:
GEMINI_MODEL, "estimate the cost of this request"
)
assert cost > 0, (
"/spend/calculate returned 0 for gemini-2.5-flash; "
"/spend/calculate returned 0 for gemini-3.8-flash; "
"cost map may be missing this model"
)

View file

@ -30,7 +30,7 @@ from models import (
from proxy_client import ProxyClient
from pydantic import BaseModel, ValidationError
pytestmark = pytest.mark.e2e
pytestmark = [pytest.mark.e2e, pytest.mark.provider_live]
BEDROCK_INVOKE_BACKEND: Final = "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0"
SEARCH_CALL_TYPE: Final = "asearch"

View file

@ -57,11 +57,9 @@ CONTENT_POLICY_PROMPT = (
COOLDOWN_SECONDS = 30.0
REPLICA_PROPAGATION_SECONDS = 15.0
# The smallest-context chat model OpenAI still serves (16385 tokens). A prompt
# past that limit comes back as a real `context_length_exceeded` 400, which is
# what litellm maps to ContextWindowExceededError.
SMALL_CONTEXT_MODEL = "openai/gpt-3.5-turbo"
SMALL_CONTEXT_LIMIT_TOKENS = 16385
SMALL_CONTEXT_MODEL: Final = "openai/gpt-5.4-nano"
# model_prices_and_context_window.json, gpt-5.4-nano, verified 2026-11-02
SMALL_CONTEXT_LIMIT_TOKENS: Final = 272000
def oversized_prompt(marker: str) -> str:
@ -107,7 +105,7 @@ def create_timeout_deployment(proxy: ProxyClient, name: str) -> str:
@step(f"Add a deployment named {{name}} that calls the small-context model {SMALL_CONTEXT_MODEL}")
def create_small_context_deployment(proxy: ProxyClient, name: str) -> str:
"""Register a deployment on the smallest-context model OpenAI still serves, so an
"""Register a deployment with a smaller context window than the backup, so an
oversized prompt earns a real context-window refusal from the provider."""
return proxy.create_model(name, LiteLLMParamsBody(model=SMALL_CONTEXT_MODEL, api_key=REAL_KEY))
@ -263,8 +261,7 @@ def spend_only_request_of(proxy: ProxyClient, spent_key: str) -> None:
@step(f"Add a deployment named {{name}} that calls {SMALL_CONTEXT_MODEL} and takes all of its group's traffic")
def create_always_picked_small_context_deployment(proxy: ProxyClient, name: str) -> str:
"""The always-picked half of a retry pair on the smallest-context model OpenAI
still serves: it holds all of the model group's shuffle weight, so an oversized
"""The always-picked half of a retry pair on the smaller-context model: it holds all of the model group's shuffle weight, so an oversized
prompt opens on it and earns a real context-window refusal, which never benches
a deployment, so only the retry itself can steer the request off it."""
return proxy.register_model(

View file

@ -37,7 +37,7 @@ export async function navigateToPage(page: PlaywrightPage, pageEnum: Page): Prom
// assert against the default page.
for (let attempt = 0; attempt < 3; attempt++) {
await page.goto(`/ui?page=${pageEnum}`);
await page.waitForLoadState("networkidle");
await expect(page.getByRole("complementary")).toBeVisible();
const url = new URL(page.url());
const onLegacyRoot = url.pathname.replace(/\/+$/, "").endsWith("/ui");
if (!onLegacyRoot || url.searchParams.get("page") === pageEnum) {

View file

@ -391,7 +391,7 @@ test.describe("Add Model", () => {
// Navigate to Deployed Models tab
await page.getByRole("tab", { name: "Deployed Models" }).click();
await page.waitForLoadState("networkidle");
await expect(page.getByPlaceholder("Search model names")).toBeVisible();
// Search for the model we just added
await page.getByPlaceholder("Search model names").fill("claude-haiku-4-5");
@ -470,7 +470,7 @@ test.describe("Add Model", () => {
// The Models table renders team-scoped models with the team id in the row.
await page.getByRole("tab", { name: "Deployed Models" }).click();
await page.waitForLoadState("networkidle");
await expect(page.getByPlaceholder("Search model names")).toBeVisible();
await page.getByPlaceholder("Search model names").fill("cohere");
@ -515,7 +515,7 @@ test.describe("Add Model", () => {
// Navigate to Deployed Models tab
await page.getByRole("tab", { name: "Deployed Models" }).click();
await page.waitForLoadState("networkidle");
await expect(page.getByPlaceholder("Search model names")).toBeVisible();
// Search for the wildcard model
await page.getByPlaceholder("Search model names").fill("cohere");

View file

@ -27,7 +27,7 @@ async function readStoredParams(
}
function paramsEditor(page: PlaywrightPage) {
return page.getByPlaceholder('"rpm": 100');
return page.getByRole("textbox", { name: "LiteLLM Params", exact: true });
}
async function editParams(

View file

@ -0,0 +1,49 @@
import json
from typing import Final
import pytest
from pydantic import JsonValue
from logging_client import LangfuseObservation, is_otel_v2_generation, observation_spend
@pytest.mark.parametrize("encoded", [False, True])
@pytest.mark.parametrize("cost", [0.0, 0.000023])
def test_observation_preserves_v2_content_usage_and_cost(encoded: bool, cost: float) -> None:
content: Final[JsonValue] = [{"role": "user", "content": "unique-request-marker"}]
output: Final[JsonValue] = [{"role": "assistant", "content": "response"}]
metadata: Final[JsonValue] = {
"attributes": {
"langfuse.observation.type": "generation",
"litellm.metadata.user_api_key_alias": "test-key-alias",
}
}
usage: Final[JsonValue] = {"input": 17, "output": 3}
observation: Final = LangfuseObservation.model_validate(
{
"id": "observation",
"input": json.dumps(content) if encoded else content,
"output": json.dumps(output) if encoded else output,
"metadata": json.dumps(metadata) if encoded else metadata,
"usageDetails": usage,
"totalCost": cost,
}
)
assert observation.input == content
assert observation.output == output
assert observation.usage_details == usage
assert observation_spend(observation) == cost
assert is_otel_v2_generation(observation, key_alias="test-key-alias")
assert not is_otel_v2_generation(observation, key_alias="another-key")
def test_observation_preserves_plain_text_and_legacy_cost() -> None:
cost: Final = 0.000031
observation: Final = LangfuseObservation.model_validate(
{"id": "observation", "input": "plain prompt", "output": "plain reply", "calculatedTotalCost": cost}
)
assert observation.input == "plain prompt"
assert observation.output == "plain reply"
assert observation_spend(observation) == cost

View file

@ -8,8 +8,21 @@ helper is worth pinning here.
from __future__ import annotations
from typing import Final
import pytest
from otel_client import TTFT_TAG, JaegerTrace, one_served_genai_span, served_genai_spans
from otel_client import (
TTFT_TAG,
CallTraces,
JaegerReference,
JaegerSpan,
JaegerTag,
JaegerTrace,
one_served_genai_span,
served_genai_spans,
trace_settled,
)
from test_otel_trace_e2e import assert_complete_trace, assert_real_ttft
GENAI_SPAN = "chat claude-haiku-4-5"
@ -79,3 +92,73 @@ def test_other_operations_are_not_counted() -> None:
trace = _trace(_span("chat gpt-5.5", ttft=0.3), _span(GENAI_SPAN, ttft=0.3))
assert len(served_genai_spans(trace, GENAI_SPAN)) == 1
def _request_trace(
*,
root_name: str = "POST",
root_kind: str = "server",
route: str = "/v1/messages",
genai_parent: str = "root",
ttft: float = 0.2,
) -> JaegerTrace:
parent: Final = JaegerReference(refType="CHILD_OF", traceID="request", spanID="root")
return JaegerTrace(
traceID="request",
spans=[
JaegerSpan(
spanID="root",
operationName=root_name,
tags=[JaegerTag(key="span.kind", value=root_kind), JaegerTag(key="http.target", value=route)],
),
JaegerSpan(spanID="auth", operationName="auth /v1/messages", references=[parent]),
JaegerSpan(spanID="db", operationName="postgres.select tokens", references=[parent]),
JaegerSpan(
spanID="genai",
operationName=GENAI_SPAN,
duration=1_000_000,
references=[JaegerReference(refType="CHILD_OF", traceID="request", spanID=genai_parent)],
tags=[JaegerTag(key="span.kind", value="client"), JaegerTag(key=TTFT_TAG, value=ttft)],
),
],
)
@pytest.mark.parametrize("root_name", ("POST", "POST /v1/messages"))
def test_connected_request_trace_is_independent_of_server_display_name(root_name: str) -> None:
trace: Final = _request_trace(root_name=root_name)
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
assert_real_ttft((trace,), genai_span=GENAI_SPAN)
assert trace_settled(trace, {"auth /v1/messages", GENAI_SPAN}, {"postgres."})
def test_incomplete_trace_cannot_settle_before_the_server_span_arrives() -> None:
trace: Final = _request_trace()
incomplete: Final = trace.model_copy(update={"spans": trace.spans[1:]})
assert not trace_settled(incomplete, {"auth /v1/messages", GENAI_SPAN}, {"postgres."})
@pytest.mark.parametrize("route", ("/v1/responses", ""))
def test_request_trace_must_still_identify_the_requested_route(route: str) -> None:
trace: Final = _request_trace(root_name="POST /v1/messages", route=route)
with pytest.raises(AssertionError, match="route"):
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
def test_request_trace_must_still_be_rooted_at_a_server_span() -> None:
trace: Final = _request_trace(root_name="POST /v1/messages", root_kind="client")
with pytest.raises(AssertionError, match="kind=server"):
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
def test_request_trace_must_still_connect_the_model_call_to_the_request() -> None:
trace: Final = _request_trace(genai_parent="absent")
with pytest.raises(AssertionError, match="orphaned trace"):
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
@pytest.mark.parametrize("ttft", (0.0, 1.0))
def test_server_name_tolerance_does_not_accept_invalid_first_token_timing(ttft: float) -> None:
trace: Final = _request_trace(ttft=ttft)
with pytest.raises(AssertionError, match="TTFT"):
assert_real_ttft((trace,), genai_span=GENAI_SPAN)

View file

@ -5,5 +5,8 @@
addopts = --strict-markers --strict-config -p no:cacheprovider
pythonpath = ../e2e ../e2e/batches ../e2e/guardrails ../e2e/logging ../e2e/perf
markers =
e2e: live assertion modules imported by harness regression tests
meta(subject): metadata on live assertion modules imported by harness regression tests
otel_tls: TLS coverage in live assertion modules imported by harness regression tests
covers(cell_id, *, exercised_on=()): coverage-registry cell(s) a test covers; exercised here only to prove the collector and the JUnit properties read it
cli_determinism: drives the real claude CLI for several seconds

View file

@ -25,9 +25,16 @@ _GEO: Final = f"au.{_ROW}"
_FINETUNE: Final = "ft:gpt-audit-reload-2025-01-01"
_CLAUDE: Final = "claude-audit-reload-sonnet"
_MINI: Final = "gpt-audit-reload-mini"
_FREE: Final = "gpt-audit-reload-free"
_RULE_FREE: Final = "gpt-audit-rule-free"
_FILL_FREE: Final = "gpt-audit-fill-free"
_RULE_UNPRICED: Final = "gpt-audit-rule-unpriced"
_REPORTED_FREE: Final = "gpt-audit-reported-free"
_USAGE: Final = re.compile(r"bill (\d+) (\d+) (\d+)")
_CUSTOM: Final = MappingProxyType({"input_cost_per_token": 1.1e-06, "output_cost_per_token": 2.3e-06})
_COST_MAP_LIMITS: Final = MappingProxyType({"MODEL_COST_MAP_MIN_MODEL_COUNT": "1", "MODEL_COST_MAP_MAX_SHRINK_RATIO": "0"})
_COST_MAP_LIMITS: Final = MappingProxyType(
{"MODEL_COST_MAP_MIN_MODEL_COUNT": "1", "MODEL_COST_MAP_MAX_SHRINK_RATIO": "0"}
)
_PUBLISHED_SCALE: Final = 2.0
_CATALOG: Final[Mapping[str, Mapping[str, JsonValue]]] = MappingProxyType(
{
@ -60,6 +67,13 @@ _CATALOG: Final[Mapping[str, Mapping[str, JsonValue]]] = MappingProxyType(
"output_cost_per_token": 1.5e-05,
"cache_read_input_token_cost": 3e-07,
},
_FILL_FREE: {"litellm_provider": "openai", "mode": "chat"},
_FREE: {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
},
_MINI: {
"litellm_provider": "openai",
"mode": "chat",
@ -99,6 +113,8 @@ class _Deployment:
prices: Mapping[str, float]
usage: _Usage
rates: Callable[[float], _Rates | None]
reported_cost: float | None = None
listed_rates: Callable[[float], _Rates | None] | None = None
@dataclass(frozen=True, slots=True)
@ -107,10 +123,23 @@ class _Charge:
expected: float
def _catalog(scale: float) -> dict[str, dict[str, JsonValue]]:
def _catalog(scale: float) -> dict[str, JsonValue]:
return {
model: {name: value * scale if isinstance(value, float) else value for name, value in row.items()}
for model, row in _CATALOG.items()
**{
model: {name: value * scale if isinstance(value, float) else value for name, value in row.items()}
for model, row in _CATALOG.items()
},
"fallback_generalizations": {
"rules": [
{
"name": "free-pricing",
"pattern": f"({_RULE_FREE}|{_FILL_FREE})",
"fill_missing_for_providers": ["openai"],
"model_info": {"mode": "chat", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0},
},
{"name": "unpriced-capabilities", "pattern": _RULE_UNPRICED, "model_info": {"mode": "chat"}},
]
},
}
@ -240,6 +269,7 @@ def _reply(request: Request) -> Reply:
"completion_tokens": usage.completion,
"total_tokens": usage.prompt + usage.completion,
"prompt_tokens_details": {"cached_tokens": usage.cached},
**({"cost": 0.0} if object_value(json.loads(request.body))["model"] == _REPORTED_FREE else {}),
},
}
).encode()
@ -281,10 +311,16 @@ def _billed(client: httpx.Client, key: str, deployment: _Deployment) -> tuple[st
def _charged_as_listed(proxy: Gateway, deployment: _Deployment, scale: float) -> _Charge:
rates: Final = deployment.rates(scale)
assert _listed_rates(proxy, deployment.name) == _expected_listing(rates), deployment.name
assert _listed_rates(proxy, deployment.name) == _expected_listing(
deployment.listed_rates(scale) if deployment.listed_rates is not None else rates
), deployment.name
request_id, charged = _billed(proxy.client, proxy.key, deployment)
expected: Final = _expected_cost(rates, deployment.usage)
assert charged == pytest.approx(expected, rel=1e-6), deployment.name
expected: Final = (
deployment.reported_cost if deployment.reported_cost is not None else _expected_cost(rates, deployment.usage)
)
assert charged == (
None if rates is None and deployment.reported_cost is None else pytest.approx(expected, rel=1e-6)
), deployment.name
return _Charge(request_id=request_id, expected=expected)
@ -352,8 +388,31 @@ def test_price_data_reload_reprices_catalog_deployments_and_keeps_custom_prices(
_Usage(2000, 1000, 1000),
_catalog_rates(_FINETUNE),
),
_Deployment("finetune-custom", f"openai/{_FINETUNE}:org:suffix:z", _CUSTOM, _Usage(2000, 0, 1000), _custom_rates),
_Deployment(
"finetune-custom", f"openai/{_FINETUNE}:org:suffix:z", _CUSTOM, _Usage(2000, 0, 1000), _custom_rates
),
_Deployment("cross-provider-unpriced", f"azure/{_CLAUDE}", {}, _Usage(2000, 1000, 1000), _unpriced),
_Deployment("catalog-free", f"openai/{_FREE}", {}, _Usage(2000, 1000, 1000), _catalog_rates(_FREE)),
_Deployment("rule-free", f"openai/{_RULE_FREE}", {}, _Usage(2000, 1000, 1000), _catalog_rates(_FREE)),
_Deployment(
"fill-rule-free",
f"openai/{_FILL_FREE}",
{},
_Usage(2000, 1000, 1000),
_catalog_rates(_FREE),
listed_rates=_unpriced,
),
_Deployment("rule-unpriced", f"openai/{_RULE_UNPRICED}", {}, _Usage(2000, 1000, 1000), _unpriced),
_Deployment(
"provider-reported-free", f"openrouter/{_REPORTED_FREE}", {}, _Usage(2000, 1000, 1000), _unpriced, 0.0
),
_Deployment(
"cross-provider-free",
f"azure/{_CLAUDE}",
{"input_cost_per_token": 0.0, "output_cost_per_token": 0.0},
_Usage(2000, 1000, 1000),
_catalog_rates(_FREE),
),
_Deployment(
"region-custom-above-threshold",
f"bedrock/ap-northeast-1.{_ROW}",

View file

@ -0,0 +1,455 @@
import itertools
import json
import os
import uuid
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
from contextlib import asynccontextmanager
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Final
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
import psycopg
import pytest
from prisma import Prisma
from psycopg import sql
from pydantic import BaseModel, ConfigDict, Field
from litellm.proxy.spend_tracking.session_page import read_session_page
from litellm.proxy.spend_tracking.spend_management_endpoints import _fetch_session_representatives
from tests.integration._support.client import Gateway
from tests.integration._support.database import write_rows
_URL: Final = "/spend/logs/ui"
_WINDOW_SECONDS: Final = 3600
@dataclass(frozen=True, slots=True)
class _Row:
request: str
session: str | None
api_key: str
second: int
status: str = "success"
model: str = "m-a"
_ROWS: Final = (
_Row("z1", "z", "key-1", -1),
_Row("a1", "a", "key-1", 10),
_Row("a2", "a", "key-1", 50),
_Row("b1", "b", "key-1", 20, model="m-b"),
_Row("b2", "b", "key-1", 60),
_Row("b3", "b", "key-1", 30),
_Row("c1", "c", "key-2", 60),
_Row("d1", "b", "key-2", 40, model="m-b"),
_Row("e1", None, "key-1", 55, model="m-b"),
_Row("f1", "", "key-1", 45),
_Row("g1", "g", "key-3", 5, status="failure"),
_Row("g2", "g", "key-3", 15, status="failure"),
*(_Row(f"h{second - 69}", "h", "key-1", second) for second in range(70, 76)),
_Row("i1", "i", "key-3", 35, status="failure"),
_Row("i2", "i", "key-3", _WINDOW_SECONDS + 1),
)
_NEWEST_FIRST: Final = (
("h", "key-1"),
("c", "key-2"),
("b", "key-1"),
("e1", "key-1"),
("a", "key-1"),
("f1", "key-1"),
("b", "key-2"),
("i", "key-3"),
("g", "key-3"),
)
class _LogRow(BaseModel):
model_config = ConfigDict(frozen=True)
request_id: str
session_id: str | None
api_key: str
session_total_count: int
class _Page(BaseModel):
model_config = ConfigDict(frozen=True)
data: tuple[_LogRow, ...]
total: int
total_is_capped: bool
has_more: bool
next_session_cursor: str | None
@dataclass(frozen=True, slots=True)
class _Seeded:
prefix: str
window: Mapping[str, str]
def session(self, row: _LogRow) -> tuple[str, str]:
key: Final = row.session_id or row.request_id
assert key.startswith(self.prefix), key
return key.removeprefix(self.prefix), row.api_key
@pytest.fixture
def seeded(gateway: Gateway) -> Iterator[_Seeded]:
prefix: Final = f"sess{uuid.uuid4().hex[:12]}-"
start: Final = datetime(1900, 1, 1) + timedelta(days=int(prefix[4:16], 16) % 200000)
rows: Final = [
{
"request_id": f"{prefix}{row.request}",
"session_id": None if row.session is None else f"{prefix}{row.session}" if row.session else "",
"api_key": row.api_key,
"start_time": (start + timedelta(seconds=row.second)).isoformat(),
"status": row.status,
"model": row.model,
}
for row in _ROWS
]
with gateway.scenario() as scenario:
scenario.cleanups.callback(
write_rows, 'DELETE FROM "LiteLLM_SpendLogs" WHERE request_id LIKE %s', (f"{prefix}%",)
)
write_rows(
'INSERT INTO "LiteLLM_SpendLogs" (request_id, call_type, api_key, "startTime", "endTime", model, '
"session_id, status) SELECT request_id, 'acompletion', api_key, start_time, start_time, model, session_id, "
"status FROM jsonb_to_recordset(%s::jsonb) AS seeded(request_id text, session_id text, api_key text, "
"start_time timestamp, status text, model text)",
(json.dumps(rows),),
)
yield _Seeded(
prefix,
{
"start_date": f"{start:%Y-%m-%d %H:%M:%S}",
"end_date": f"{start + timedelta(seconds=_WINDOW_SECONDS):%Y-%m-%d %H:%M:%S}",
"group_by_session": "true",
},
)
def _page(gateway: Gateway, seeded: _Seeded, params: Mapping[str, str]) -> _Page:
return _Page.model_validate(gateway.get(_URL, {**seeded.window, **params}))
def _cursor_walk(gateway: Gateway, seeded: _Seeded, params: Mapping[str, str], number: int = 1) -> tuple[_Page, ...]:
page: Final = _page(gateway, seeded, {**params, "page_size": "2", "page": str(number)})
if page.next_session_cursor is None or number > len(_NEWEST_FIRST):
return (page,)
following: Final = {**params, "session_cursor": page.next_session_cursor}
return (page, *_cursor_walk(gateway, seeded, following, number + 1))
def _rows(pages: tuple[_Page, ...]) -> Iterator[_LogRow]:
return itertools.chain.from_iterable(page.data for page in pages)
def _sessions(seeded: _Seeded, pages: tuple[_Page, ...]) -> tuple[tuple[str, str], ...]:
return tuple(seeded.session(row) for row in _rows(pages))
def test_cursor_pages_show_every_session_once_newest_first_with_its_newest_row(
gateway: Gateway, seeded: _Seeded
) -> None:
pages: Final = _cursor_walk(gateway, seeded, {})
assert _sessions(seeded, pages) == _NEWEST_FIRST
assert [(page.total, page.total_is_capped) for page in pages] == [(9, False)] * 5
assert [page.has_more for page in pages] == [True, True, True, True, False]
representatives: Final = tuple(
(row.request_id.removeprefix(seeded.prefix), row.session_total_count) for row in _rows(pages)
)
assert representatives == (
("h6", 6),
("c1", 1),
("b2", 3),
("e1", 1),
("a2", 2),
("f1", 1),
("d1", 1),
("i1", 2),
("g2", 2),
)
@pytest.mark.parametrize(
"sort_order, expected",
[("desc", _NEWEST_FIRST), ("asc", tuple(reversed(_NEWEST_FIRST)))],
ids=["newest-first", "oldest-first"],
)
def test_numbered_pages_show_the_same_sessions_as_the_cursor_walk(
gateway: Gateway, seeded: _Seeded, sort_order: str, expected: tuple[tuple[str, str], ...]
) -> None:
pages: Final = tuple(
_page(gateway, seeded, {"page_size": "2", "page": str(number), "sort_order": sort_order})
for number in range(1, 7)
)
assert _sessions(seeded, pages) == expected
assert [len(page.data) for page in pages] == [2, 2, 2, 2, 1, 0]
assert [page.has_more for page in pages] == [True, True, True, True, False, False]
assert {page.total for page in pages} == {len(expected)}
def test_failure_filter_keeps_only_sessions_whose_outcome_failed(gateway: Gateway, seeded: _Seeded) -> None:
pages: Final = _cursor_walk(gateway, seeded, {"status_filter": "failure"})
assert _sessions(seeded, pages) == (("i", "key-3"), ("g", "key-3"))
assert pages[0].total == 2
def test_model_filter_ranks_each_session_by_its_matching_rows_only(gateway: Gateway, seeded: _Seeded) -> None:
pages: Final = _cursor_walk(gateway, seeded, {"model": "m-b"})
assert _sessions(seeded, pages) == (("e1", "key-1"), ("b", "key-2"), ("b", "key-1"))
assert tuple(row.request_id.removeprefix(seeded.prefix) for row in _rows(pages)) == ("e1", "d1", "b1")
assert pages[0].total == 3
_COPIED_ROWS: Final = 20_000
_COPY_START: Final = datetime(2000, 1, 1, tzinfo=UTC)
_COPY_WINDOW: Final = (
"\"startTime\" >= ($1::timestamptz AT TIME ZONE 'UTC') AND \"startTime\" <= ($2::timestamptz AT TIME ZONE 'UTC')"
)
_COPY_WINDOW_PARAMS: Final = (_COPY_START, _COPY_START + timedelta(days=1))
def _generated_sessions(rows_per_session: int) -> tuple[sql.SQL, tuple[object, ...]]:
"""`_COPIED_ROWS` rows a second apart in sessions of `rows_per_session` rows, a one-row session keyed by request_id."""
return sql.SQL(
'INSERT INTO {table} (request_id, call_type, api_key, "startTime", "endTime", session_id) '
"SELECT 'r' || n, 'acompletion', 'key-1', moment, moment, "
"CASE WHEN %s > 1 THEN 's' || (n - 1) / %s END FROM generate_series(1, %s) n, "
"LATERAL (SELECT %s::timestamp + n * INTERVAL '1 second' AS moment) seconds"
), (rows_per_session, rows_per_session, _COPIED_ROWS, _COPY_START.replace(tzinfo=None))
def _sessions_tied_at_the_newest_moment() -> tuple[sql.SQL, tuple[object, ...]]:
"""`_generated_sessions(1)` plus thirty one-row sessions sharing a newer moment, inserted and keyed against session
order so a read cut inside the tie misses the sessions that rank first."""
start: Final = _COPY_START.replace(tzinfo=None)
return sql.SQL(
'INSERT INTO {table} (request_id, call_type, api_key, "startTime", "endTime", session_id) '
"SELECT request_id, 'acompletion', 'key-1', moment, moment, session_id FROM ("
"SELECT n AS ordinal, 'r' || n AS request_id, %s::timestamp + n * INTERVAL '1 second' AS moment, "
"NULL AS session_id FROM generate_series(1, %s) n UNION ALL "
"SELECT %s + 30 - n, 't' || lpad((29 - n)::text, 2, '0'), %s::timestamp + %s * INTERVAL '1 second', "
"'s' || lpad(n::text, 2, '0') FROM generate_series(0, 29) n) seeded ORDER BY ordinal"
), (start, _COPIED_ROWS, _COPIED_ROWS, start, _COPIED_ROWS + 1)
@dataclass(frozen=True, slots=True)
class _Call:
session: str
call_type: str
status: str | None
second: int
def _calls(calls: tuple[_Call, ...]) -> tuple[sql.SQL, tuple[object, ...]]:
"""One row per call, `second` seconds into the window, ending a second later."""
seeded: Final = tuple(
{
"request_id": f"{call.session}-{call.second}",
"call_type": call.call_type,
"session_id": call.session,
"status": call.status,
"start_time": (_COPY_START + timedelta(seconds=call.second)).replace(tzinfo=None).isoformat(),
}
for call in calls
)
return sql.SQL(
'INSERT INTO {table} (request_id, call_type, api_key, "startTime", "endTime", session_id, status) '
"SELECT request_id, call_type, 'key-1', start_time, start_time + INTERVAL '1 second', session_id, status "
"FROM jsonb_to_recordset(%s::jsonb) AS calls(request_id text, call_type text, session_id text, status text, "
"start_time timestamp)"
), (json.dumps(seeded),)
@asynccontextmanager
async def _spend_logs_copy(insert: sql.SQL, args: tuple[object, ...]) -> AsyncIterator[Prisma]:
"""A private schema holding a copy of the migrated `LiteLLM_SpendLogs` with its indexes, seeded by `insert`."""
schema: Final = f"integration_{uuid.uuid4().hex}"
url: Final = os.environ["DATABASE_URL"]
table: Final = sql.Identifier(schema, "LiteLLM_SpendLogs")
with psycopg.connect(url, autocommit=True) as setup:
setup.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema)))
try:
setup.execute(sql.SQL('CREATE TABLE {} (LIKE "LiteLLM_SpendLogs" INCLUDING ALL)').format(table))
setup.execute(insert.format(table=table), args)
setup.execute(sql.SQL("ANALYZE {}").format(table))
parsed: Final = urlsplit(url)
query: Final = urlencode({**dict(parse_qsl(parsed.query)), "schema": schema})
database: Final = Prisma(datasource={"url": urlunsplit(parsed._replace(query=query))})
await database.connect()
try:
yield database
finally:
await database.disconnect()
finally:
setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema)))
class _PlanNode(BaseModel):
model_config = ConfigDict(frozen=True)
relation_name: str | None = Field(default=None, alias="Relation Name")
actual_rows: float = Field(alias="Actual Rows")
rows_removed_by_filter: float = Field(default=0, alias="Rows Removed by Filter")
actual_loops: int = Field(alias="Actual Loops")
plans: tuple["_PlanNode", ...] = Field(default=(), alias="Plans")
def spend_log_rows_scanned(self) -> int:
own: Final = (
(self.actual_rows + self.rows_removed_by_filter) * self.actual_loops
if self.relation_name == "LiteLLM_SpendLogs"
else 0
)
return round(own) + sum(child.spend_log_rows_scanned() for child in self.plans)
class _Explained(BaseModel):
model_config = ConfigDict(frozen=True)
plan: _PlanNode = Field(alias="Plan")
class _ExplainedReads:
"""Runs each read through Prisma, first under EXPLAIN ANALYZE to count the `LiteLLM_SpendLogs` rows it scans."""
def __init__(self, database: Prisma) -> None:
self.database: Final = database
self.rows_scanned: tuple[int, ...] = ()
async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]:
explained: Final = await self.database.query_raw(f"EXPLAIN (ANALYZE, FORMAT JSON) {query}", *args)
plan: Final = _Explained.model_validate(explained[0]["QUERY PLAN"][0]).plan
self.rows_scanned = (*self.rows_scanned, plan.spend_log_rows_scanned())
return await self.database.query_raw(query, *args)
@pytest.mark.parametrize(
"rows_per_session, page, newest_session, rows_scanned_per_read, total",
[
pytest.param(1, 1, ("r", 20_000), (102,), (100, True), id="first-page"),
pytest.param(1, 3, ("r", 19_980), (102,), (100, True), id="third-page"),
pytest.param(10, 1, ("s", 1_999), (102, 1_171), (100, True), id="second-read-sized-by-the-first-reads-density"),
pytest.param(100, 1, ("s", 199), (102, 6_439, 12_878), (100, True), id="third-read-at-least-doubled"),
pytest.param(1_000, 1, ("s", 19), (102, 12_878, 20_000), (20, False), id="last-read-runs-out-of-rows"),
],
)
async def test_a_newest_first_page_scans_only_the_rows_its_reads_ask_for(
rows_per_session: int,
page: int,
newest_session: tuple[str, int],
rows_scanned_per_read: tuple[int, ...],
total: tuple[int, bool],
) -> None:
async with _spend_logs_copy(*_generated_sessions(rows_per_session)) as database:
reads: Final = _ExplainedReads(database)
session_page: Final = await read_session_page(
reads,
where_clause=_COPY_WINDOW,
params=_COPY_WINDOW_PARAMS,
page=page,
page_size=10,
status=None,
sort_desc=True,
session_cursor=None,
cap=100,
)
prefix, newest = newest_session
assert tuple(session.key for session in session_page.sessions) == tuple(
f"{prefix}{newest - n}" for n in range(10)
)
assert (session_page.total, session_page.total_is_capped) == total
assert reads.rows_scanned == rows_scanned_per_read
async def test_a_page_cut_inside_a_tie_reads_on_until_the_tie_is_whole() -> None:
async with _spend_logs_copy(*_sessions_tied_at_the_newest_moment()) as database:
reads: Final = _ExplainedReads(database)
session_page: Final = await read_session_page(
reads,
where_clause=_COPY_WINDOW,
params=_COPY_WINDOW_PARAMS,
page=1,
page_size=5,
status=None,
sort_desc=True,
session_cursor=None,
cap=10,
)
assert tuple(session.key for session in session_page.sessions) == ("s29", "s28", "s27", "s26", "s25")
assert reads.rows_scanned == (12, 24, 48)
@dataclass(frozen=True, slots=True)
class _ExplainedPrisma:
db: _ExplainedReads
async def test_a_pages_representatives_are_found_by_session_not_by_scanning_the_window() -> None:
async with _spend_logs_copy(*_generated_sessions(rows_per_session=10)) as database:
reads: Final = _ExplainedReads(database)
representatives: Final = await _fetch_session_representatives(
prisma_client=_ExplainedPrisma(reads), # pyright: ignore[reportArgumentType] # only .db is used
where_clause=_COPY_WINDOW,
sql_params=_COPY_WINDOW_PARAMS,
next_param_index=len(_COPY_WINDOW_PARAMS) + 1,
session_keys=(("s1999", "key-1"), ("s1998", "key-1")),
)
assert tuple(row["request_id"] for row in representatives) == ("r20000", "r19990")
assert reads.rows_scanned == (20,), "only the two sessions' rows, not the window's 20,000"
async def test_a_page_number_far_past_the_cap_shows_no_session() -> None:
async with _spend_logs_copy(*_generated_sessions(rows_per_session=1)) as database:
session_page: Final = await read_session_page(
_ExplainedReads(database),
where_clause=_COPY_WINDOW,
params=_COPY_WINDOW_PARAMS,
page=9_300_000_000_000_000,
page_size=1000,
status=None,
sort_desc=True,
session_cursor=None,
cap=100,
)
assert (session_page.sessions, session_page.total, session_page.total_is_capped) == ((), 100, True)
_OUTCOME_CALLS: Final = (
_Call("agent", "asend_message", "failure", 10),
_Call("agent", "acompletion", "success", 20),
_Call("mcp", "acompletion", "failure", 5),
_Call("mcp", "call_mcp_tool", "success", 30),
_Call("unknown", "acompletion", None, 40),
_Call("failed", "acompletion", "success", 1),
_Call("failed", "acompletion", "failure", 50),
_Call("recovered", "acompletion", "failure", 2),
_Call("recovered", "acompletion", "success", 45),
)
@pytest.mark.parametrize(
"status, expected",
[("failure", ("failed", "mcp", "agent")), ("success", ("recovered", "unknown"))],
)
async def test_a_status_filter_keeps_sessions_whose_representative_call_has_that_status(
status: str, expected: tuple[str, ...]
) -> None:
"""An agent message stands for its session, then the newest call that is not an MCP tool call, and a missing status
counts as success. The page stays ordered by each session's newest call."""
async with _spend_logs_copy(*_calls(_OUTCOME_CALLS)) as database:
session_page: Final = await read_session_page(
_ExplainedReads(database),
where_clause=_COPY_WINDOW,
params=_COPY_WINDOW_PARAMS,
status=status,
page=1,
page_size=10,
sort_desc=True,
session_cursor=None,
cap=100,
)
assert tuple(session.key for session in session_page.sessions) == expected
assert (session_page.total, session_page.total_is_capped) == (len(expected), False)

View file

@ -1401,6 +1401,109 @@ def _marks_the_route_sends(model: str, custom_llm_provider: str | None, messages
_SECOND_SEARCH_CALL: Final = {**_SEARCH_CALL, "id": "srvtoolu_02Search"}
_CLIENT_MARKED_CONVERSATION: Final = (
{"role": "system", "content": [{"type": "text", "text": "Handbook", "prompt_cache_breakpoint": {"mode": "explicit"}}]},
{"role": "user", "content": [{"type": "text", "text": "First", "prompt_cache_breakpoint": {"mode": "explicit"}}]},
{"role": "assistant", "content": "First answer"},
{"role": "user", "content": "Second"},
)
def _hook_result(
model: str, points: tuple[Mapping[str, object], ...] = ({"location": "message", "index": -1},), **params: object
) -> tuple[list[AllMessageValues], Mapping[str, object]]:
_, processed, returned_params = AnthropicCacheControlHook().get_chat_completion_prompt(
model=model,
messages=cast(list[AllMessageValues], copy.deepcopy(list(_CLIENT_MARKED_CONVERSATION))),
non_default_params={"cache_control_injection_points": list(points), **params},
prompt_id=None,
prompt_variables=None,
dynamic_callback_params={},
)
return processed, returned_params
def _hook_output(
model: str, points: tuple[Mapping[str, object], ...] = ({"location": "message", "index": -1},), **params: object
) -> list[AllMessageValues]:
return _hook_result(model, points, **params)[0]
def test_cache_control_injection_points_replace_client_prompt_cache_breakpoints_on_claude():
request: Final = AnthropicConfig().transform_request(
model="claude-sonnet-5-5",
messages=_hook_output("anthropic/claude-sonnet-5-5"),
optional_params={"max_tokens": 64},
litellm_params={},
headers={},
)
assert _marks_in(request["system"]) == 0
assert [_marks_in(message) for message in request["messages"]] == [0, 0, 1]
assert "prompt_cache_breakpoint" not in json.dumps(request)
@pytest.mark.parametrize(
"sibling_model_infos",
[
pytest.param((), id="one-deployment"),
pytest.param(
({"supports_prompt_cache_breakpoint": True, "base_model": "anthropic.claude-unmapped-v1:0"},),
id="beside-a-deployment-of-the-same-profile-with-a-base-model",
),
],
)
@pytest.mark.parametrize("route", ["bedrock", "bedrock/converse"])
def test_cache_control_injection_points_replace_client_prompt_cache_breakpoints_on_a_flagged_bedrock_profile(
local_model_cost_map, sibling_model_infos: tuple[Mapping[str, object], ...], route: str
):
from litellm import Router
model: Final = f"{route}/arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-flagged"
Router(
model_list=[
{
"model_name": f"claude-profile-{position}",
"litellm_params": {"model": model, "aws_region_name": "us-east-1"},
"model_info": model_info,
}
for position, model_info in enumerate(({"supports_prompt_cache_breakpoint": True}, *sibling_model_infos))
]
)
processed, params = _hook_result(model)
assert "prompt_cache_breakpoint" not in json.dumps(processed)
assert "prompt_cache_options" not in params
assert [_marks_in(message) for message in processed] == [0, 0, 0, 1]
def test_tool_config_injection_point_leaves_client_prompt_cache_breakpoints_on_claude():
processed: Final = _hook_output("anthropic/claude-sonnet-5-5", points=({"location": "tool_config"},))
assert [message["content"] for message in processed[:2]] == [
message["content"] for message in _CLIENT_MARKED_CONVERSATION[:2]
]
@pytest.mark.parametrize(
("model", "params"),
[
pytest.param("openai/gpt-5.6", {"api_base": "https://gateway.example.com/v1"}, id="reads-them"),
pytest.param("openai/gpt-4o", {}, id="openai-model-that-predates-them"),
pytest.param("hosted_vllm/qwen3", {"api_base": "http://vllm.example.com/v1"}, id="self-hosted"),
pytest.param("azure_ai/qa-unflagged-gpt", {"api_base": "https://qa.services.ai.azure.com/models"}, id="foundry"),
],
)
def test_cache_control_injection_points_keep_client_prompt_cache_breakpoints_on_a_route_that_does_not_translate_them(
model: str, params: Mapping[str, object]
):
processed: Final = _hook_output(model, **params)
assert [message["content"] for message in processed[:2]] == [
message["content"] for message in _CLIENT_MARKED_CONVERSATION[:2]
]
@pytest.mark.parametrize(
("model", "custom_llm_provider"),
[
@ -4965,6 +5068,33 @@ class TestMalformedInjectionPointsAreIgnored:
assert kwargs == {"cache_control_injection_points": value}
def test_capping_thousands_of_marks_on_one_tool_result_counts_the_request_a_logarithmic_number_of_times():
def marked_parts(prefix: str, count: int) -> list[dict]:
return [
{"type": "text", "text": f"{prefix} {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}
for index in range(count)
]
messages: Final = [
{"role": "user", "content": marked_parts("part", 4)},
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
},
{"role": "tool", "tool_call_id": "call_1", "content": marked_parts("result", 2000)},
]
sent_breakpoints: Final = MagicMock(side_effect=AnthropicCacheControlHook.count_converse_cache_points)
capped: Final = AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control(
cast(List[AllMessageValues], messages), True, sent_breakpoints, lambda: 0
)
assert [part.get("cache_control") for part in capped[0]["content"]] == [{"type": "ephemeral"}] * 3 + [None]
assert ["cache_control" in part for part in capped[2]["content"]] == [False] * 1997 + [True] * 3
assert sent_breakpoints.call_count < 3 * 20
def test_cache_control_hook_leaves_the_duplicate_tool_result_warning_to_the_request_builder(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
):

View file

@ -1,4 +1,5 @@
import copy
import itertools
import json
import logging
from collections.abc import Iterator
@ -4422,6 +4423,184 @@ def test_transform_request_includes_cache_control():
assert result["cache_control"] == {"type": "ephemeral"}
def _anthropic_body(messages: list, litellm_params: dict | None = None, tools: list | None = None) -> dict:
return AnthropicConfig().transform_request(
model="claude-sonnet-5-5",
messages=messages,
optional_params={"max_tokens": 100, **({"tools": tools} if tools else {})},
litellm_params=litellm_params or {},
headers={},
)
def test_transform_request_carries_prompt_cache_breakpoint_to_cache_control():
body: Final = _anthropic_body(
[
{
"role": "system",
"content": [
{"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}}
],
},
{
"role": "user",
"content": [
{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit"}},
{
"type": "text",
"text": "Bye",
"cache_control": {"type": "ephemeral", "ttl": "1h"},
"prompt_cache_breakpoint": {"mode": "explicit"},
},
],
},
]
)
assert body["system"] == [{"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}]
assert body["messages"] == [
{
"role": "user",
"content": [
{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}},
{"type": "text", "text": "Bye", "cache_control": {"type": "ephemeral", "ttl": "1h"}},
],
}
]
@pytest.mark.parametrize(
"marker, expected_block",
[
({"mode": "explicit", "ttl": "30m"}, {"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}),
({"mode": "implicit"}, {"type": "text", "text": "Hello"}),
],
)
def test_transform_request_prompt_cache_breakpoint_without_claude_equivalent(marker, expected_block):
messages: Final = [{"role": "user", "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": marker}]}]
with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"):
_anthropic_body(copy.deepcopy(messages))
body: Final = _anthropic_body(messages, litellm_params={"drop_params": True})
assert body["messages"] == [{"role": "user", "content": [expected_block]}]
def test_transform_request_names_the_marker_ttl_rule_for_a_ttl_only_claude_takes():
messages: Final = [
{"role": "user", "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "1h"}}]}
]
with pytest.raises(litellm.utils.UnsupportedParamsError, match="optional ttl of '30m' or '12h'"):
_anthropic_body(messages)
def test_transform_request_keeps_the_first_and_last_prompt_cache_breakpoints_within_the_cap():
client_marked_part: Final = {"type": "text", "text": "part 0", "cache_control": {"type": "ephemeral"}}
marked_parts: Final = [
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}
for index in range(1, 5)
]
messages: Final = [{"role": "user", "content": [client_marked_part, *marked_parts]}]
tools: Final = [{"name": "lookup", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}}]
with pytest.raises(litellm.utils.UnsupportedParamsError, match="at most 4 cache breakpoints"):
_anthropic_body(copy.deepcopy(messages), tools=copy.deepcopy(tools))
body: Final = _anthropic_body(messages, litellm_params={"drop_params": True}, tools=tools)
assert [part.get("cache_control") for part in body["messages"][0]["content"]] == [
{"type": "ephemeral"},
{"type": "ephemeral"},
None,
None,
{"type": "ephemeral"},
]
assert body["tools"][0]["cache_control"] == {"type": "ephemeral"}
def test_transform_request_counts_a_cache_marked_tool_search_tool_toward_the_cap():
config: Final = AnthropicConfig()
optional_params: Final = config.map_openai_params(
non_default_params={
"max_tokens": 100,
"tools": [
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex",
"cache_control": {"type": "ephemeral"},
}
],
},
optional_params={},
model="claude-sonnet-5-5",
drop_params=False,
)
marked_turns: Final = [
{
"role": "user",
"content": [{"type": "text", "text": f"turn {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}],
}
for index in range(4)
]
def body(litellm_params: dict) -> dict:
return config.transform_request(
model="claude-sonnet-5-5",
messages=copy.deepcopy(marked_turns),
optional_params=copy.deepcopy(optional_params),
litellm_params=litellm_params,
headers={},
)
with pytest.raises(litellm.utils.UnsupportedParamsError, match="this request has 5"):
body({})
capped: Final = body({"drop_params": True})
assert capped["tools"][0]["cache_control"] == {"type": "ephemeral"}
sent_parts: Final = itertools.chain.from_iterable(message["content"] for message in capped["messages"])
assert [part.get("cache_control") for part in sent_parts] == [
{"type": "ephemeral"},
{"type": "ephemeral"},
None,
{"type": "ephemeral"},
]
@pytest.mark.parametrize(
"unsent_marks",
[
pytest.param(
[
{
"role": "assistant",
"content": [{"type": "text", "text": "Noted.", "prompt_cache_breakpoint": {"mode": "explicit"}}],
}
],
id="assistant_part",
),
pytest.param(
[{"role": "user", "content": "Also this.", "prompt_cache_breakpoint": {"mode": "explicit"}}],
id="message_level",
),
],
)
def test_transform_request_caps_only_the_marks_claude_receives(unsent_marks: list[dict]):
marked_parts: Final = [
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} for index in range(4)
]
body: Final = _anthropic_body(
[{"role": "user", "content": marked_parts}, *unsent_marks, {"role": "user", "content": "Go on."}]
)
sent_marks: Final = [
part.get("cache_control")
for part in itertools.chain.from_iterable(message["content"] for message in body["messages"])
if isinstance(part, dict)
]
assert sent_marks.count({"type": "ephemeral"}) == 4
def test_compaction_block_empty_list_not_added():
"""
Test that empty compaction_blocks list is not added to provider_specific_fields.

View file

@ -22,6 +22,7 @@ TOKEN_PRICED_NAMES: Final = (
"cohere-command-a",
"grok-4-20-reasoning",
"grok-4-20-non-reasoning",
"claude-mythos-5-1",
)
GROK_4_20_NAMES: Final = ("grok-4-20-reasoning", "grok-4-20-non-reasoning")
CATALOG_NAMES: Final = TOKEN_PRICED_NAMES + ("whisper",)

View file

@ -10073,6 +10073,461 @@ def test_bedrock_prompt_caching_message(messages, expected_cache_control):
assert "cachePoint" not in json.dumps(transformed_messages)
def test_converse_carries_prompt_cache_breakpoint_to_cache_point():
request: Final = litellm.AmazonConverseConfig()._transform_request(
model="anthropic.claude-sonnet-5-5",
messages=[
{
"role": "system",
"content": [
{"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}}
],
},
{
"role": "user",
"content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit"}}],
},
],
optional_params={},
litellm_params={},
)
assert request["system"] == [{"text": "You are terse."}, {"cachePoint": {"type": "default"}}]
assert request["messages"] == [
{"role": "user", "content": [{"text": "Hello"}, {"cachePoint": {"type": "default"}}]}
]
def test_converse_keeps_the_first_and_last_prompt_cache_breakpoints_within_the_cap():
messages: Final = [
{
"role": "user",
"content": [
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}
for index in range(5)
],
}
]
def transform(litellm_params: dict) -> dict:
return litellm.AmazonConverseConfig()._transform_request(
model="anthropic.claude-sonnet-5-5",
messages=copy.deepcopy(messages),
optional_params={},
litellm_params=litellm_params,
)
with pytest.raises(litellm.utils.UnsupportedParamsError, match="at most 4 cache breakpoints"):
transform({})
cache_point: Final = {"cachePoint": {"type": "default"}}
assert transform({"drop_params": True})["messages"] == [
{
"role": "user",
"content": [
{"text": "part 0"},
cache_point,
{"text": "part 1"},
cache_point,
{"text": "part 2"},
{"text": "part 3"},
cache_point,
{"text": "part 4"},
cache_point,
],
}
]
_CONVERSE_CAP_TOOL: Final[dict[str, object]] = {
"type": "function",
"function": {"name": "lookup", "parameters": {"type": "object", "properties": {}}},
}
@pytest.mark.parametrize(
"tool_cache_params",
[
{"tools": [{**_CONVERSE_CAP_TOOL, "cache_control": {"type": "ephemeral"}}]},
{
"tools": [_CONVERSE_CAP_TOOL],
"cache_control_injection_points": [{"location": "tool_config", "control": {"type": "ephemeral"}}],
},
],
ids=["client_tool_cache_control", "tool_config_injection_point"],
)
def test_converse_reserves_the_tool_cache_point_within_the_cap(tool_cache_params: dict[str, object]):
messages: Final = [
{
"role": "user",
"content": [
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}
for index in range(4)
],
}
]
def transform(litellm_params: dict) -> dict:
return litellm.AmazonConverseConfig()._transform_request(
model="anthropic.claude-sonnet-5-5",
messages=copy.deepcopy(messages),
optional_params=copy.deepcopy(tool_cache_params),
litellm_params=litellm_params,
)
with pytest.raises(litellm.utils.UnsupportedParamsError, match="this request has 5"):
transform({})
request: Final = transform({"drop_params": True})
cache_point: Final = {"cachePoint": {"type": "default"}}
assert request["messages"][0]["content"] == [
{"text": "part 0"},
cache_point,
{"text": "part 1"},
cache_point,
{"text": "part 2"},
{"text": "part 3"},
cache_point,
]
assert request["toolConfig"]["tools"][-1] == cache_point
@pytest.mark.parametrize(
"unsent_tool_params",
[
{
"tools": [
_CONVERSE_CAP_TOOL,
{"type": "tool_search_tool_regex_20251119", "name": "find", "cache_control": {"type": "ephemeral"}},
]
},
{"tools": [_CONVERSE_CAP_TOOL, {"type": "web_search", "cache_control": {"type": "ephemeral"}}]},
{
"tools": [{"type": "web_search"}],
"cache_control_injection_points": [{"location": "tool_config", "control": {"type": "ephemeral"}}],
},
],
ids=["tool_search_tool", "built_in_tool", "tool_config_point_with_no_tool_sent"],
)
@pytest.mark.parametrize("litellm_params", [{}, {"drop_params": True}], ids=["strict", "drop_params"])
def test_converse_leaves_the_cap_to_markers_when_a_marked_tool_is_not_sent(
unsent_tool_params: dict[str, object], litellm_params: dict[str, object]
):
request: Final = litellm.AmazonConverseConfig()._transform_request(
model="anthropic.claude-sonnet-5-5",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}
for index in range(4)
],
}
],
optional_params=copy.deepcopy(unsent_tool_params),
litellm_params=dict(litellm_params),
)
cache_point: Final = {"cachePoint": {"type": "default"}}
assert request["messages"][0]["content"] == [
{"text": "part 0"},
cache_point,
{"text": "part 1"},
cache_point,
{"text": "part 2"},
cache_point,
{"text": "part 3"},
cache_point,
]
assert cache_point not in request.get("toolConfig", {}).get("tools", [])
def _converse_tool_turn(*result_parts: dict[str, object]) -> list[dict[str, object]]:
return [
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
},
{"role": "tool", "tool_call_id": "call_1", "content": list(result_parts)},
]
def _marked_text_parts(count: int) -> list[dict[str, object]]:
return [
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}
for index in range(count)
]
def _marked_tool_result_parts() -> list[dict[str, object]]:
return [
{"type": "text", "text": f"result {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}
for index in range(2)
]
def test_converse_counts_one_cache_point_for_a_tool_result_with_several_marks():
request: Final = litellm.AmazonConverseConfig()._transform_request(
model="anthropic.claude-sonnet-5-5",
messages=[
{"role": "user", "content": _marked_text_parts(3)},
*_converse_tool_turn(*_marked_tool_result_parts()),
],
optional_params={"tools": [_CONVERSE_CAP_TOOL]},
litellm_params={},
)
assert len(_collect_cache_points(request["messages"])) == 4
def test_converse_drops_marks_until_a_merged_tool_result_fits_the_cap():
messages: Final = [
{"role": "user", "content": _marked_text_parts(4)},
*_converse_tool_turn(*_marked_tool_result_parts()),
]
def transform(litellm_params: dict) -> dict:
return litellm.AmazonConverseConfig()._transform_request(
model="anthropic.claude-sonnet-5-5",
messages=copy.deepcopy(messages),
optional_params={"tools": [_CONVERSE_CAP_TOOL]},
litellm_params=litellm_params,
)
with pytest.raises(litellm.UnsupportedParamsError, match="this request has 5"):
transform({})
request: Final = transform({"drop_params": True})
cache_point: Final = {"cachePoint": {"type": "default"}}
assert request["messages"][0]["content"] == [
{"text": "part 0"},
cache_point,
{"text": "part 1"},
cache_point,
{"text": "part 2"},
cache_point,
{"text": "part 3"},
]
assert len(_collect_cache_points(request["messages"][1:])) == 1
def test_converse_drops_prompt_cache_breakpoint_from_a_nova_tool_result():
messages: Final = [
{
"role": "system",
"content": [{"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}}],
},
{"role": "user", "content": "Look it up."},
*_converse_tool_turn(*_marked_tool_result_parts()),
]
def transform(litellm_params: dict) -> dict:
return litellm.AmazonConverseConfig()._transform_request(
model="us.amazon.nova-lite-v1:0",
messages=copy.deepcopy(messages),
optional_params={"tools": [_CONVERSE_CAP_TOOL]},
litellm_params=litellm_params,
)
with pytest.raises(litellm.UnsupportedParamsError, match="tool result"):
transform({})
request: Final = transform({"drop_params": True})
assert request["system"] == [{"text": "You are terse."}, {"cachePoint": {"type": "default"}}]
assert _collect_cache_points(request["messages"]) == []
def test_converse_translates_prompt_cache_breakpoint_on_an_unmapped_arn_only_when_flagged(local_model_cost_map):
from litellm import Router
flagged_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-flagged"
unflagged_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/mistral"
Router(
model_list=[
{
"model_name": "claude-profile",
"litellm_params": {"model": f"bedrock/{flagged_arn}", "aws_region_name": "us-east-1"},
"model_info": {"supports_prompt_cache_breakpoint": True},
},
{
"model_name": "mistral-profile",
"litellm_params": {"model": f"bedrock/{unflagged_arn}", "aws_region_name": "us-east-1"},
},
]
)
def user_content(model: str, ttl: dict) -> list:
return litellm.AmazonConverseConfig()._transform_request(
model=model,
messages=[
{
"role": "user",
"content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}}],
}
],
optional_params={},
litellm_params={},
)["messages"][0]["content"]
assert user_content(flagged_arn, {}) == [{"text": "Hello"}, {"cachePoint": {"type": "default"}}]
assert user_content(unflagged_arn, {}) == [{"text": "Hello"}]
assert user_content(unflagged_arn, {"ttl": "30m"}) == [{"text": "Hello"}]
def test_converse_prompt_cache_breakpoint_follows_the_arn_flag_over_its_base_model(local_model_cost_map):
from litellm import Router
opted_in_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-next"
opted_out_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-opted-out"
Router(
model_list=[
{
"model_name": "opted-in",
"litellm_params": {"model": f"bedrock/{opted_in_arn}", "aws_region_name": "us-east-1"},
"model_info": {"supports_prompt_cache_breakpoint": True},
},
{
"model_name": "opted-out",
"litellm_params": {"model": f"bedrock/{opted_out_arn}", "aws_region_name": "us-east-1"},
"model_info": {"supports_prompt_cache_breakpoint": False},
},
]
)
def user_content(model: str, base_model: str, ttl: dict) -> list:
return litellm.AmazonConverseConfig()._transform_request(
model=model,
messages=[
{
"role": "user",
"content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}}],
}
],
optional_params={},
litellm_params={"base_model": base_model},
)["messages"][0]["content"]
assert user_content(opted_in_arn, "anthropic.claude-unmapped-v1:0", {}) == [
{"text": "Hello"},
{"cachePoint": {"type": "default"}},
]
assert user_content(opted_out_arn, "anthropic.claude-sonnet-5-5", {"ttl": "30m"}) == [{"text": "Hello"}]
def test_converse_translates_prompt_cache_breakpoint_only_for_families_that_take_cache_points(local_model_cost_map):
litellm.register_model(
{"zai.glm-next-v1:0": {"litellm_provider": "bedrock_converse", "mode": "chat", "supports_prompt_caching": True}}
)
def user_content(model: str, mark: dict) -> list:
return litellm.AmazonConverseConfig()._transform_request(
model=model,
messages=[{"role": "user", "content": [{"type": "text", "text": "Hello", **mark}]}],
optional_params={},
litellm_params={},
)["messages"][0]["content"]
marker: Final = {"prompt_cache_breakpoint": {"mode": "explicit"}}
client_mark: Final = {"cache_control": {"type": "ephemeral"}}
cache_point: Final = {"cachePoint": {"type": "default"}}
assert user_content("zai.glm-next-v1:0", marker) == [{"text": "Hello"}]
assert user_content("zai.glm-next-v1:0", client_mark) == [{"text": "Hello"}, cache_point]
assert user_content("us.anthropic.claude-next-v1:0", marker) == [{"text": "Hello"}, cache_point]
assert user_content("us.amazon.nova-next-v1:0", marker) == [{"text": "Hello"}, cache_point]
def test_converse_prompt_cache_breakpoint_on_guarded_user_text_keeps_its_cache_point(local_model_cost_map):
guardrail: Final = {"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "DRAFT"}}
def request(system_marks: int) -> dict:
return litellm.AmazonConverseConfig()._transform_request(
model="anthropic.claude-sonnet-5-5",
messages=[
*(
{
"role": "system",
"content": [{"type": "text", "text": f"Rule {i}", "cache_control": {"type": "ephemeral"}}],
}
for i in range(system_marks)
),
{
"role": "user",
"content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit"}}],
},
],
optional_params=dict(guardrail),
litellm_params={},
)
assert request(0)["messages"][0]["content"] == [
{"guardContent": {"text": {"text": "Hello"}}},
{"cachePoint": {"type": "default"}},
]
with pytest.raises(litellm.UnsupportedParamsError):
request(4)
client_marked: Final = litellm.AmazonConverseConfig()._transform_request(
model="anthropic.claude-sonnet-5-5",
messages=[
*(
{
"role": "system",
"content": [{"type": "text", "text": f"Rule {i}", "cache_control": {"type": "ephemeral"}}],
}
for i in range(4)
),
{"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}]},
],
optional_params=dict(guardrail),
litellm_params={},
)
assert sum("cachePoint" in block for block in client_marked["system"]) == 4
assert client_marked["messages"][0]["content"] == [{"guardContent": {"text": {"text": "Hello"}}}]
def test_converse_guarded_user_text_keeps_no_cache_point_on_a_model_not_known_to_take_one(local_model_cost_map):
request: Final = litellm.AmazonConverseConfig()._transform_request(
model="arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/unmapped1",
messages=[{"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}]}],
optional_params={"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "DRAFT"}},
litellm_params={},
)
assert request["messages"][0]["content"] == [{"guardContent": {"text": {"text": "Hello"}}}]
def _converse_request_with_30m_breakpoint(model: str, litellm_params: dict) -> dict:
return litellm.AmazonConverseConfig()._transform_request(
model=model,
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}}
],
}
],
optional_params={},
litellm_params=litellm_params,
)
def test_converse_rejects_30m_breakpoint_ttl_only_where_a_cache_point_is_written():
claude: Final = "anthropic.claude-sonnet-5-5"
with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"):
_converse_request_with_30m_breakpoint(claude, {})
assert _converse_request_with_30m_breakpoint(claude, {"drop_params": True})["messages"] == [
{"role": "user", "content": [{"text": "Hello"}, {"cachePoint": {"type": "default"}}]}
]
assert _converse_request_with_30m_breakpoint("meta.llama3-1-8b-instruct-v1:0", {})["messages"] == [
{"role": "user", "content": [{"text": "Hello"}]}
]
@pytest.mark.parametrize(
"model, expected_supports_tool_call",
[

View file

@ -16,7 +16,7 @@ from fastapi.testclient import TestClient
from collections.abc import Callable
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Final, Optional
from typing import Any, Dict, Final, Literal, Optional
from unittest.mock import MagicMock, Mock, patch
from botocore.awsrequest import AWSPreparedRequest, AWSRequest
@ -2744,6 +2744,69 @@ def test_is_already_running_as_role_irsa_different_role():
)
@pytest.mark.parametrize("auth_flow", ("resolve", "access_denied"))
@pytest.mark.parametrize(
("env_region", "configured_region", "endpoint", "expected_region"),
(
("us-east-1", None, None, "us-east-1"),
(None, "us-west-2", None, "us-west-2"),
("us-east-1", "us-west-2", "https://sts.us-west-2.amazonaws.com", "us-west-2"),
),
)
def test_role_identity_uses_configured_sts_endpoint(
monkeypatch: pytest.MonkeyPatch,
auth_flow: Literal["resolve", "access_denied"],
env_region: str | None,
configured_region: str | None,
endpoint: str | None,
expected_region: str,
) -> None:
for name in ("AWS_ROLE_ARN", "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_REGION", "AWS_DEFAULT_REGION", "AWS_PROFILE"):
monkeypatch.delenv(name, raising=False)
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-access-key")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret-key")
if env_region is not None:
monkeypatch.setenv("AWS_REGION", env_region)
sts: Final = MagicMock()
sts.get_caller_identity.return_value = {
"Arn": "arn:aws:sts::123456789012:assumed-role/TargetRole/test-session"
}
sts.assume_role.side_effect = ClientError({"Error": {"Code": "AccessDenied"}}, "AssumeRole")
auth: Final = BaseAWSLLM()
with patch("boto3.client", return_value=sts) as create_client:
if auth_flow == "resolve":
credentials, _ = auth._resolve_role_credentials(
aws_access_key_id=None,
aws_secret_access_key=None,
aws_session_token=None,
aws_role_name="arn:aws:iam::123456789012:role/TargetRole",
aws_session_name="test-session",
aws_region_name=configured_region,
aws_sts_endpoint=endpoint,
aws_external_id=None,
aws_session_tags=None,
ssl_verify=True,
)
else:
credentials, _ = auth._auth_with_aws_role(
aws_access_key_id=None,
aws_secret_access_key=None,
aws_session_token=None,
aws_role_name="arn:aws:iam::123456789012:role/TargetRole",
aws_session_name="test-session",
aws_region_name=configured_region,
aws_sts_endpoint=endpoint,
ssl_verify=True,
)
assert credentials.access_key == "test-access-key"
sts.get_caller_identity.assert_called_once_with()
for request in create_client.call_args_list:
assert request.args == ("sts",)
assert request.kwargs["region_name"] == expected_region
assert request.kwargs.get("endpoint_url") == endpoint
assert request.kwargs["verify"] is True
def test_is_already_running_as_role_ecs_task_role():
"""Test ECS/EC2 path: GetCallerIdentity shows assumed-role matching target."""
base_aws_llm = BaseAWSLLM()

View file

@ -14,6 +14,7 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.opencode.chat.transformation import OpenCodeGoMessagesChatConfig
from litellm.llms.opencode.common_utils import go_headers
@ -591,3 +592,22 @@ def test_stream_chunk_builder_reads_list_cost_from_dict_chunks():
assert assembled is not None
assert assembled._hidden_params["response_cost"] == GO_REPORTED_USAGE["opencode"]["list_cost"]
@pytest.mark.parametrize(("model", "cache_marks"), [("qwen3.8-flash", 0), ("minimax-m3", 0), ("claude-haiku-5-5", 4)])
def test_go_messages_route_carries_prompt_cache_breakpoint_to_cache_control_only_for_claude(model: str, cache_marks: int):
marked_notes: Final = [
{"type": "text", "text": f"Note {n}", "prompt_cache_breakpoint": {"mode": "explicit"}} for n in range(4)
]
request: Final = OpenCodeGoMessagesChatConfig().transform_request(
model=model,
messages=[{"role": "user", "content": [*marked_notes, {"type": "text", "text": "Answer in one line."}]}],
optional_params={"max_tokens": 64},
litellm_params={},
headers={},
)
wire: Final = json.dumps(request)
assert wire.count('"cache_control"') == cache_marks
assert "prompt_cache_breakpoint" not in wire

View file

@ -19,6 +19,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import NameOID
from litellm import Router
from litellm.utils import AZURE_MODEL_PLACEHOLDER
def test_get_team_models_for_all_models_and_team_only_models():
@ -1105,13 +1106,13 @@ def test_expand_wildcard_invalid_litellm_params_passthrough():
@pytest.mark.parametrize(
("allowed_model_names", "expected_names", "expected_listings"),
[
pytest.param(None, ["hosted_vllm/qwen2.5"], 1, id="unscoped"),
pytest.param(["hosted_vllm/qwen2.5"], ["hosted_vllm/qwen2.5"], 1, id="granted-a-listed-id"),
pytest.param(["qwen-open"], [], 0, id="granted-a-name-the-alias-matches"),
pytest.param(None, ["qwen-qwen2.5"], 1, id="unscoped"),
pytest.param(["qwen-qwen2.5"], ["qwen-qwen2.5"], 1, id="granted-a-listed-name"),
pytest.param(["hosted_vllm/qwen2.5"], [], 0, id="granted-only-the-provider-id"),
],
)
@respx.mock
def test_expand_wildcard_alias_without_a_slash_lists_and_filters_by_its_provider_pattern(
def test_expand_wildcard_alias_without_a_slash_lists_and_filters_by_its_own_pattern(
monkeypatch: pytest.MonkeyPatch,
request: pytest.FixtureRequest,
allowed_model_names: list[str] | None,
@ -2099,6 +2100,34 @@ _ROUTE_CF_IDS: Final = {_ROUTE_CF_ID, _ROUTE_CF_SUFFIXED_ID}
pytest.param(
"*", "openai/*", "openai/gpt-partial-route-9*", set(), id="provider-prefixed-key-filter-over-catch-all"
),
pytest.param(
"gpt-partial-route-*",
"openai/gpt-partial-route-*",
None,
{f"openai/{_ROUTE_OPENAI_ID}"},
id="slashless-name-repeating-the-deployment-partial-word",
),
pytest.param("team-*", "cloudflare/@cf/*", None, _ROUTE_CF_IDS, id="slashless-alias-over-partial-deployment"),
pytest.param(
"team-*", "openai/*", None, {f"openai/{_ROUTE_OPENAI_ID}"}, id="slashless-alias-over-provider-catch-all"
),
pytest.param(
"team-*", "cloudflare/@cf/*/*", None, set(), id="slashless-alias-with-fewer-wildcards-than-its-deployment"
),
pytest.param(
"*partial-route-9*",
"openai/gpt-partial-route-9*",
None,
{f"openai/{_ROUTE_OPENAI_ID}"},
id="slashless-name-with-more-wildcards-than-its-deployment",
),
pytest.param(
"gemini/*partial-route-9*",
"gemini/gemini-*",
None,
{_ROUTE_GEMINI_ID},
id="slashed-name-with-more-wildcards-than-its-deployment",
),
pytest.param("*", "cloudflare/@cf/*", None, _ROUTE_CF_IDS, id="catch-all-over-partial-deployment"),
pytest.param("*", "gemini/gemini-partial-route-9*", None, set(), id="catch-all-over-mid-word-deployment"),
pytest.param("*", "openai/responses/gpt-partial-route-9*", None, set(), id="catch-all-over-mid-word-route"),
@ -2176,6 +2205,114 @@ def test_every_listed_partial_wildcard_id_routes_to_the_catalog_model_it_names(
assert listed_model_info_names == sorted(reachable)
@pytest.mark.parametrize(
("model_name", "expected_v1_models"),
[
pytest.param("azure/*", [f"azure/{AZURE_MODEL_PLACEHOLDER}"], id="provider-wildcard-lists-as-before"),
pytest.param("*", [AZURE_MODEL_PLACEHOLDER], id="catch-all-lists-as-before"),
pytest.param("my-*", [], id="slashless-alias-skips-the-placeholder"),
],
)
def test_only_slashless_azure_wildcards_skip_the_catalog_placeholder(
model_name: str, expected_v1_models: list[str]
) -> None:
from litellm import Router
from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list
router: Final = Router(model_list=[{"model_name": model_name, "litellm_params": {"model": "azure/*"}}])
v1_models: Final = get_complete_model_list(
key_models=[],
team_models=[],
proxy_model_list=[model_name],
user_model=None,
infer_model_from_keys=False,
llm_router=router,
)
model_info_names: Final = [
str(row["model_name"])
for row in expand_wildcard_deployments_for_model_info(
[dict(deployment) for deployment in router.get_model_list() or []], router
)
]
assert v1_models == expected_v1_models
assert model_info_names == (expected_v1_models or [model_name])
@pytest.mark.parametrize(
("model_name", "deployment_model"),
[
pytest.param("gpt-*", "gpt-*", id="same-wildcards-as-its-deployment"),
pytest.param("*gpt-5*", "gpt-5*", id="more-wildcards-than-its-deployment"),
],
)
def test_slashless_wildcard_over_a_bare_deployment_lists_its_custom_llm_provider_catalog(
model_name: str, deployment_model: str
) -> None:
from fnmatch import fnmatchcase
from litellm import Router
from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list
from litellm.utils import get_valid_models
router: Final = Router(
model_list=[
{
"model_name": model_name,
"litellm_params": {"model": deployment_model, "custom_llm_provider": "openai", "api_key": "sk-test"},
}
]
)
v1_models: Final = get_complete_model_list(
key_models=[],
team_models=[],
proxy_model_list=[model_name],
user_model=None,
infer_model_from_keys=False,
llm_router=router,
)
model_info_rows: Final = expand_wildcard_deployments_for_model_info(
[dict(deployment) for deployment in router.get_model_list() or []], router
)
openai_ids: Final = frozenset(model.removeprefix("openai/") for model in get_valid_models(custom_llm_provider="openai"))
routed_models: Final = {
listed_id: [deployment["litellm_params"]["model"] for deployment in router.wildcard_route_deployments(listed_id, None)]
for listed_id in v1_models
}
assert v1_models
assert [
listed_id for listed_id in v1_models if listed_id not in openai_ids or not fnmatchcase(listed_id, model_name)
] == []
assert {listed_id: models for listed_id, models in routed_models.items() if models != [listed_id]} == {}
assert sorted(str(row["model_name"]) for row in model_info_rows) == sorted(v1_models)
assert [row for row in model_info_rows if row["litellm_params"]["model"] != row["model_name"]] == []
@pytest.mark.parametrize(
("model_name", "deployment_model"),
[
pytest.param("*", "*", id="catch-all-over-a-bare-catch-all"),
pytest.param("foo/gpt-*", "gpt-*", id="slashed-name-over-a-bare-deployment"),
pytest.param("gpt-*", "openai/gpt-*", id="slashless-name-over-a-provider-prefixed-deployment"),
],
)
def test_custom_llm_provider_changes_no_listing_outside_slashless_names_over_bare_deployments(
model_name: str, deployment_model: str
) -> None:
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.types.router import LiteLLM_Params
with_provider: Final = get_known_models_from_wildcard(
model_name, LiteLLM_Params(model=deployment_model, custom_llm_provider="openai")
)
without_provider: Final = get_known_models_from_wildcard(model_name, LiteLLM_Params(model=deployment_model))
assert with_provider == without_provider
def test_inference_provider_route_lists_every_id_the_plain_provider_wildcard_lists() -> None:
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.types.router import LiteLLM_Params

View file

@ -1,8 +1,9 @@
import asyncio
from datetime import datetime, timezone
from typing import Final
from typing import Final, Literal
import pytest
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
@ -450,18 +451,17 @@ async def test_completed_call_keeps_its_charge_when_post_call_processing_fails(
disable_spend_updates: bool,
) -> None:
from unittest.mock import create_autospec
import anyio
from fastapi import Request, Response
from litellm.proxy._types import ProxyException
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy import proxy_server
from litellm.proxy._types import ProxyException
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.proxy_server import ProxyConfig
from litellm.proxy import proxy_server
from litellm.proxy.utils import ProxyLogging
scope: Final = anyio.CancelScope()
@ -598,6 +598,7 @@ async def test_guardrail_json_rejection_retains_completed_provider_usage(
logging_obj: Logging, monkeypatch: pytest.MonkeyPatch
) -> None:
import httpx
from litellm.caching.dual_cache import DualCache
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.proxy.utils import ProxyLogging
@ -633,3 +634,222 @@ async def test_guardrail_json_rejection_retains_completed_provider_usage(
assert (payload["prompt_tokens"], payload["completion_tokens"]) == (10, 5)
assert payload["response_cost"] == 0.5
ProxyLogging._callback_capabilities_cache.clear()
@pytest.mark.asyncio
@pytest.mark.parametrize("price", [None, 0.0, 0.02], ids=["unpriced", "free", "priced"])
@pytest.mark.parametrize("discount", [0.0, 1.0], ids=["no-discount", "fully-discounted"])
async def test_finalized_header_distinguishes_missing_prices_from_free_models(
monkeypatch: pytest.MonkeyPatch, price: float | None, discount: float
) -> None:
model: Final = "claude-header-pricing-test"
monkeypatch.setattr(litellm, "cost_discount_config", {"azure": discount})
monkeypatch.setitem(
litellm.model_cost,
f"azure/{model}",
{
"litellm_provider": "azure",
"mode": "chat",
"regional_endpoint_uplift_multiplier": 1.0,
**({"input_cost_per_token": price, "output_cost_per_token": price} if price is not None else {}),
},
)
logging: Final = Logging(
model=model,
messages=[],
stream=False,
call_type="acompletion",
start_time=NOW,
litellm_call_id="pricing-header",
function_id="pricing-header",
)
logging.update_environment_variables(
model=model, custom_llm_provider="azure", litellm_params={}, optional_params={}
)
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
finalized: Final = await finalize_response_cost(logging, result, None, NOW)
assert finalized is not None
assert finalized.amount == pytest.approx((price or 0.0) * 15 * (1 - discount))
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=UserAPIKeyAuth(),
response_cost=finalized.amount,
litellm_logging_obj=logging,
)
assert headers.get("x-litellm-response-cost") == (None if price is None else str(finalized.amount))
@pytest.mark.asyncio
@pytest.mark.parametrize(
"source",
["unpriced", "cache", "non-inference", "provider", "provider-header", "per-token", "per-second", "passthrough"],
)
@pytest.mark.parametrize("guardrail_cost", [0.0, 0.1])
async def test_zero_cost_headers_preserve_known_free_calls_and_guardrail_charges(
source: Literal[
"unpriced", "cache", "non-inference", "provider", "provider-header", "per-token", "per-second", "passthrough"
],
guardrail_cost: float,
) -> None:
model: Final = "unmapped-header-cost"
logging: Final = Logging(
model=model,
messages=[],
stream=False,
call_type="adelete_responses" if source == "non-inference" else "acompletion",
start_time=NOW,
litellm_call_id="zero-cost-header",
function_id="zero-cost-header",
)
logging.update_environment_variables(
model=model, custom_llm_provider="openai", litellm_params={}, optional_params={}
)
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
result._hidden_params = {"response_cost": 0.0, "cache_hit": source == "cache"}
match source:
case "provider":
logging._provider_reported_cost = 0.0
case "provider-header":
result._hidden_params["additional_headers"] = {"llm_provider-x-litellm-response-cost": 0.0}
case "per-token":
logging.custom_cost_per_token = {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}
case "per-second":
logging.custom_cost_per_second = 0.0
case "passthrough":
logging.model_call_details["passthrough_logging_payload"] = {"cost_per_request": 0.0}
case _:
pass
finalized: Final = await finalize_response_cost(logging, result, None, NOW, guardrail_cost=guardrail_cost)
assert finalized is not None
assert finalized.amount == guardrail_cost
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=UserAPIKeyAuth(),
response_cost=finalized.amount,
litellm_logging_obj=logging,
)
assert headers.get("x-litellm-response-cost") == (
None if source == "unpriced" and guardrail_cost == 0 else str(guardrail_cost)
)
@pytest.mark.asyncio
async def test_free_model_priced_only_per_request_keeps_the_zero_cost_header(monkeypatch: pytest.MonkeyPatch) -> None:
model: Final = "gpt-free-per-request-header"
monkeypatch.setitem(
litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat", "input_cost_per_request": 0.0}
)
logging: Final = Logging(
model=model,
messages=[],
stream=False,
call_type="acompletion",
start_time=NOW,
litellm_call_id="per-request-header",
function_id="per-request-header",
)
logging.update_environment_variables(
model=model, custom_llm_provider="openai", litellm_params={}, optional_params={}
)
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
finalized: Final = await finalize_response_cost(logging, result, None, NOW)
assert finalized is not None and finalized.amount == 0.0
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=UserAPIKeyAuth(),
response_cost=finalized.amount,
litellm_logging_obj=logging,
)
assert headers["x-litellm-response-cost"] == "0.0"
@pytest.mark.asyncio
async def test_free_off_peak_pricing_keeps_the_finalized_zero_cost_header(monkeypatch: pytest.MonkeyPatch) -> None:
model: Final = "gpt-free-off-peak-header"
monkeypatch.setitem(
litellm.model_cost,
model,
{
"litellm_provider": "openai",
"mode": "chat",
"off_peak_pricing": {"hours_utc": "00:00-23:59", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0},
},
)
logging: Final = Logging(
model=model,
messages=[],
stream=False,
call_type="acompletion",
start_time=NOW,
litellm_call_id="off-peak-header",
function_id="off-peak-header",
)
logging.update_environment_variables(
model=model, custom_llm_provider="openai", litellm_params={}, optional_params={}
)
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
result._hidden_params["response_cost"] = 0.0
finalized: Final = await finalize_response_cost(logging, result, None, NOW)
assert finalized is not None and finalized.amount == 0.0
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=UserAPIKeyAuth(),
response_cost=finalized.amount,
litellm_logging_obj=logging,
)
assert headers["x-litellm-response-cost"] == "0.0"
@pytest.mark.asyncio
@pytest.mark.parametrize("mapped", [False, True], ids=["unmapped-rule", "fill-missing-rule"])
@pytest.mark.parametrize("price", [None, 0.0, 0.02], ids=["unpriced", "free", "priced"])
async def test_rule_pricing_preserves_free_headers_without_treating_capabilities_as_prices(
monkeypatch: pytest.MonkeyPatch, mapped: bool, price: float | None
) -> None:
from litellm.litellm_core_utils.fallback_generalizations import (
get_fallback_generalization_rules,
set_fallback_generalizations,
)
model: Final = "gpt-rule-header-model"
previous_rules: Final = TypeAdapter(list[object]).validate_python(get_fallback_generalization_rules())
if mapped:
monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat"})
try:
set_fallback_generalizations(
[
{
"name": "header-pricing",
"pattern": model,
"fill_missing_for_providers": ["openai"],
"model_info": {
"mode": "chat",
**(
{"input_cost_per_token": price, "output_cost_per_token": price} if price is not None else {}
),
},
}
]
)
logging: Final = Logging(
model=model,
messages=[],
stream=False,
call_type="acompletion",
start_time=NOW,
litellm_call_id="rule-header",
function_id="rule-header",
)
logging.update_environment_variables(
model=model, custom_llm_provider="openai", litellm_params={}, optional_params={}
)
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
finalized: Final = await finalize_response_cost(logging, result, None, NOW)
assert finalized is not None
assert finalized.amount == pytest.approx((price or 0.0) * 15)
headers: Final = TypeAdapter(dict[str, str]).validate_python(
ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=UserAPIKeyAuth(),
response_cost=finalized.amount,
litellm_logging_obj=logging,
)
)
assert headers.get("x-litellm-response-cost") == (None if price is None else str(finalized.amount))
finally:
set_fallback_generalizations(previous_rules)

View file

@ -0,0 +1,327 @@
import itertools
import random
import re
from collections.abc import Sequence
from typing import Final, TypeAlias
import pytest
from litellm.proxy.spend_tracking.session_page import (
Session,
_parse_session_cursor,
_Read,
_ReadRow,
_Request,
_session_page,
read_session_page,
)
_Row: TypeAlias = tuple[int, str, str] # (startTime, session key, api_key)
_Ranked: TypeAlias = tuple[int, str, str] # (last activity, session key, api_key)
def _session_of(row: _Row) -> tuple[str, str]:
return row[1], row[2]
def _ranked(rows: Sequence[_Row], request: _Request) -> tuple[_Ranked, ...]:
"""The sessions in ``rows`` in page order, after the request's cursor or offset."""
by_session: Final = itertools.groupby(sorted(rows, key=_session_of), key=_session_of)
ranked: Final = tuple(
sorted(
((max(start for start, _, _ in group), key, api_key) for (key, api_key), group in by_session),
reverse=request.sort_desc,
)
)
if request.cursor is None:
return ranked[request.offset :]
cursor: Final = (int(request.cursor[0]), request.cursor[1], request.cursor[2])
return tuple(session for session in ranked if (session < cursor if request.sort_desc else session > cursor))
class _Reads:
"""Reads like the page query, whose LIMIT cuts tied rows lowest key first, the opposite of page order. It fails a
read no larger than the one before, since only growing reads end the loop."""
def __init__(self, rows: Sequence[_Row], request: _Request, every_row: bool = False) -> None:
self.newest_first: Final = tuple(sorted(rows, key=lambda row: (-row[0], row[1], row[2])))
self.request: Final = request
self.every_row: Final = every_row
self.budgets: tuple[int | None, ...] = ()
async def __call__(self, budget: int | None) -> _Read:
assert not self.budgets or (self.budgets[-1] is not None and (budget is None or budget > self.budgets[-1])), (
self.budgets,
budget,
)
self.budgets = (*self.budgets, budget)
taken: Final = self.newest_first if budget is None or self.every_row else self.newest_first[:budget]
page: Final = _ranked(taken, self.request)[: self.request.page_limit + 1] if self.request.page_limit > 0 else ()
return _Read(
candidates=tuple(Session(key, api_key, f"{at:06d}") for at, key, api_key in page),
last_settled=bool(page) and page[-1][0] > taken[-1][0],
rows_read=len(taken),
session_count=len(frozenset(_session_of(row) for row in taken)),
saw_every_row=budget is None or self.every_row or len(taken) < budget,
)
def _session_rows(rng: random.Random, index: int, clock: int) -> tuple[_Row, ...]:
start, length = rng.randrange(0, clock), rng.choice((1, 1, 2, 3, 8, 40))
return tuple((start + rng.randrange(0, 30), f"s{index:03d}", rng.choice(("k1", "k2"))) for _ in range(length))
def _sessions(rng: random.Random) -> tuple[_Row, ...]:
"""Sessions of one to forty rows over a short clock, so many rows share a time."""
clock: Final = rng.choice((3, 40, 400))
return tuple(
itertools.chain.from_iterable(_session_rows(rng, index, clock) for index in range(rng.randrange(1, 120)))
)
def _random_cursor(rng: random.Random, ranked: Sequence[_Ranked]) -> tuple[str, str, str]:
at, key, api_key = ranked[rng.randrange(len(ranked))]
return f"{at:06d}", key, api_key
def _request(rng: random.Random, rows: Sequence[_Row]) -> _Request:
cap, page_size, sort_desc = rng.randrange(2, 60), rng.randrange(1, 12), rng.random() < 0.8
mode: Final = rng.choice(("first", "offset", "cursor"))
full: Final = _ranked(rows, _Request(None, 0, 0, sort_desc, None, cap))
cursor: Final = _random_cursor(rng, full) if mode == "cursor" and full else None
offset: Final = (rng.randrange(1, 2 * cap // page_size + 3) - 1) * page_size if mode == "offset" else 0
return _Request(cursor, offset, min(page_size, cap - offset), sort_desc, None, cap)
@pytest.mark.parametrize("seed", range(60))
@pytest.mark.asyncio
async def test_a_page_read_from_the_newest_rows_matches_grouping_every_row(seed: int) -> None:
rng: Final = random.Random(seed)
rows: Final = _sessions(rng)
for request in tuple(_request(rng, rows) for _ in range(12)):
grouped = await _session_page(_Reads(rows, request, every_row=True), request)
assert await _session_page(_Reads(rows, request), request) == grouped, request
@pytest.mark.parametrize(
"rows_per_session, expected_budgets",
[
pytest.param(1, (102,), id="one-read-holds-the-cap"),
pytest.param(2, (102, 253), id="sized-by-the-first-reads-density"),
pytest.param(50, (102, 4293, 8586), id="doubles-when-the-density-misleads"),
pytest.param(500, (102, 12878, 62533), id="re-sized-by-each-reads-density"),
],
)
@pytest.mark.asyncio
async def test_reads_grow_by_the_last_reads_rows_per_session_until_one_settles_the_page(
rows_per_session: int, expected_budgets: tuple[int, ...]
) -> None:
"""150 sessions a row per second apart. A cap of 100 starts at 100 + 2 rows."""
rows: Final = tuple((index, f"s{index // rows_per_session:05d}", "k") for index in range(150 * rows_per_session))
request: Final = _Request(cursor=None, offset=0, page_limit=10, sort_desc=True, status=None, cap=100)
reads: Final = _Reads(rows, request)
await _session_page(reads, request)
assert reads.budgets == expected_budgets
@pytest.mark.asyncio
async def test_a_session_tied_with_the_reads_oldest_row_is_not_trusted() -> None:
"""Thirty one-row sessions share one time and the first read takes 12 of them. The page's sixth session sits at the
read's oldest time, where the LIMIT may have cut a session that ranks before it, so the page reads on."""
rows: Final = tuple((0, f"s{index:02d}", "k") for index in range(30))
request: Final = _Request(cursor=None, offset=0, page_limit=5, sort_desc=True, status=None, cap=10)
reads: Final = _Reads(rows, request)
page: Final = await _session_page(reads, request)
assert tuple(session.key for session in page.sessions) == ("s29", "s28", "s27", "s26", "s25")
assert reads.budgets == (12, 24, 48)
@pytest.mark.parametrize("session_count, expected_total", [(100, (100, False)), (101, (100, True))])
@pytest.mark.asyncio
async def test_the_total_is_capped_only_past_the_cap(session_count: int, expected_total: tuple[int, bool]) -> None:
rows: Final = tuple((index, f"s{index:05d}", "k") for index in range(session_count))
request: Final = _Request(cursor=None, offset=0, page_limit=10, sort_desc=True, status=None, cap=100)
page: Final = await _session_page(_Reads(rows, request), request)
assert (page.total, page.total_is_capped) == expected_total
@pytest.mark.asyncio
async def test_a_page_number_near_the_bigint_limit_reads_once() -> None:
rows: Final = tuple((index, f"s{index:05d}", "k") for index in range(500))
offset: Final = (9_300_000_000_000_000 - 1) * 1000
request: Final = _Request(
cursor=None, offset=offset, page_limit=min(1000, 100 - offset), sort_desc=True, status=None, cap=100
)
reads: Final = _Reads(rows, request)
page: Final = await _session_page(reads, request)
assert reads.budgets == (102,)
assert (page.sessions, page.total, page.total_is_capped) == ((), 100, True)
@pytest.mark.parametrize(
"sort_desc, status",
[pytest.param(False, None, id="oldest-first"), pytest.param(True, "failure", id="filtered-by-outcome")],
)
@pytest.mark.asyncio
async def test_a_page_that_needs_every_row_reads_them_once(sort_desc: bool, status: str | None) -> None:
rows: Final = tuple((index, f"s{index:05d}", "k") for index in range(500))
request: Final = _Request(cursor=None, offset=0, page_limit=10, sort_desc=sort_desc, status=status, cap=100)
reads: Final = _Reads(rows, request)
await _session_page(reads, request)
assert reads.budgets == (None,)
def test_parse_session_cursor() -> None:
assert _parse_session_cursor(None) is None
assert _parse_session_cursor("no-delimiter") is None
assert _parse_session_cursor("2026-08-29 10:00:00|sess-1") is None
assert _parse_session_cursor("|hashed-key|sess-1") is None
assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|") is None
assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|sess-1") == (
"2026-08-29 10:00:00",
"sess-1",
"hashed-key",
)
assert _parse_session_cursor("2026-08-29 10:00:00.123|hashed-key|sess|with|pipes") == (
"2026-08-29 10:00:00.123",
"sess|with|pipes",
"hashed-key",
)
def _bound(sql: str, args: tuple[object, ...], pattern: str) -> tuple[object, ...]:
"""The values bound to the ``$N`` placeholders that ``pattern``'s groups capture in ``sql``."""
found: Final = re.search(pattern, sql)
assert found, f"{pattern!r} not found in:\n{sql}"
assert all(1 <= int(index) <= len(args) for index in found.groups()), f"{found.groups()} outside $1..${len(args)}"
return tuple(args[int(index) - 1] for index in found.groups())
class _TableDb:
"""A table of ``rows`` matching rows in ``sessions`` sessions, whose reads return no page row."""
def __init__(self, rows: int, sessions: int) -> None:
self.rows: Final = rows
self.sessions: Final = sessions
self.calls: tuple[tuple[str, tuple[object, ...]], ...] = ()
async def query_raw(self, query: str, *args: object) -> tuple[_ReadRow, ...]:
self.calls = (*self.calls, (query, args))
budget: Final = re.search(r"DESC LIMIT (\d+)", query)
rows_read: Final = min(int(budget.group(1)), self.rows) if budget else self.rows
return (
_ReadRow(
session_key=None,
api_key=None,
last_activity=None,
settled=None,
rows_read=rows_read,
session_count=self.sessions,
),
)
def _page_bounds(sql: str) -> tuple[int, int]:
"""The LIMIT and OFFSET of the read's sessions."""
found: Final = re.search(r"LIMIT (\d+)(?: OFFSET (\d+))?\s*\) AS page", sql)
assert found, sql
return int(found.group(1)), int(found.group(2) or 0)
def _read_size(sql: str) -> int | None:
found: Final = re.search(r"DESC LIMIT (\d+)", sql)
return int(found.group(1)) if found else None
@pytest.mark.asyncio
async def test_a_read_that_fills_its_size_is_not_taken_for_every_row() -> None:
"""A read that took every row it asked for may have stopped short, so its 50 sessions are no total: the reads
grow until one takes fewer rows than it asked for."""
db: Final = _TableDb(rows=9_000, sessions=50)
page: Final = await read_session_page(
db,
where_clause="TRUE",
params=(),
page=1,
page_size=10,
status=None,
sort_desc=True,
session_cursor=None,
cap=100,
)
assert tuple(_read_size(sql) for sql, _ in db.calls) == (102, 258, 652, 1647, 4159, 10502)
assert (page.total, page.total_is_capped) == (50, False)
@pytest.mark.parametrize("page_number", [pytest.param(5, id="at-the-cap"), pytest.param(6, id="past-the-cap")])
@pytest.mark.asyncio
async def test_a_page_starting_at_or_past_the_cap_reads_once_and_shows_no_session(page_number: int) -> None:
db: Final = _TableDb(rows=1_000, sessions=150)
page: Final = await read_session_page(
db,
where_clause="TRUE",
params=(),
page=page_number,
page_size=25,
status=None,
sort_desc=True,
session_cursor=None,
cap=100,
)
assert len(db.calls) == 1, "the first read already holds more sessions than the cap"
assert _page_bounds(db.calls[0][0])[0] == 0
assert (page.total, page.total_is_capped) == (100, True)
@pytest.mark.asyncio
async def test_the_read_binds_the_filters_and_the_cursor_to_their_placeholders() -> None:
db: Final = _TableDb(rows=50, sessions=40)
where, params = "model = $1 AND api_key = $2", ("gpt-4o", "key-a")
for page, page_size, session_cursor in ((1, 3, "2026-10-28 09:00:00|key-a|sess|9"), (3, 1, None)):
await read_session_page(
db,
where_clause=where,
params=params,
page=page,
page_size=page_size,
status=None,
sort_desc=True,
session_cursor=session_cursor,
cap=100,
)
(cursor_sql, cursor_args), (offset_sql, _) = db.calls
for sql, args in db.calls:
assert {int(index) for index in re.findall(r"\$(\d+)", sql)} == set(range(1, len(args) + 1)), (
"every argument needs a placeholder and every placeholder an argument"
)
assert _bound(sql, args, r"model = \$(\d+) AND api_key = \$(\d+)") == params
assert _bound(
cursor_sql, cursor_args, r"\(last_activity, session_key, api_key\) < \(\$(\d+)::timestamp, \$(\d+), \$(\d+)\)"
) == ("2026-10-28 09:00:00", "sess|9", "key-a")
assert _page_bounds(cursor_sql) == (4, 0)
assert _page_bounds(offset_sql) == (2, 2)
@pytest.mark.asyncio
async def test_a_status_filtered_read_binds_the_status_between_the_filters_and_the_cursor() -> None:
db: Final = _TableDb(rows=50, sessions=40)
await read_session_page(
db,
where_clause="model = $1",
params=("gpt-4o",),
status="failure",
page=1,
page_size=3,
sort_desc=True,
session_cursor="2026-10-28 09:00:00|key-a|sess-9",
cap=100,
)
((sql, args),) = db.calls
assert {int(index) for index in re.findall(r"\$(\d+)", sql)} == set(range(1, len(args) + 1))
assert _bound(sql, args, r"model = \$(\d+)") == ("gpt-4o",)
assert _bound(sql, args, r"COALESCE\(status, 'success'\) = \$(\d+)") == ("failure",)
assert _bound(sql, args, r"\(last_activity, session_key, api_key\) < \(\$(\d+)::timestamp, \$(\d+), \$(\d+)\)") == (
"2026-10-28 09:00:00",
"sess-9",
"key-a",
)

View file

@ -5,7 +5,7 @@ import hashlib
import json
import re
import sqlite3
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from datetime import timezone
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock, patch
@ -7151,37 +7151,19 @@ def test_scoped_spend_report_range_at_max_allowed(client, monkeypatch):
app.dependency_overrides.pop(ps.user_api_key_auth, None)
def test_parse_session_cursor():
from litellm.proxy.spend_tracking.spend_management_endpoints import (
_parse_session_cursor,
)
assert _parse_session_cursor(None) is None
assert _parse_session_cursor("no-delimiter") is None
assert _parse_session_cursor("2026-08-29 10:00:00|sess-1") is None
assert _parse_session_cursor("|hashed-key|sess-1") is None
assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|") is None
assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|sess-1") == (
"2026-08-29 10:00:00",
"sess-1",
"hashed-key",
)
assert _parse_session_cursor("2026-08-29 10:00:00.123|hashed-key|sess|with|pipes") == (
"2026-08-29 10:00:00.123",
"sess|with|pipes",
"hashed-key",
)
SESSION_GROUP_KEY_SQL = "COALESCE(NULLIF(session_id, ''), request_id), api_key"
def _session_grouped_mock_prisma(session_page_rows, session_total, representative_rows):
def _session_grouped_mock_prisma(
session_page_rows: Sequence[Mapping[str, object]],
session_total: int,
representative_rows: Sequence[Mapping[str, object]],
) -> MagicMock:
"""Mock prisma dispatching the raw queries the keyset grouped path emits."""
async def mock_query_raw(sql_query, *params):
if "COUNT(*) AS total_count" in sql_query:
return [{"total_count": session_total}]
async def mock_query_raw(sql_query: str, *params: object) -> Sequence[Mapping[str, object]]:
if "rows_read" in sql_query:
return _session_read_rows(session_page_rows, session_total)
if "DISTINCT ON" in sql_query:
return representative_rows
if "COALESCE(SUM(spend)" in sql_query:
@ -7199,7 +7181,7 @@ def _session_grouped_mock_prisma(session_page_rows, session_total, representativ
"session_models": ["gpt-4o"],
}
]
return session_page_rows
raise AssertionError(f"unexpected query:\n{sql_query}")
mock_prisma = MagicMock()
mock_prisma.db = MagicMock()
@ -7228,23 +7210,33 @@ def _session_page_row(session_key, last_activity):
return {"session_key": session_key, "api_key": "hashed-key", "last_activity": last_activity}
def _session_grouped_paginating_prisma(sessions, counted_total=None):
"""Mock prisma serving the grouped page query out of ``sessions``, honoring the LIMIT and OFFSET it asks for."""
def _session_read_rows(
page_rows: Sequence[Mapping[str, object]], session_count: int
) -> tuple[Mapping[str, object], ...]:
"""The read query's rows for ``page_rows``, as if the read held every matching row."""
summary: Final = {"rows_read": 0, "session_count": session_count}
if not page_rows:
return ({"session_key": None, "api_key": None, "last_activity": None, "settled": None, **summary},)
return tuple({**row, "settled": True, **summary} for row in page_rows)
async def mock_query_raw(sql_query, *params):
if "COUNT(*) AS total_count" in sql_query:
return [{"total_count": min(len(sessions) if counted_total is None else counted_total, params[-1])}]
def _session_grouped_paginating_prisma(sessions: Sequence[tuple[str, str]]) -> MagicMock:
"""Mock prisma serving the grouped page read out of ``sessions``, honoring the LIMIT and OFFSET it asks for."""
async def mock_query_raw(sql_query: str, *params: object) -> Sequence[Mapping[str, object]]:
if "DISTINCT ON" in sql_query:
return [_session_representative_row(f"req-{session_key}", session_key) for session_key in params[-2]]
if "COALESCE(SUM(spend)" in sql_query:
return []
bounds = re.search(r"LIMIT \$(\d+)(?: OFFSET \$(\d+))?", sql_query)
limit = params[int(bounds.group(1)) - 1]
offset = params[int(bounds.group(2)) - 1] if bounds.group(2) else 0
return [
_session_page_row(session_key, last_activity)
for session_key, last_activity in sessions[offset : offset + limit]
]
bounds: Final = re.search(r"LIMIT (\d+)(?: OFFSET (\d+))?\s+\) AS page", sql_query)
limit, offset = int(bounds.group(1)), int(bounds.group(2) or 0)
return _session_read_rows(
[
_session_page_row(session_key, last_activity)
for session_key, last_activity in sessions[offset : offset + limit]
],
len(sessions),
)
mock_prisma = MagicMock()
mock_prisma.db = MagicMock()
@ -7296,20 +7288,15 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc
assert data["data"][1]["session_total_count"] == 1
emitted = [call.args for call in mock_prisma.db.query_raw.await_args_list]
assert len(emitted) == 3, "one read picks the page and counts it, then the representatives and their totals"
page_query_sql = emitted[0][0]
assert f"GROUP BY {SESSION_GROUP_KEY_SQL}" in page_query_sql
assert "OFFSET" not in page_query_sql
assert "HAVING" not in page_query_sql
assert emitted[0][-1] == 3, "page query fetches page_size + 1 sessions to detect has_more"
assert "LIMIT 3 " in page_query_sql, "the read fetches page_size + 1 sessions to detect has_more"
count_sql = emitted[1][0]
assert f"GROUP BY {SESSION_GROUP_KEY_SQL}" in count_sql
assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded"
rep_call = emitted[2]
rep_call: Final = emitted[1]
assert f"DISTINCT ON ({SESSION_GROUP_KEY_SQL})" in rep_call[0]
assert (
f"ORDER BY {SESSION_GROUP_KEY_SQL}, " + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL
f"ORDER BY {SESSION_GROUP_KEY_SQL}, " + spend_management_endpoints.SESSION_REPRESENTATIVE_ORDER_SQL
) in rep_call[0]
assert rep_call[-2] == ["sess-1", "req-solo"]
assert rep_call[-1] == ["hashed-key", "hashed-key"]
@ -7319,7 +7306,7 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc
@pytest.mark.asyncio
async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypatch):
"""A session_cursor becomes a HAVING keyset predicate instead of an OFFSET."""
"""A session_cursor becomes a keyset predicate on the read's sessions instead of an OFFSET."""
page_rows = [_session_page_row("sess-2", "2026-08-29 07:00:00")]
reps = [_session_representative_row("req-2", "sess-2")]
mock_prisma = _session_grouped_mock_prisma(page_rows, 3, reps)
@ -7352,7 +7339,7 @@ async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypat
page_query_call = mock_prisma.db.query_raw.await_args_list[0]
page_query_sql = page_query_call.args[0]
assert f'HAVING (MAX("startTime"), {SESSION_GROUP_KEY_SQL}) <' in page_query_sql
assert "WHERE (last_activity, session_key, api_key) <" in page_query_sql
assert "OFFSET" not in page_query_sql
cursor_index = page_query_call.args.index("2026-08-29 09:00:00")
assert page_query_call.args[cursor_index : cursor_index + 3] == (
@ -7403,9 +7390,9 @@ async def test_ui_view_spend_logs_group_by_session_jumps_to_page_without_cursor(
@pytest.mark.asyncio
async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(client, monkeypatch):
"""A page that runs out of sessions is the end of the list, so the total comes from it and nothing is counted."""
"""A page that runs out of sessions has read every matching row, so that one read gives the total."""
sessions = tuple((f"sess-{index:02d}", f"2026-08-29 10:{59 - index:02d}:00") for index in range(10))
mock_prisma = _session_grouped_paginating_prisma(sessions, counted_total=999)
mock_prisma: Final = _session_grouped_paginating_prisma(sessions)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma)
monkeypatch.setattr(
"litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe",
@ -7429,10 +7416,12 @@ async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(clie
)
assert response.status_code == 200, response.text
data = response.json()
assert data["total"] == 10, "the count query's 999 would have won if it had been asked"
assert data["total"] == 10
assert data["total_is_capped"] is False
assert data["total_pages"] == 1
assert len(data["data"]) == 10
reads: Final = tuple(call for call in mock_prisma.db.query_raw.await_args_list if "rows_read" in call.args[0])
assert len(reads) == 1, "a page that ends the list is not read again for its total"
finally:
app.dependency_overrides.pop(ps.user_api_key_auth, None)
@ -7517,7 +7506,7 @@ async def test_ui_view_spend_logs_group_by_session_page_past_count_cap_is_empty(
assert past_cap_data["data"] == []
assert past_cap_data["has_more"] is False
assert past_cap_data["total"] == cap
assert mock_prisma.db.query_raw.await_count == 1, "only the bounded count query runs past the capped window"
assert mock_prisma.db.query_raw.await_count == 1, "only the bounded read runs past the capped window"
finally:
app.dependency_overrides.pop(ps.user_api_key_auth, None)
@ -7820,14 +7809,14 @@ def test_session_representative_uses_completed_agent_outcome(parent_status, chil
)
result = connection.execute(
"SELECT request_id, status FROM logs ORDER BY "
+ spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL
+ spend_management_endpoints.SESSION_REPRESENTATIVE_ORDER_SQL
+ " LIMIT 1"
).fetchone()
assert result == ("parent", expected)
connection.execute("DELETE FROM logs WHERE call_type = 'asend_message'")
fallback = connection.execute(
"SELECT request_id, status FROM logs ORDER BY "
+ spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL
+ spend_management_endpoints.SESSION_REPRESENTATIVE_ORDER_SQL
+ " LIMIT 1"
).fetchone()
assert fallback == ("llm", "success")

View file

@ -6,7 +6,9 @@ GitHub Issue: #17487
"""
import datetime
from collections.abc import Mapping, Sequence
from datetime import timezone
from typing import Final
from unittest.mock import AsyncMock, MagicMock
import pytest
@ -508,20 +510,10 @@ async def test_global_spend_report_team_group_forwards_team_id(monkeypatch):
@pytest.mark.asyncio
async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch):
"""
With group_by_session=true, /spend/logs/ui must page and count SESSIONS,
not raw calls: the page is one representative row per session (DISTINCT
ON the session group key, preferring non-MCP calls, newest first), the
bounded count counts groups, and on the default startTime sort the page is
selected by keyset (no OFFSET) so deep pages do not degrade. Otherwise the
UI collapses a server page of N calls into fewer visible rows while the
footer still claims N (issue #38060).
"""
"""With group_by_session=true, /spend/logs/ui pages and counts sessions, not calls, one representative row per
session, so the footer's total matches the rows shown (issue #38060)."""
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.spend_tracking.spend_management_endpoints import (
SPEND_LOGS_PAGINATION_COUNT_CAP,
ui_view_spend_logs,
)
from litellm.proxy.spend_tracking.spend_management_endpoints import ui_view_spend_logs
group_key = "COALESCE(NULLIF(session_id, ''), request_id), api_key"
session_rows = [
@ -533,12 +525,12 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch):
{"request_id": "req-2", "api_key": "k", "metadata": "{}", "session_id": None},
]
async def mock_query_raw(sql_query, *params):
if "COUNT(*) AS total_count" in sql_query:
return [{"total_count": 60}]
async def mock_query_raw(sql_query: str, *params: object) -> Sequence[Mapping[str, object]]:
if "rows_read" in sql_query:
return [{**row, "settled": True, "rows_read": 0, "session_count": 60} for row in session_rows]
if "AS session_representatives" in sql_query:
return representative_rows
return session_rows
return []
mock_prisma = MagicMock()
mock_prisma.db = MagicMock()
@ -565,22 +557,16 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch):
user_api_key_dict=auth,
group_by_session=True,
session_cursor=None,
status_filter=None,
)
emitted = [call[0] for call in mock_prisma.db.query_raw.call_args_list]
page_sql = emitted[0][0]
assert f"GROUP BY {group_key}" in page_sql, f"page must select sessions, not calls. SQL was:\n{page_sql}"
assert "GROUP BY session_key, api_key" in page_sql, f"page must select sessions, not calls. SQL was:\n{page_sql}"
assert "OFFSET" not in page_sql, "the startTime page must be keyset-selected, not offset-selected"
assert emitted[0][-1] == 51, "the page query fetches page_size + 1 sessions to detect has_more"
assert "LIMIT 51 " in page_sql, "the page read fetches page_size + 1 sessions to detect has_more"
count_call = emitted[1]
count_sql = count_call[0]
assert f"GROUP BY {group_key}" in count_sql, f"grouped total must count sessions. SQL was:\n{count_sql}"
assert "COUNT(*) OVER ()" not in count_sql
assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded"
assert count_call[-1] == SPEND_LOGS_PAGINATION_COUNT_CAP + 1
rep_sql = emitted[2][0]
rep_sql: Final = emitted[1][0]
assert f"DISTINCT ON ({group_key})" in rep_sql, f"page must return one row per session. SQL was:\n{rep_sql}"
assert (
f"ORDER BY {group_key}, (call_type = 'asend_message') DESC, "

View file

@ -18,6 +18,27 @@ from litellm.types.proxy.management_endpoints.ui_sso import (
client = TestClient(app)
@pytest.mark.asyncio
async def test_sso_settings_without_prisma_reports_database_not_connected(monkeypatch: pytest.MonkeyPatch) -> None:
import runpy
import sys
from fastapi import HTTPException
from litellm.proxy import proxy_server
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.ui_crud_endpoints import proxy_setting_endpoints
monkeypatch.setattr(proxy_server, "prisma_client", None)
with patch.dict(sys.modules, {"prisma": None, "prisma.errors": None}):
endpoints: Final = runpy.run_path(proxy_setting_endpoints.__file__)
with pytest.raises(HTTPException) as error:
await endpoints["update_sso_settings"](SSOConfig(), UserAPIKeyAuth())
assert error.value.status_code == 500
assert error.value.detail == {"error": "Database not connected. Please connect a database."}
@pytest.mark.asyncio
@pytest.mark.parametrize("config_value,stored_value,expected", [
(None, None, False), (None, False, False), (None, True, True),

View file

@ -1,4 +1,5 @@
import asyncio
from collections.abc import Mapping
from typing import Final, cast
import pytest
@ -377,6 +378,32 @@ def test_extract_cacheable_prefix_mixed_string_and_list_content():
assert isinstance(result[2]["content"], list)
def test_prompt_cache_breakpoint_marks_the_same_cacheable_prefix_as_cache_control():
def _messages(marker: Mapping[str, Mapping[str, str]]) -> list[AllMessageValues]:
return cast( # cast-ok: the text part type does not declare the prompt_cache_breakpoint key
list[AllMessageValues],
[
{
"role": "system",
"content": [
{"type": "text", "text": "Here is the full text of a complex legal agreement" * 400, **marker}
],
},
{"role": "user", "content": "what are the key terms?"},
],
)
marked_key: Final = PromptCachingCache.get_prompt_caching_cache_key(
_messages({"prompt_cache_breakpoint": {"mode": "explicit"}}), None
)
assert marked_key is not None
assert marked_key == PromptCachingCache.get_prompt_caching_cache_key(
_messages({"cache_control": {"type": "ephemeral"}}), None
)
assert PromptCachingCache.get_prompt_caching_cache_key(_messages({}), None) is None
@pytest.mark.asyncio
async def test_same_cacheable_prefix_routes_to_same_deployment() -> None:
from litellm import Router

View file

@ -1026,6 +1026,7 @@ _COST_MAP_ROW_SCHEMA: Final = {
"cache_read_input_image_token_cost": {"type": "number"},
"audio_transcription_config": {"type": "string"},
"deprecation_date": {"type": "string"},
"successor_model": {"type": "string"},
"input_cost_per_audio_per_second": {"type": "number"},
"input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"},
"google_maps_grounding_cost_per_query": {"type": "number"},

View file

@ -2,6 +2,7 @@ import { FilterInput } from "@/components/common_components/Filters/FilterInput"
import { FiltersButton } from "@/components/common_components/Filters/FiltersButton";
import { ResetFiltersButton } from "@/components/common_components/Filters/ResetFiltersButton";
import { Search, User } from "lucide-react";
import { useTranslation } from "react-i18next";
interface OrganizationFiltersProps {
filters: FilterState;
@ -23,6 +24,7 @@ const OrganizationFilters = ({
onChange,
onReset,
}: OrganizationFiltersProps) => {
const { t } = useTranslation();
const hasActiveFilters = !!(filters.org_id || filters.org_alias);
return (
@ -30,7 +32,7 @@ const OrganizationFilters = ({
{/* Search and Filter Controls */}
<div className="flex flex-wrap items-center gap-3">
<FilterInput
placeholder="Search by Organization Name"
placeholder={t("organizations.filters.searchByName")}
value={filters.org_alias}
onChange={(value) => onChange("org_alias", value)}
icon={Search}
@ -50,7 +52,7 @@ const OrganizationFilters = ({
{showFilters && (
<div className="flex flex-wrap items-center gap-3 mt-3">
<FilterInput
placeholder="Search by Organization ID"
placeholder={t("organizations.filters.searchById")}
value={filters.org_id}
onChange={(value) => onChange("org_id", value)}
icon={User}

View file

@ -4,6 +4,7 @@ import OrganizationFilters, { FilterState } from "@/app/(dashboard)/organization
import { useQueryClient } from "@tanstack/react-query";
import { parseAsString, parseAsStringLiteral, useQueryStates } from "nuqs";
import React, { useState } from "react";
import { Trans, useTranslation } from "react-i18next";
import DeleteResourceModal from "@/components/common_components/DeleteResourceModal";
import { toast } from "@/lib/toast";
import { organizationDeleteCall } from "@/components/networking";
@ -28,6 +29,7 @@ const ORGANIZATION_DETAIL_STATE = {
const ORGANIZATION_DETAIL_URL_KEYS = { tab: ORGANIZATION_TAB_URL_KEY };
const OrganizationsPanel: React.FC<OrganizationsPanelProps> = ({ userRole, accessToken, premiumUser }) => {
const { t } = useTranslation();
const [{ org: selectedOrgId }, setOrganizationDetail] = useQueryStates(ORGANIZATION_DETAIL_STATE, {
history: "push",
urlKeys: ORGANIZATION_DETAIL_URL_KEYS,
@ -78,7 +80,7 @@ const OrganizationsPanel: React.FC<OrganizationsPanelProps> = ({ userRole, acces
try {
setIsDeleting(true);
await organizationDeleteCall(accessToken, orgToDelete);
toast.success("Organization deleted successfully");
toast.success(t("organizations.panel.toasts.deleted"));
setIsDeleteModalOpen(false);
setOrgToDelete(null);
@ -99,16 +101,19 @@ const OrganizationsPanel: React.FC<OrganizationsPanelProps> = ({ userRole, acces
return (
<div className="mx-4 mt-4">
<p className="text-sm text-muted-foreground">
This is a LiteLLM Enterprise feature, and requires a valid key to use. Get a trial key{" "}
<a
href="https://www.litellm.ai/#pricing"
target="_blank"
rel="noopener noreferrer"
className="text-primary underline-offset-4 hover:underline"
>
here
</a>
.
<Trans
i18nKey="organizations.panel.enterpriseNotice"
components={{
pricing: (
<a
href="https://www.litellm.ai/#pricing"
target="_blank"
rel="noopener noreferrer"
className="text-primary underline-offset-4 hover:underline"
/>
),
}}
/>
</p>
</div>
);
@ -118,7 +123,7 @@ const OrganizationsPanel: React.FC<OrganizationsPanelProps> = ({ userRole, acces
<div className="mx-4 mt-4 flex flex-col gap-4">
{(userRole === "Admin" || userRole === "Org Admin") && (
<Button className="w-fit" onClick={() => setIsOrgModalVisible(true)}>
+ Create New Organization
{t("organizations.panel.createButton")}
</Button>
)}
@ -133,7 +138,7 @@ const OrganizationsPanel: React.FC<OrganizationsPanelProps> = ({ userRole, acces
/>
) : (
<>
<p className="text-sm text-muted-foreground">Click on an organization ID to view its details.</p>
<p className="text-sm text-muted-foreground">{t("organizations.panel.listHint")}</p>
<OrganizationFilters
filters={filters}
showFilters={showFilters}
@ -157,10 +162,10 @@ const OrganizationsPanel: React.FC<OrganizationsPanelProps> = ({ userRole, acces
<DeleteResourceModal
isOpen={isDeleteModalOpen}
title="Delete Organization?"
message="Are you sure you want to delete this organization? This action cannot be undone."
resourceInformationTitle="Organization Information"
resourceInformation={[{ label: "Organization ID", value: orgToDelete, code: true }]}
title={t("organizations.panel.deleteModal.title")}
message={t("organizations.panel.deleteModal.message")}
resourceInformationTitle={t("organizations.panel.deleteModal.infoTitle")}
resourceInformation={[{ label: t("organizations.panel.deleteModal.idLabel"), value: orgToDelete, code: true }]}
onCancel={cancelDelete}
onOk={confirmDelete}
confirmLoading={isDeleting}

View file

@ -2,6 +2,7 @@
import { Building2, SearchX } from "lucide-react";
import React, { useMemo } from "react";
import { useTranslation } from "react-i18next";
import { DataTable } from "@/components/shared/DataTable";
import { Organization } from "@/components/networking";
@ -20,6 +21,7 @@ interface OrganizationsTableProps {
}
function EmptyState({ searchActive }: { searchActive: boolean }) {
const { t } = useTranslation();
const Icon = searchActive ? SearchX : Building2;
return (
<div className="flex flex-col items-center gap-1 py-6">
@ -27,12 +29,12 @@ function EmptyState({ searchActive }: { searchActive: boolean }) {
<Icon className="size-5 text-muted-foreground" />
</div>
<div className="text-sm font-medium text-foreground">
{searchActive ? "No matching organizations" : "No organizations yet"}
{searchActive ? t("organizations.table.empty.noMatchesTitle") : t("organizations.table.empty.noneTitle")}
</div>
<div className="text-sm text-muted-foreground">
{searchActive
? "No organizations match your search. Try a different name or ID."
: "Create an organization to group teams, models, and budgets."}
? t("organizations.table.empty.noMatchesDescription")
: t("organizations.table.empty.noneDescription")}
</div>
</div>
);
@ -47,12 +49,13 @@ const OrganizationsTable: React.FC<OrganizationsTableProps> = ({
onEditClick,
onDeleteClick,
}) => {
const { t } = useTranslation();
const { sorting, onSortingChange, pagination, onPaginationChange } = useOrganizationsTableState();
const columns = useMemo(() => {
const deps = { userRole, onOrganizationClick, onEditClick, onDeleteClick };
const deps = { userRole, onOrganizationClick, onEditClick, onDeleteClick, t };
return getOrganizationsTableColumns(deps);
}, [userRole, onOrganizationClick, onEditClick, onDeleteClick]);
}, [userRole, onOrganizationClick, onEditClick, onDeleteClick, t]);
return (
<DataTable
@ -67,7 +70,7 @@ const OrganizationsTable: React.FC<OrganizationsTableProps> = ({
sorting={sorting}
onSortingChange={onSortingChange}
isLoading={isLoading}
loadingMessage="Loading organizations…"
loadingMessage={t("organizations.table.loading")}
noDataMessage={<EmptyState searchActive={searchActive} />}
size="compact"
/>

View file

@ -1,7 +1,9 @@
"use client";
import { ColumnDef } from "@tanstack/react-table";
import type { TFunction } from "i18next";
import { MoreHorizontal, Pencil, Trash2 } from "lucide-react";
import { useTranslation } from "react-i18next";
import { DataTableMultiSortHeader, DataTableSortHeader } from "@/components/shared/DataTable";
import { DateCell, IdentityCell, ModelsCell, MoneyCell } from "@/components/shared/table_cells";
@ -25,11 +27,18 @@ const getOrganizationBudget = (organization: Organization): OrganizationBudget =
(organization.litellm_budget_table ?? {}) as OrganizationBudget;
function OrganizationLimitsCell({ organization }: { organization: Organization }) {
const { t } = useTranslation();
const { tpm_limit, rpm_limit } = getOrganizationBudget(organization);
return (
<div className="flex flex-col text-xs text-muted-foreground">
<span>TPM: {tpm_limit ?? "Unlimited"}</span>
<span>RPM: {rpm_limit ?? "Unlimited"}</span>
<span>
{t("organizations.table.limits.tpmLabel")}
{tpm_limit ?? t("organizations.table.limits.unlimited")}
</span>
<span>
{t("organizations.table.limits.rpmLabel")}
{rpm_limit ?? t("organizations.table.limits.unlimited")}
</span>
</div>
);
}
@ -41,10 +50,11 @@ interface OrganizationRowActionsProps {
}
function OrganizationRowActions({ organization, onEditClick, onDeleteClick }: OrganizationRowActionsProps) {
const { t } = useTranslation();
return (
<DropdownMenu>
<DropdownMenuTrigger
aria-label="Open organization actions"
aria-label={t("organizations.table.actions.open")}
data-testid={`organization-actions-${organization.organization_id}`}
className={cn(buttonVariants({ variant: "ghost", size: "icon-sm" }), "text-muted-foreground")}
>
@ -56,7 +66,7 @@ function OrganizationRowActions({ organization, onEditClick, onDeleteClick }: Or
onClick={() => onEditClick(organization.organization_id)}
>
<Pencil />
Edit
{t("organizations.table.actions.edit")}
</DropdownMenuItem>
<DropdownMenuItem
variant="destructive"
@ -64,7 +74,7 @@ function OrganizationRowActions({ organization, onEditClick, onDeleteClick }: Or
onClick={() => onDeleteClick(organization.organization_id)}
>
<Trash2 />
Delete
{t("organizations.table.actions.delete")}
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
@ -76,6 +86,7 @@ export interface OrganizationsTableColumnsDeps {
onOrganizationClick: (organizationId: string) => void;
onEditClick: (organizationId: string) => void;
onDeleteClick: (organizationId: string) => void;
t: TFunction;
}
export const getOrganizationsTableColumns = ({
@ -83,12 +94,13 @@ export const getOrganizationsTableColumns = ({
onOrganizationClick,
onEditClick,
onDeleteClick,
t,
}: OrganizationsTableColumnsDeps): ColumnDef<Organization>[] => [
{
id: "organization_id",
accessorKey: "organization_id",
meta: { title: "Organization ID" },
header: ({ column }) => <DataTableSortHeader column={column} title="Organization ID" />,
meta: { title: t("organizations.table.columns.id") },
header: ({ column }) => <DataTableSortHeader column={column} title={t("organizations.table.columns.id")} />,
size: 220,
enableSorting: true,
cell: ({ row }) => (
@ -103,8 +115,8 @@ export const getOrganizationsTableColumns = ({
{
id: "organization_alias",
accessorKey: "organization_alias",
meta: { title: "Organization Name" },
header: ({ column }) => <DataTableSortHeader column={column} title="Organization Name" />,
meta: { title: t("organizations.table.columns.name") },
header: ({ column }) => <DataTableSortHeader column={column} title={t("organizations.table.columns.name")} />,
size: 200,
enableSorting: true,
cell: ({ row }) => {
@ -120,8 +132,8 @@ export const getOrganizationsTableColumns = ({
id: "created_at",
accessorKey: "created_at",
sortingFn: "datetime",
meta: { title: "Created" },
header: ({ column }) => <DataTableSortHeader column={column} title="Created" />,
meta: { title: t("organizations.table.columns.created") },
header: ({ column }) => <DataTableSortHeader column={column} title={t("organizations.table.columns.created")} />,
size: 130,
enableSorting: true,
cell: ({ row }) => <DateCell value={row.original.created_at} precision="date" />,
@ -129,8 +141,8 @@ export const getOrganizationsTableColumns = ({
{
id: "spend",
accessorKey: "spend",
meta: { title: "Spend (USD)", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="Spend (USD)" />,
meta: { title: t("organizations.table.columns.spend"), numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title={t("organizations.table.columns.spend")} />,
size: 120,
enableSorting: true,
cell: ({ row }) => <MoneyCell value={row.original.spend} decimals={4} />,
@ -140,12 +152,17 @@ export const getOrganizationsTableColumns = ({
accessorFn: (organization) => getOrganizationBudget(organization).max_budget ?? Number.POSITIVE_INFINITY,
sortUndefined: "last",
sortDescFirst: true,
meta: { title: "Budget (USD)", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="Budget (USD)" />,
meta: { title: t("organizations.table.columns.budget"), numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title={t("organizations.table.columns.budget")} />,
size: 120,
enableSorting: true,
cell: ({ row }) => (
<MoneyCell value={getOrganizationBudget(row.original).max_budget} decimals={2} emptyText="Unlimited" showZero />
<MoneyCell
value={getOrganizationBudget(row.original).max_budget}
decimals={2}
emptyText={t("organizations.table.budgetUnlimited")}
showZero
/>
),
},
{
@ -156,21 +173,21 @@ export const getOrganizationsTableColumns = ({
: undefined,
sortUndefined: "last",
sortDescFirst: false,
meta: { title: "Models", skeleton: "chips" },
header: ({ column }) => <DataTableSortHeader column={column} title="Models" />,
meta: { title: t("organizations.table.columns.models"), skeleton: "chips" },
header: ({ column }) => <DataTableSortHeader column={column} title={t("organizations.table.columns.models")} />,
size: 260,
enableSorting: true,
cell: ({ row }) => <ModelsCell models={row.original.models} />,
},
{
id: "limits",
meta: { title: "TPM / RPM Limits" },
meta: { title: t("organizations.table.columns.limits") },
header: ({ table }) => (
<DataTableMultiSortHeader
table={table}
fields={[
{ id: "tpm_limit", label: "TPM" },
{ id: "rpm_limit", label: "RPM" },
{ id: "tpm_limit", label: t("organizations.table.columns.tpmSort") },
{ id: "rpm_limit", label: t("organizations.table.columns.rpmSort") },
]}
/>
),
@ -183,11 +200,15 @@ export const getOrganizationsTableColumns = ({
accessorFn: (organization) => organization.members?.length ?? 0,
sortUndefined: "last",
sortDescFirst: true,
meta: { title: "Members", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="Members" />,
meta: { title: t("organizations.table.columns.members"), numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title={t("organizations.table.columns.members")} />,
size: 100,
enableSorting: true,
cell: ({ row }) => <span className="text-sm">{row.original.members?.length ?? 0} Members</span>,
cell: ({ row }) => (
<span className="text-sm">
{t("organizations.table.memberCount", { count: row.original.members?.length ?? 0 })}
</span>
),
},
...(["tpm_limit", "rpm_limit"] as const).map(
(field): ColumnDef<Organization> => ({
@ -201,7 +222,7 @@ export const getOrganizationsTableColumns = ({
{
id: "actions",
meta: { className: "text-right", headerClassName: "text-right" },
header: () => <span className="sr-only">Actions</span>,
header: () => <span className="sr-only">{t("organizations.table.columns.actions")}</span>,
size: 64,
enableSorting: false,
enableHiding: false,

View file

@ -1,4 +1,5 @@
import React, { useId, useState } from "react";
import { Trans, useTranslation } from "react-i18next";
import { userBulkUpdateUserCall, teamBulkMemberAddCall, Member } from "@/components/networking";
import { UserEditView } from "./user_edit_view";
import { toast } from "@/lib/toast";
@ -46,6 +47,7 @@ const BulkEditUserModal: React.FC<BulkEditUserModalProps> = ({
userModels,
allowAllUsers = false,
}) => {
const { t } = useTranslation();
const { premiumUser } = useAuthorized();
const [loading, setLoading] = useState(false);
const [selectedTeams, setSelectedTeams] = useState<string[]>([]);
@ -89,7 +91,7 @@ const BulkEditUserModal: React.FC<BulkEditUserModalProps> = ({
const handleSubmit = async (formValues: any) => {
if (!accessToken) {
toast.fromError("Access token not found");
toast.fromError(t("users.bulkEdit.toasts.noAccessToken"));
return;
}
@ -125,7 +127,7 @@ const BulkEditUserModal: React.FC<BulkEditUserModalProps> = ({
const hasTeamAdditions = addToTeams && selectedTeams.length > 0;
if (!hasUserUpdates && !hasTeamAdditions) {
toast.fromError("Please modify at least one field or select teams to add users to");
toast.fromError(t("users.bulkEdit.toasts.nothingToUpdate"));
return;
}
@ -135,10 +137,10 @@ const BulkEditUserModal: React.FC<BulkEditUserModalProps> = ({
if (hasUserUpdates) {
if (updateAllUsers) {
const result = await userBulkUpdateUserCall(accessToken, updatePayload, undefined, true);
successMessages.push(`Updated all users (${result.total_requested} total)`);
successMessages.push(t("users.bulkEdit.toasts.updatedAll", { total: result.total_requested }));
} else {
await userBulkUpdateUserCall(accessToken, updatePayload, userIds);
successMessages.push(`Updated ${userIds.length} user(s)`);
successMessages.push(t("users.bulkEdit.toasts.updated", { count: userIds.length }));
}
}
@ -190,11 +192,13 @@ const BulkEditUserModal: React.FC<BulkEditUserModalProps> = ({
if (successfulTeams.length > 0) {
const totalAdditions = successfulTeams.reduce((sum, r) => sum + r.successfulAdditions, 0);
successMessages.push(`Added users to ${successfulTeams.length} team(s) (${totalAdditions} total additions)`);
successMessages.push(
t("users.bulkEdit.toasts.addedToTeams", { count: successfulTeams.length, additions: totalAdditions }),
);
}
if (failedTeams.length > 0) {
toast.warning(`Failed to add users to ${failedTeams.length} team(s)`);
toast.warning(t("users.bulkEdit.toasts.teamAddFailed", { count: failedTeams.length }));
}
}
@ -212,7 +216,7 @@ const BulkEditUserModal: React.FC<BulkEditUserModalProps> = ({
onCancel();
} catch (error) {
console.error("Bulk operation failed:", error);
toast.fromError("Failed to perform bulk operations");
toast.fromError(t("users.bulkEdit.toasts.bulkFailed"));
} finally {
setLoading(false);
}
@ -223,7 +227,7 @@ const BulkEditUserModal: React.FC<BulkEditUserModalProps> = ({
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[800px]">
<DialogHeader>
<DialogTitle>
{updateAllUsers ? "Bulk Edit All Users" : `Bulk Edit ${selectedUsers.length} User(s)`}
{updateAllUsers ? t("users.bulkEdit.titleAll") : t("users.bulkEdit.title", { count: selectedUsers.length })}
</DialogTitle>
</DialogHeader>
{allowAllUsers && (
@ -233,17 +237,15 @@ const BulkEditUserModal: React.FC<BulkEditUserModalProps> = ({
id={updateAllUsersId}
checked={updateAllUsers}
onCheckedChange={(checked) => setUpdateAllUsers(checked === true)}
aria-label="Update ALL users in the system"
aria-label={t("users.bulkEdit.updateAll")}
/>
<label htmlFor={updateAllUsersId} className="cursor-pointer text-sm font-medium text-foreground">
Update ALL users in the system
{t("users.bulkEdit.updateAll")}
</label>
</div>
{updateAllUsers && (
<div className="mt-2">
<span className="text-xs text-warning">
⚠️ This will apply changes to ALL users in the system, not just the selected ones.
</span>
<span className="text-xs text-warning">{t("users.bulkEdit.updateAllWarning")}</span>
</div>
)}
</div>
@ -251,15 +253,19 @@ const BulkEditUserModal: React.FC<BulkEditUserModalProps> = ({
{!updateAllUsers && (
<div className="mb-4">
<h5 className="mb-2 text-sm font-semibold text-foreground">Selected Users ({selectedUsers.length}):</h5>
<h5 className="mb-2 text-sm font-semibold text-foreground">
{t("users.bulkEdit.selectedUsers", { count: selectedUsers.length })}
</h5>
<div className="max-h-[200px] overflow-y-auto rounded-md border border-border">
<Table>
<TableHeader>
<TableRow>
<TableHead className="w-[30%]">User ID</TableHead>
<TableHead className="w-[25%]">Email</TableHead>
<TableHead className="w-[25%]">Current Role</TableHead>
<TableHead className={cn("w-[20%]", NUMERIC_CELL_CLASS)}>Budget</TableHead>
<TableHead className="w-[30%]">{t("users.bulkEdit.table.userId")}</TableHead>
<TableHead className="w-[25%]">{t("users.bulkEdit.table.email")}</TableHead>
<TableHead className="w-[25%]">{t("users.bulkEdit.table.currentRole")}</TableHead>
<TableHead className={cn("w-[20%]", NUMERIC_CELL_CLASS)}>
{t("users.bulkEdit.table.budget")}
</TableHead>
</TableRow>
</TableHeader>
<TableBody>
@ -268,12 +274,19 @@ const BulkEditUserModal: React.FC<BulkEditUserModalProps> = ({
<TableCell className="text-xs font-medium text-foreground">
{user.user_id.length > 20 ? `${user.user_id.slice(0, 20)}...` : user.user_id}
</TableCell>
<TableCell className="text-xs text-muted-foreground">{user.user_email || "No email"}</TableCell>
<TableCell className="text-xs text-muted-foreground">
{user.user_email || t("users.bulkEdit.table.noEmail")}
</TableCell>
<TableCell className="text-xs text-foreground">
{possibleUIRoles?.[user.user_role]?.ui_label || user.user_role}
</TableCell>
<TableCell className={NUMERIC_CELL_CLASS}>
<MoneyCell value={user.max_budget} decimals={2} emptyText="Unlimited" showZero />
<MoneyCell
value={user.max_budget}
decimals={2}
emptyText={t("users.bulkEdit.table.unlimited")}
showZero
/>
</TableCell>
</TableRow>
))}
@ -287,15 +300,14 @@ const BulkEditUserModal: React.FC<BulkEditUserModalProps> = ({
<div className="mb-4">
<p className="text-sm text-foreground">
<strong>Instructions:</strong> Fill in the fields below with the values you want to apply to all selected
users. You can bulk edit: role, budget, models, and metadata. You can also add users to teams.
<Trans i18nKey="users.bulkEdit.instructions" components={{ strong: <strong /> }} />
</p>
</div>
{/* Team Management Section */}
<Card size="sm" className="mb-4 bg-muted/50">
<CardHeader>
<CardTitle>Team Management</CardTitle>
<CardTitle>{t("users.bulkEdit.teamManagement.title")}</CardTitle>
</CardHeader>
<CardContent>
<div className="flex flex-col gap-2">
@ -304,10 +316,10 @@ const BulkEditUserModal: React.FC<BulkEditUserModalProps> = ({
id={addToTeamsId}
checked={addToTeams}
onCheckedChange={(checked) => setAddToTeams(checked === true)}
aria-label="Add selected users to teams"
aria-label={t("users.bulkEdit.teamManagement.addToTeams")}
/>
<label htmlFor={addToTeamsId} className="cursor-pointer text-sm text-foreground">
Add selected users to teams
{t("users.bulkEdit.teamManagement.addToTeams")}
</label>
</div>
@ -315,12 +327,12 @@ const BulkEditUserModal: React.FC<BulkEditUserModalProps> = ({
<>
<div>
<label htmlFor={selectedTeamsId} className="block text-sm font-medium text-foreground">
Select Teams:
{t("users.bulkEdit.teamManagement.selectTeams")}
</label>
<MultiSelect
id={selectedTeamsId}
className="mt-2"
placeholder="Select teams to add users to"
placeholder={t("users.bulkEdit.teamManagement.selectTeamsPlaceholder")}
value={selectedTeams}
onValueChange={setSelectedTeams}
options={
@ -334,12 +346,12 @@ const BulkEditUserModal: React.FC<BulkEditUserModalProps> = ({
<div>
<label htmlFor={teamBudgetId} className="block text-sm font-medium text-foreground">
Team Budget (Optional):
{t("users.bulkEdit.teamManagement.teamBudget")}
</label>
<NumericalInput
id={teamBudgetId}
className="mt-2"
placeholder="Max budget per user in team"
placeholder={t("users.bulkEdit.teamManagement.teamBudgetPlaceholder")}
value={teamBudget ?? ""}
onChange={(event: React.ChangeEvent<HTMLInputElement>) =>
setTeamBudget(event.target.value === "" ? null : Number(event.target.value))
@ -348,13 +360,12 @@ const BulkEditUserModal: React.FC<BulkEditUserModalProps> = ({
step={0.01}
/>
<span className="text-xs text-muted-foreground">
Leave empty for unlimited budget within team limits
{t("users.bulkEdit.teamManagement.teamBudgetHint")}
</span>
</div>
<span className="text-xs text-muted-foreground">
Users will be added with &quot;user&quot; role by default. All users will be added to each selected
team.
{t("users.bulkEdit.teamManagement.defaultRoleNote")}
</span>
</>
)}
@ -379,7 +390,9 @@ const BulkEditUserModal: React.FC<BulkEditUserModalProps> = ({
{loading && (
<div className="mt-2.5 text-center">
<span className="text-sm text-foreground">
Updating {updateAllUsers ? "all users" : selectedUsers.length} user(s)...
{updateAllUsers
? t("users.bulkEdit.updatingAll")
: t("users.bulkEdit.updating", { count: selectedUsers.length })}
</span>
</div>
)}

View file

@ -1,8 +1,10 @@
"use client";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import type { ParseKeys } from "i18next";
import * as React from "react";
import { useFieldArray, type Control } from "react-hook-form";
import { useTranslation } from "react-i18next";
import { useInfiniteTeams } from "@/app/(dashboard)/hooks/teams/useTeams";
import { ModelSelect, MODEL_SENTINEL_OPTIONS } from "@/components/ModelSelect/ModelSelect";
@ -29,21 +31,21 @@ import {
const NO_RESET = "never";
const BUDGET_DURATION_OPTIONS = [
{ value: NO_RESET, label: "No reset" },
{ value: "1h", label: "hourly" },
{ value: "24h", label: "daily" },
{ value: "7d", label: "weekly" },
{ value: "30d", label: "monthly" },
] as const;
const BUDGET_DURATION_OPTIONS: readonly { value: string; labelKey: ParseKeys }[] = [
{ value: NO_RESET, labelKey: "users.defaultSettings.budgetDurations.noReset" },
{ value: "1h", labelKey: "users.defaultSettings.budgetDurations.hourly" },
{ value: "24h", labelKey: "users.defaultSettings.budgetDurations.daily" },
{ value: "7d", labelKey: "users.defaultSettings.budgetDurations.weekly" },
{ value: "30d", labelKey: "users.defaultSettings.budgetDurations.monthly" },
];
const TEAM_ROLE_OPTIONS = [
{ value: "user", label: "User" },
{ value: "admin", label: "Admin" },
] as const;
const TEAM_ROLE_OPTIONS: readonly { value: "user" | "admin"; labelKey: ParseKeys }[] = [
{ value: "user", labelKey: "users.defaultSettings.teamRoles.user" },
{ value: "admin", labelKey: "users.defaultSettings.teamRoles.admin" },
];
const MODEL_SENTINEL_LABELS: ReadonlyMap<string, string> = new Map(
MODEL_SENTINEL_OPTIONS.map(({ value, label }) => [value, label]),
const MODEL_SENTINEL_LABEL_KEYS: ReadonlyMap<string, ParseKeys> = new Map(
MODEL_SENTINEL_OPTIONS.map(({ value, labelKey }) => [value, labelKey]),
);
const TEAMS_PAGE_SIZE = 50;
@ -71,6 +73,7 @@ interface RoleOption {
type SettingsControl = Control<DefaultUserSettingsFormValues, unknown, DefaultUserSettingsSubmitValues>;
const TeamPickerField = ({ control, index }: { control: SettingsControl; index: number }) => {
const { t } = useTranslation();
const [search, setSearch] = React.useState("");
const { data, fetchNextPage, hasNextPage, isFetchingNextPage, isLoading } = useInfiniteTeams(
TEAMS_PAGE_SIZE,
@ -90,7 +93,7 @@ const TeamPickerField = ({ control, index }: { control: SettingsControl; index:
);
return (
<FormField control={control} name={`teams.${index}.team_id`} label="Team">
<FormField control={control} name={`teams.${index}.team_id`} label={t("users.defaultSettings.teams.team")}>
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
<PaginatedSearchSelect
options={options}
@ -101,8 +104,8 @@ const TeamPickerField = ({ control, index }: { control: SettingsControl; index:
hasNextPage={hasNextPage}
isLoading={isLoading}
isFetchingNextPage={isFetchingNextPage}
placeholder="Search a team"
emptyText="No teams found"
placeholder={t("users.defaultSettings.teams.teamPlaceholder")}
emptyText={t("users.defaultSettings.teams.teamEmpty")}
inputId={id}
aria-invalid={ariaInvalid}
aria-describedby={ariaDescribedBy}
@ -113,42 +116,53 @@ const TeamPickerField = ({ control, index }: { control: SettingsControl; index:
};
const TeamsField = ({ control }: { control: SettingsControl }) => {
const { t } = useTranslation();
const { fields, append, remove } = useFieldArray({ control, name: "teams" });
const teamRoleItems = TEAM_ROLE_OPTIONS.map((option) => ({ value: option.value, label: t(option.labelKey) }));
return (
<div className="flex w-full flex-col gap-3">
<div>
<p className="text-sm font-medium">Default Teams</p>
<p className="text-sm text-muted-foreground">
New users are added to these teams. Only teams that already exist can be selected.
</p>
<p className="text-sm font-medium">{t("users.defaultSettings.fields.teams")}</p>
<p className="text-sm text-muted-foreground">{t("users.defaultSettings.teams.description")}</p>
</div>
{fields.map((field, index) => (
<div key={field.id} className="rounded-lg border border-border p-4">
<div className="mb-3 flex items-center justify-between">
<p className="text-sm font-medium">Team {index + 1}</p>
<p className="text-sm font-medium">{t("users.defaultSettings.teams.rowTitle", { number: index + 1 })}</p>
<Button type="button" variant="destructive" size="sm" onClick={() => remove(index)}>
Remove
{t("users.defaultSettings.teams.remove")}
</Button>
</div>
<div className="grid grid-cols-1 gap-3 md:grid-cols-3">
<TeamPickerField control={control} index={index} />
<FormField control={control} name={`teams.${index}.max_budget_in_team`} label="Max Budget in Team (USD)">
<FormField
control={control}
name={`teams.${index}.max_budget_in_team`}
label={t("users.defaultSettings.teams.maxBudget")}
>
{({ ref, ...budgetField }) => (
<Input {...budgetField} ref={ref} type="number" step="any" min={0} placeholder="Optional" />
<Input
{...budgetField}
ref={ref}
type="number"
step="any"
min={0}
placeholder={t("users.defaultSettings.teams.maxBudgetPlaceholder")}
/>
)}
</FormField>
<FormField control={control} name={`teams.${index}.user_role`} label="Team Role">
<FormField
control={control}
name={`teams.${index}.user_role`}
label={t("users.defaultSettings.teams.role")}
>
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
<Select
items={TEAM_ROLE_OPTIONS}
value={value}
onValueChange={(selected) => onChange(selected ?? "user")}
>
<Select items={teamRoleItems} value={value} onValueChange={(selected) => onChange(selected ?? "user")}>
<SelectTrigger
id={id}
className="w-full"
@ -158,7 +172,7 @@ const TeamsField = ({ control }: { control: SettingsControl }) => {
<SelectValue />
</SelectTrigger>
<SelectContent>
{TEAM_ROLE_OPTIONS.map((option) => (
{teamRoleItems.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
</SelectItem>
@ -172,7 +186,7 @@ const TeamsField = ({ control }: { control: SettingsControl }) => {
))}
<Button type="button" variant="outline" onClick={() => append(EMPTY_TEAM_ROW)}>
Add Team
{t("users.defaultSettings.teams.add")}
</Button>
</div>
);
@ -191,30 +205,41 @@ interface SettingsViewProps {
}
const SettingsView = ({ values, roleOptions }: SettingsViewProps) => {
const { t } = useTranslation();
const roleLabel = roleOptions.find((option) => option.value === values.user_role)?.label ?? values.user_role;
const durationValue = values.budget_duration === "" ? NO_RESET : values.budget_duration;
const durationLabel =
BUDGET_DURATION_OPTIONS.find((option) => option.value === durationValue)?.label ?? values.budget_duration;
const durationOption = BUDGET_DURATION_OPTIONS.find((option) => option.value === durationValue);
const durationLabel = durationOption === undefined ? values.budget_duration : t(durationOption.labelKey);
const notSet = t("users.defaultSettings.view.notSet");
return (
<div className="flex flex-col gap-4">
<ViewRow label="Default Role">{roleLabel === "" ? "Not set" : roleLabel}</ViewRow>
<ViewRow label="Max Budget (USD)">{values.max_budget === "" ? "Not set" : values.max_budget}</ViewRow>
<ViewRow label="Reset Budget">{durationLabel}</ViewRow>
<ViewRow label="Default Models">
<ViewRow label={t("users.defaultSettings.fields.role")}>{roleLabel === "" ? notSet : roleLabel}</ViewRow>
<ViewRow label={t("users.defaultSettings.fields.maxBudget")}>
{values.max_budget === "" ? notSet : values.max_budget}
</ViewRow>
<ViewRow label={t("users.defaultSettings.fields.resetBudget")}>{durationLabel}</ViewRow>
<ViewRow label={t("users.defaultSettings.fields.models")}>
{values.models.length === 0
? "Not set"
: values.models.map((model) => MODEL_SENTINEL_LABELS.get(model) ?? model).join(", ")}
? notSet
: values.models
.map((model) => {
const labelKey = MODEL_SENTINEL_LABEL_KEYS.get(model);
return labelKey === undefined ? model : t(labelKey);
})
.join(", ")}
</ViewRow>
<div>
<p className="text-sm font-medium">Default Teams</p>
<p className="text-sm font-medium">{t("users.defaultSettings.fields.teams")}</p>
{values.teams.length === 0 ? (
<p className="text-sm text-muted-foreground">None</p>
<p className="text-sm text-muted-foreground">{t("users.defaultSettings.view.none")}</p>
) : (
values.teams.map((team) => (
<p key={team.team_id} className="text-sm text-muted-foreground">
{team.team_id}
{team.max_budget_in_team !== "" && <> · ${team.max_budget_in_team} max budget</>}
{team.max_budget_in_team !== "" && (
<> · {t("users.defaultSettings.view.teamMaxBudget", { budget: team.max_budget_in_team })}</>
)}
<> · {team.user_role}</>
</p>
))
@ -233,20 +258,25 @@ interface SettingsFormProps {
}
const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, onSaved }: SettingsFormProps) => {
const { t } = useTranslation();
const queryClient = useQueryClient();
const budgetDurationItems = BUDGET_DURATION_OPTIONS.map((option) => ({
value: option.value,
label: t(option.labelKey),
}));
const form = useZodForm(defaultUserSettingsSchema, { defaultValues: initialValues });
const { isDirty } = form.formState;
const mutation = useMutation({
mutationFn: (values: DefaultUserSettingsSubmitValues) => updateSettings(buildBody(values)),
onSuccess: (_result, values) => {
toast.success("Default user settings updated successfully");
toast.success(t("users.defaultSettings.toasts.updated"));
queryClient.invalidateQueries({ queryKey: SETTINGS_QUERY_KEY });
form.reset(values);
onSaved();
},
onError: (error: unknown) =>
toast.fromError(error instanceof Error ? error.message : "Failed to update default user settings"),
toast.fromError(error instanceof Error ? error.message : t("users.defaultSettings.toasts.updateFailed")),
});
const onSubmit = form.handleSubmit((values) => mutation.mutate(values));
@ -257,8 +287,8 @@ const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, on
<FormField
control={form.control}
name="user_role"
label="Default Role"
description="Role assigned to new users"
label={t("users.defaultSettings.fields.role")}
description={t("users.defaultSettings.fields.roleDescription")}
>
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
<Select
@ -267,7 +297,7 @@ const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, on
onValueChange={(selected) => onChange(selected ?? "")}
>
<SelectTrigger id={id} className="w-full" aria-invalid={ariaInvalid} aria-describedby={ariaDescribedBy}>
<SelectValue placeholder="Not set" />
<SelectValue placeholder={t("users.defaultSettings.fields.rolePlaceholder")} />
</SelectTrigger>
<SelectContent>
{roleOptions.map((option) => (
@ -286,8 +316,8 @@ const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, on
<FormField
control={form.control}
name="max_budget"
label="Max Budget (USD)"
description="Default maximum budget for new users"
label={t("users.defaultSettings.fields.maxBudget")}
description={t("users.defaultSettings.fields.maxBudgetDescription")}
>
{({ ref, ...field }) => <Input {...field} ref={ref} type="number" step="any" min={0} />}
</FormField>
@ -295,12 +325,12 @@ const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, on
<FormField
control={form.control}
name="budget_duration"
label="Reset Budget"
description="How often the default budget resets"
label={t("users.defaultSettings.fields.resetBudget")}
description={t("users.defaultSettings.fields.resetBudgetDescription")}
>
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
<Select
items={BUDGET_DURATION_OPTIONS}
items={budgetDurationItems}
value={value === "" ? NO_RESET : value}
onValueChange={(selected) => onChange(selected === null || selected === NO_RESET ? "" : selected)}
>
@ -308,7 +338,7 @@ const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, on
<SelectValue />
</SelectTrigger>
<SelectContent>
{BUDGET_DURATION_OPTIONS.map((option) => (
{budgetDurationItems.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
</SelectItem>
@ -321,8 +351,8 @@ const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, on
<FormField
control={form.control}
name="models"
label="Default Models"
description="Models new users can access"
label={t("users.defaultSettings.fields.models")}
description={t("users.defaultSettings.fields.modelsDescription")}
>
{(field) => (
<ModelSelect
@ -347,28 +377,29 @@ const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, on
}}
disabled={mutation.isPending}
>
Cancel
{t("users.defaultSettings.cancel")}
</Button>
<Button type="submit" disabled={!isDirty || mutation.isPending}>
{mutation.isPending ? "Saving..." : "Save Changes"}
{mutation.isPending ? t("users.defaultSettings.saving") : t("users.defaultSettings.save")}
</Button>
</div>
</form>
);
};
const SettingsCard = ({ action, children }: { action?: React.ReactNode; children: React.ReactNode }) => (
<Card>
<CardHeader>
<CardTitle>Default User Settings</CardTitle>
<CardDescription>
Applied to every new internal user created through SSO or the user management APIs.
</CardDescription>
{action !== undefined && <CardAction>{action}</CardAction>}
</CardHeader>
<CardContent>{children}</CardContent>
</Card>
);
const SettingsCard = ({ action, children }: { action?: React.ReactNode; children: React.ReactNode }) => {
const { t } = useTranslation();
return (
<Card>
<CardHeader>
<CardTitle>{t("users.defaultSettings.card.title")}</CardTitle>
<CardDescription>{t("users.defaultSettings.card.description")}</CardDescription>
{action !== undefined && <CardAction>{action}</CardAction>}
</CardHeader>
<CardContent>{children}</CardContent>
</Card>
);
};
export interface DefaultUserSettingsFormProps {
possibleUIRoles?: Record<string, Record<string, string>> | null;
@ -381,6 +412,7 @@ export const DefaultUserSettingsForm = ({
fetchSettings = defaultFetchSettings,
updateSettings = defaultUpdateSettings,
}: DefaultUserSettingsFormProps) => {
const { t } = useTranslation();
const [isEditing, setIsEditing] = React.useState(false);
const { data, isPending, isError } = useQuery({ queryKey: SETTINGS_QUERY_KEY, queryFn: fetchSettings });
@ -405,7 +437,7 @@ export const DefaultUserSettingsForm = ({
if (isError || initialValues === undefined) {
return (
<SettingsCard>
<p role="alert">Could not load the default user settings.</p>
<p role="alert">{t("users.defaultSettings.loadError")}</p>
</SettingsCard>
);
}
@ -415,7 +447,7 @@ export const DefaultUserSettingsForm = ({
action={
isEditing ? undefined : (
<Button type="button" onClick={() => setIsEditing(true)}>
Edit Settings
{t("users.defaultSettings.editSettings")}
</Button>
)
}

View file

@ -1,19 +1,24 @@
import { z } from "zod";
import i18n from "@/i18n/i18n";
const isBlank = (value: string): boolean => value.trim() === "";
const amountOrEmpty = z
.string()
.refine(
(value) => isBlank(value) || (Number.isFinite(Number(value)) && Number(value) >= 0),
"Must be a non-negative number",
);
.refine((value) => isBlank(value) || (Number.isFinite(Number(value)) && Number(value) >= 0), {
error: () => i18n.t("users.defaultSettings.errors.nonNegative"),
});
const defaultTeamRowSchema = z.object({
team_id: z
.string()
.nullable()
.pipe(z.string({ error: "Select a team" }).min(1, "Select a team")),
.pipe(
z
.string({ error: () => i18n.t("users.defaultSettings.errors.selectTeam") })
.min(1, { error: () => i18n.t("users.defaultSettings.errors.selectTeam") }),
),
max_budget_in_team: amountOrEmpty,
user_role: z.enum(["user", "admin"]),
});
@ -38,7 +43,7 @@ export const defaultUserSettingsSchema = z.object(defaultUserSettingsShape).supe
repeatedRows.forEach((index) =>
ctx.addIssue({
code: "custom",
message: "This team is already listed",
message: i18n.t("users.defaultSettings.errors.duplicateTeam"),
path: ["teams", index, "team_id"],
}),
);

View file

@ -1,4 +1,6 @@
import type { TFunction } from "i18next";
import React, { useMemo, useState } from "react";
import { useTranslation } from "react-i18next";
import { z } from "zod";
import { all_admin_roles } from "@/utils/roles";
import BudgetDurationDropdown from "@/components/common_components/budget_duration_dropdown";
@ -21,14 +23,16 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/comp
import { useZodForm } from "@/lib/forms/useZodForm";
import { CircleHelp } from "lucide-react";
const RATE_LIMIT_ERROR = "Enter a non-negative whole number, or leave empty for unlimited";
const isBlank = (value: string | number | null | undefined): boolean =>
value === null || value === undefined || String(value).trim() === "";
const rateLimitField = z
.union([z.string(), z.number()])
.nullish()
.transform((value) => (isBlank(value) ? null : Number(value)))
.pipe(z.number({ error: RATE_LIMIT_ERROR }).int(RATE_LIMIT_ERROR).nonnegative(RATE_LIMIT_ERROR).nullable());
const rateLimitField = (t: TFunction) => {
const rateLimitError = t("users.editView.errors.rateLimit");
return z
.union([z.string(), z.number()])
.nullish()
.transform((value) => (isBlank(value) ? null : Number(value)))
.pipe(z.number({ error: rateLimitError }).int(rateLimitError).nonnegative(rateLimitError).nullable());
};
interface UserEditViewProps {
userData: any;
@ -54,7 +58,7 @@ const MCP_SELECTION_SHAPE = z.object({
// The proxy stores unset user fields as null rather than leaving them out, and
// antd forwarded whatever it was handed, so nullish is what keeps a loaded user
// editable at all.
const userEditShape = {
const userEditShape = (t: TFunction) => ({
user_id: z.string().nullish(),
user_email: z.string().nullish(),
user_alias: z.string().nullish(),
@ -62,19 +66,19 @@ const userEditShape = {
models: z.array(z.string()),
budget_duration: z.string().nullish(),
metadata: z.string().nullish(),
tpm_limit: rateLimitField,
rpm_limit: rateLimitField,
tpm_limit: rateLimitField(t),
rpm_limit: rateLimitField(t),
mcp_servers_and_groups: MCP_SELECTION_SHAPE.optional(),
mcp_tool_permissions: z.record(z.string(), z.array(z.string())).optional(),
};
});
const userEditSchema = (unlimitedBudget: boolean) =>
const userEditSchema = (unlimitedBudget: boolean, t: TFunction) =>
z.object({
...userEditShape,
...userEditShape(t),
max_budget: z
.union([z.string(), z.number()])
.nullish()
.refine((value) => unlimitedBudget || !isBlank(value), "Please enter a budget or select Unlimited Budget"),
.refine((value) => unlimitedBudget || !isBlank(value), t("users.editView.errors.budgetRequired")),
});
type UserEditFormInput = z.input<ReturnType<typeof userEditSchema>>;
@ -160,13 +164,14 @@ export function UserEditView({
objectPermission,
premiumUser = false,
}: UserEditViewProps) {
const { t } = useTranslation();
const canEditMcpPermissions = !isBulkEdit && all_admin_roles.includes(userRole || "");
const [unlimitedBudget, setUnlimitedBudget] = useState(false);
const [modelMaxBudget, setModelMaxBudget] = useSeededState<ModelMaxBudget>(
userData.user_id,
() => userData.user_info?.model_max_budget ?? {},
);
const schema = useMemo(() => userEditSchema(unlimitedBudget), [unlimitedBudget]);
const schema = useMemo(() => userEditSchema(unlimitedBudget, t), [unlimitedBudget, t]);
const form = useZodForm(schema, {
defaultValues: toFormValues(userData, objectPermission, isBulkEdit, canEditMcpPermissions),
});
@ -207,8 +212,8 @@ export function UserEditView({
};
const modelOptions = [
{ label: "All Proxy Models", value: "all-proxy-models" },
{ label: "No Default Models", value: "no-default-models" },
{ label: t("users.editView.modelOptions.allProxyModels"), value: "all-proxy-models" },
{ label: t("users.editView.modelOptions.noDefaultModels"), value: "no-default-models" },
...userModels.map((model) => ({ label: getModelDisplayName(model), value: model })),
];
@ -223,18 +228,18 @@ export function UserEditView({
<form onSubmit={form.handleSubmit(handleSubmit)}>
<FieldGroup>
{!isBulkEdit && (
<FormField control={form.control} name="user_id" label="User ID">
<FormField control={form.control} name="user_id" label={t("users.editView.fields.userId")}>
{({ ref, value, ...control }) => <Input {...control} ref={ref} value={value ?? ""} disabled />}
</FormField>
)}
{!isBulkEdit && (
<FormField control={form.control} name="user_email" label="Email">
<FormField control={form.control} name="user_email" label={t("users.editView.fields.email")}>
{({ ref, value, ...control }) => <Input {...control} ref={ref} value={value ?? ""} />}
</FormField>
)}
<FormField control={form.control} name="user_alias" label="User Alias">
<FormField control={form.control} name="user_alias" label={t("users.editView.fields.userAlias")}>
{({ ref, value, ...control }) => <Input {...control} ref={ref} value={value ?? ""} />}
</FormField>
@ -242,8 +247,8 @@ export function UserEditView({
control={form.control}
name="user_role"
label={labelWithHint(
"Global Proxy Role",
"This is the role that the user will globally on the proxy. This role is independent of any team/org specific roles.",
t("users.editView.fields.globalProxyRole"),
t("users.editView.fields.globalProxyRoleHint"),
)}
>
{({ id, value, onChange }) => (
@ -271,8 +276,8 @@ export function UserEditView({
control={form.control}
name="models"
label={labelWithHint(
"Personal Models",
"Select which models this user can access outside of team-scope. Choose 'All Proxy Models' to grant access to all models available on the proxy.",
t("users.editView.fields.personalModels"),
t("users.editView.fields.personalModelsHint"),
)}
>
{({ value, onChange }) => (
@ -280,7 +285,7 @@ export function UserEditView({
options={modelOptions}
value={value}
onValueChange={onChange}
placeholder="Select models"
placeholder={t("users.editView.fields.modelsPlaceholder")}
disabled={!all_admin_roles.includes(userRole || "")}
/>
)}
@ -291,10 +296,10 @@ export function UserEditView({
name="max_budget"
label={
<>
Max Budget (USD)
{t("users.editView.fields.maxBudget")}
<label className="ml-3 inline-flex items-center gap-2 font-normal">
<Checkbox checked={unlimitedBudget} onCheckedChange={handleUnlimitedBudgetChange} />
Unlimited Budget
{t("users.editView.fields.unlimitedBudget")}
</label>
</>
}
@ -308,19 +313,19 @@ export function UserEditView({
value={value ?? ""}
onChange={(event) => onChange(event.target.value)}
onWheel={(event) => event.currentTarget.blur()}
placeholder="Enter a numerical value"
placeholder={t("users.editView.fields.maxBudgetPlaceholder")}
disabled={unlimitedBudget}
/>
)}
</FormField>
<FormField control={form.control} name="budget_duration" label="Reset Budget">
<FormField control={form.control} name="budget_duration" label={t("users.editView.fields.resetBudget")}>
{({ id, value, onChange }) => (
<BudgetDurationDropdown
id={id}
value={value}
onChange={onChange}
placeholder={isBulkEdit ? "No change" : undefined}
placeholder={isBulkEdit ? t("users.editView.fields.noChange") : undefined}
/>
)}
</FormField>
@ -330,10 +335,7 @@ export function UserEditView({
<FormField
control={form.control}
name="tpm_limit"
label={labelWithHint(
"TPM Limit",
"Applies across all keys owned by this user. Team and key limits still apply as ceilings.",
)}
label={labelWithHint(t("users.editView.fields.tpmLimit"), t("users.editView.fields.rateLimitHint"))}
>
{({ ref, value, onChange, ...control }) => (
<Input
@ -345,7 +347,7 @@ export function UserEditView({
value={value ?? ""}
onChange={(event) => onChange(event.target.value)}
onWheel={(event) => event.currentTarget.blur()}
placeholder="Unlimited"
placeholder={t("users.editView.fields.rateLimitPlaceholder")}
/>
)}
</FormField>
@ -353,10 +355,7 @@ export function UserEditView({
<FormField
control={form.control}
name="rpm_limit"
label={labelWithHint(
"RPM Limit",
"Applies across all keys owned by this user. Team and key limits still apply as ceilings.",
)}
label={labelWithHint(t("users.editView.fields.rpmLimit"), t("users.editView.fields.rateLimitHint"))}
>
{({ ref, value, onChange, ...control }) => (
<Input
@ -368,7 +367,7 @@ export function UserEditView({
value={value ?? ""}
onChange={(event) => onChange(event.target.value)}
onWheel={(event) => event.currentTarget.blur()}
placeholder="Unlimited"
placeholder={t("users.editView.fields.rateLimitPlaceholder")}
/>
)}
</FormField>
@ -385,13 +384,19 @@ export function UserEditView({
onChange={setModelMaxBudget}
availableModels={userModels}
usage={userData.user_info?.model_max_budget_usage}
hint="Cap this user's spend on individual models, each with its own reset window. Applies across every key the user holds."
hint={t("users.editView.fields.modelMaxBudgetHint")}
/>
)}
<FormField control={form.control} name="metadata" label="Metadata">
<FormField control={form.control} name="metadata" label={t("users.editView.fields.metadata")}>
{({ ref, value, ...control }) => (
<Textarea {...control} ref={ref} value={value ?? ""} rows={4} placeholder="Enter metadata as JSON" />
<Textarea
{...control}
ref={ref}
value={value ?? ""}
rows={4}
placeholder={t("users.editView.fields.metadataPlaceholder")}
/>
)}
</FormField>
@ -400,10 +405,7 @@ export function UserEditView({
<FormField
control={form.control}
name="mcp_servers_and_groups"
label={labelWithHint(
"MCP Servers / Access Groups",
"Caps which MCP servers, access groups, and tools this user may reach. Every key the user holds is limited to this set.",
)}
label={labelWithHint(t("users.editView.fields.mcpServers"), t("users.editView.fields.mcpServersHint"))}
>
{({ value, onChange }) => (
<MCPServerSelector
@ -411,7 +413,7 @@ export function UserEditView({
value={value}
accessToken={accessToken || ""}
userId={isBulkEdit ? null : userData.user_id}
placeholder="Select MCP servers or access groups (optional)"
placeholder={t("users.editView.fields.mcpServersPlaceholder")}
/>
)}
</FormField>
@ -430,9 +432,9 @@ export function UserEditView({
<div className="mt-6 flex justify-end gap-2">
<Button variant="secondary" type="button" onClick={onCancel}>
Cancel
{t("users.editView.actions.cancel")}
</Button>
<Button type="submit">Save Changes</Button>
<Button type="submit">{t("users.editView.actions.save")}</Button>
</div>
</form>
</TooltipProvider>

View file

@ -1,5 +1,6 @@
import { parseAsString, useQueryState } from "nuqs";
import React, { useCallback, useEffect, useMemo, useState } from "react";
import { useTranslation } from "react-i18next";
import BulkEditUserModal from "./BulkEditUsers";
import BulkCreateUsersButton from "@/components/bulk_create_users_button";
@ -58,6 +59,7 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
teams,
orgAdminOrgIds,
}) => {
const { t } = useTranslation();
const isProxyAdmin = userRole ? isProxyAdminRole(userRole) : false;
const queryClient = useQueryClient();
@ -157,19 +159,19 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
const handleResetPassword = useCallback(
async (userId: string) => {
if (!accessToken) {
toast.fromError("Access token not found");
toast.fromError(t("users.page.toasts.accessTokenMissing"));
return;
}
try {
toast.success("Generating password reset link...");
toast.success(t("users.page.toasts.generatingResetLink"));
const data = await invitationCreateCall(accessToken, userId);
setInvitationLinkData(data);
setIsInvitationLinkModalVisible(true);
} catch (error) {
toast.fromError("Failed to generate password reset link");
toast.fromError(t("users.page.toasts.resetLinkFailed"));
}
},
[accessToken],
[accessToken, t],
);
const confirmDelete = async () => {
@ -185,10 +187,10 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
return { ...previousData, users: updatedUsers };
});
toast.success("User deleted successfully");
toast.success(t("users.page.toasts.deleted"));
} catch (error) {
console.error("Error deleting user:", error);
toast.fromError("Failed to delete user");
toast.fromError(t("users.page.toasts.deleteFailed"));
} finally {
setIsDeleteModalOpen(false);
setUserToDelete(null);
@ -344,7 +346,7 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
variant={selectionMode ? "default" : "outline"}
data-testid="toggle-user-selection"
>
{selectionMode ? "Cancel Selection" : "Select Users"}
{selectionMode ? t("users.page.selection.cancel") : t("users.page.selection.start")}
</Button>
)}
@ -355,7 +357,7 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
disabled={selectedUsers.length === 0}
data-testid="bulk-edit-users"
>
Bulk Edit ({selectedUsers.length} selected)
{t("users.page.selection.bulkEdit", { count: selectedUsers.length })}
</Button>
)}
</>
@ -367,10 +369,10 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
<Tabs defaultValue="users" className="gap-0">
<TabsList variant="line" className="mb-4">
<TabsTrigger value="users" className="flex-none data-active:text-primary after:bg-primary">
Users
{t("users.page.tabs.users")}
</TabsTrigger>
<TabsTrigger value="default-settings" className="flex-none data-active:text-primary after:bg-primary">
Default User Settings
{t("users.page.tabs.defaultSettings")}
</TabsTrigger>
</TabsList>
@ -383,7 +385,7 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
<div
className="flex h-64 items-center justify-center"
role="status"
aria-label="Loading default user settings"
aria-label={t("users.page.defaultSettingsLoading")}
>
<div className="w-full max-w-lg space-y-3">
<Skeleton className="h-5 w-1/3" />
@ -404,20 +406,20 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
{/* Existing Modals */}
<DeleteResourceModal
isOpen={isDeleteModalOpen}
title="Delete User?"
message="Are you sure you want to delete this user? This action cannot be undone."
resourceInformationTitle="User Information"
title={t("users.page.deleteModal.title")}
message={t("users.page.deleteModal.message")}
resourceInformationTitle={t("users.page.deleteModal.informationTitle")}
resourceInformation={[
{ label: "Email", value: userToDelete?.user_email },
{ label: "User ID", value: userToDelete?.user_id, code: true },
{ label: t("users.page.deleteModal.email"), value: userToDelete?.user_email },
{ label: t("users.page.deleteModal.userId"), value: userToDelete?.user_id, code: true },
{
label: "Global Proxy Role",
label: t("users.page.deleteModal.globalProxyRole"),
value:
(userToDelete?.user_role && possibleUIRoles?.[userToDelete.user_role]?.ui_label) ||
userToDelete?.user_role ||
"-",
},
{ label: "Total Spend (USD)", value: userToDelete?.spend?.toFixed(2) },
{ label: t("users.page.deleteModal.totalSpend"), value: userToDelete?.spend?.toFixed(2) },
]}
onCancel={cancelDelete}
onOk={confirmDelete}

View file

@ -7,8 +7,10 @@ import {
RowSelectionState,
SortingState,
} from "@tanstack/react-table";
import type { ParseKeys } from "i18next";
import { Users } from "lucide-react";
import { useMemo, useState } from "react";
import { useTranslation } from "react-i18next";
import { UserInfo } from "@/components/networking";
import {
@ -50,21 +52,22 @@ interface UsersTableProps {
onResetPassword: (userId: string) => void;
}
const FILTER_LABELS: Record<string, string> = {
user_id: "User ID",
sso_user_id: "SSO ID",
user_role: "Role",
team: "Team",
const FILTER_LABEL_KEYS: Readonly<Record<string, ParseKeys>> = {
user_id: "users.table.filters.userId",
sso_user_id: "users.table.filters.ssoId",
user_role: "users.table.filters.role",
team: "users.table.filters.team",
};
function EmptyState() {
const { t } = useTranslation();
return (
<div className="flex flex-col items-center gap-1 py-6">
<div className="mb-1 flex size-10 items-center justify-center rounded-lg bg-muted">
<Users className="size-5 text-muted-foreground" />
</div>
<div className="text-sm font-medium text-foreground">No users found</div>
<div className="text-sm text-muted-foreground">Try adjusting your search or filters.</div>
<div className="text-sm font-medium text-foreground">{t("users.table.emptyState.title")}</div>
<div className="text-sm text-muted-foreground">{t("users.table.emptyState.description")}</div>
</div>
);
}
@ -91,6 +94,7 @@ export function UsersTable({
onDeleteUser,
onResetPassword,
}: UsersTableProps) {
const { t } = useTranslation();
const [filtersOpen, setFiltersOpen] = useState(false);
const columns = useMemo(() => {
@ -101,9 +105,15 @@ export function UsersTable({
onUserClick,
onDeleteUser,
onResetPassword,
t,
};
return getUsersTableColumns(columnDeps);
}, [possibleUIRoles, selectionEnabled, canViewUserUsage, onUserClick, onDeleteUser, onResetPassword]);
}, [possibleUIRoles, selectionEnabled, canViewUserUsage, onUserClick, onDeleteUser, onResetPassword, t]);
const filterLabels = useMemo(
() => Object.fromEntries(Object.entries(FILTER_LABEL_KEYS).map(([columnId, key]) => [columnId, t(key)])),
[t],
);
const roleOptions = useMemo(
() =>
@ -152,7 +162,7 @@ export function UsersTable({
rowSelection={rowSelection}
onRowSelectionChange={onRowSelectionChange}
isLoading={isLoading}
loadingMessage="Loading users…"
loadingMessage={t("users.table.loading")}
noDataMessage={<EmptyState />}
size="compact"
toolbar={(table) => (
@ -161,52 +171,52 @@ export function UsersTable({
table={table}
searchValue={searchValue}
onSearchChange={onSearchChange}
searchPlaceholder="Search by email or ID…"
searchPlaceholder={t("users.table.searchPlaceholder")}
onOpenFilters={() => setFiltersOpen(true)}
filterLabels={FILTER_LABELS}
filterLabels={filterLabels}
formatFilterValue={formatFilterValue}
/>
<DataTableFilterDrawer
table={table}
open={filtersOpen}
onOpenChange={setFiltersOpen}
title="Filters"
description="Narrow down your users"
title={t("users.table.filters.title")}
description={t("users.table.filters.description")}
>
{({ get, set }) => (
<>
<DataTableFilterField label="User ID">
<DataTableFilterField label={t("users.table.filters.userId")}>
<Input
value={(get("user_id") as string) ?? ""}
onChange={(event) => set("user_id", event.target.value)}
placeholder="Enter user ID…"
placeholder={t("users.table.filters.userIdPlaceholder")}
data-testid="users-filter-user-id"
/>
</DataTableFilterField>
<DataTableFilterField label="SSO ID">
<DataTableFilterField label={t("users.table.filters.ssoId")}>
<Input
value={(get("sso_user_id") as string) ?? ""}
onChange={(event) => set("sso_user_id", event.target.value)}
placeholder="Enter SSO ID…"
placeholder={t("users.table.filters.ssoIdPlaceholder")}
data-testid="users-filter-sso-id"
/>
</DataTableFilterField>
<DataTableFilterField label="Role">
<DataTableFilterField label={t("users.table.filters.role")}>
<SearchSelect
options={roleOptions}
value={(get("user_role") as string) || undefined}
onValueChange={(value) => set("user_role", value ?? undefined)}
placeholder="Select a role…"
emptyText="No roles found"
placeholder={t("users.table.filters.rolePlaceholder")}
emptyText={t("users.table.filters.roleEmpty")}
/>
</DataTableFilterField>
<DataTableFilterField label="Team">
<DataTableFilterField label={t("users.table.filters.team")}>
<SearchSelect
options={teamOptions}
value={(get("team") as string) || undefined}
onValueChange={(value) => set("team", value ?? undefined)}
placeholder="Select a team…"
emptyText="No teams found"
placeholder={t("users.table.filters.teamPlaceholder")}
emptyText={t("users.table.filters.teamEmpty")}
/>
</DataTableFilterField>
</>

View file

@ -1,8 +1,10 @@
"use client";
import { ColumnDef } from "@tanstack/react-table";
import type { TFunction } from "i18next";
import { ChartColumn, Copy, Info, KeyRound, MoreHorizontal, Pencil, Trash2 } from "lucide-react";
import Link from "next/link";
import { useTranslation } from "react-i18next";
import { UserInfo } from "@/components/networking";
import { createSelectionColumn, DataTableSortHeader } from "@/components/shared/DataTable";
@ -20,11 +22,6 @@ import { cn } from "@/lib/cva.config";
import { usageHrefForUser } from "@/app/(dashboard)/usage/_components/usageUrlState";
import { copyToClipboard } from "@/utils/dataUtils";
const SSO_ID_HINT =
"SSO ID is the ID of the user in the SSO provider. If the user is not using SSO, this will be null.";
const SCIM_INACTIVE_HINT = "Deactivated via SCIM (external identity provider). The user's virtual keys are blocked.";
function isScimInactive(user: UserInfo): boolean {
return (user.metadata as Record<string, unknown> | null | undefined)?.scim_active === false;
}
@ -38,10 +35,11 @@ interface UserRowActionsProps {
}
function UserRowActions({ user, canViewUserUsage, onUserClick, onDeleteUser, onResetPassword }: UserRowActionsProps) {
const { t } = useTranslation();
return (
<DropdownMenu>
<DropdownMenuTrigger
aria-label="Open user actions"
aria-label={t("users.table.rowActions.open")}
data-testid={`user-actions-${user.user_id}`}
className={cn(buttonVariants({ variant: "ghost", size: "icon-sm" }), "text-muted-foreground")}
>
@ -50,18 +48,18 @@ function UserRowActions({ user, canViewUserUsage, onUserClick, onDeleteUser, onR
<DropdownMenuContent align="end" className="w-48">
<DropdownMenuItem onClick={() => onUserClick(user.user_id, true)} data-testid="user-action-edit">
<Pencil />
Edit user
{t("users.table.rowActions.edit")}
</DropdownMenuItem>
<DropdownMenuItem onClick={() => onResetPassword(user.user_id)} data-testid="user-action-reset-password">
<KeyRound />
Reset password
{t("users.table.rowActions.resetPassword")}
</DropdownMenuItem>
<DropdownMenuItem
onClick={() => void copyToClipboard(user.user_id, "User ID copied")}
onClick={() => void copyToClipboard(user.user_id, t("users.table.rowActions.copied"))}
data-testid="user-action-copy"
>
<Copy />
Copy user ID
{t("users.table.rowActions.copyId")}
</DropdownMenuItem>
{canViewUserUsage && (
<DropdownMenuItem
@ -69,13 +67,13 @@ function UserRowActions({ user, canViewUserUsage, onUserClick, onDeleteUser, onR
data-testid="user-action-view-usage"
>
<ChartColumn />
View usage
{t("users.table.rowActions.viewUsage")}
</DropdownMenuItem>
)}
<DropdownMenuSeparator />
<DropdownMenuItem variant="destructive" onClick={() => onDeleteUser(user)} data-testid="user-action-delete">
<Trash2 />
Delete user
{t("users.table.rowActions.delete")}
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
@ -89,6 +87,7 @@ export interface UsersTableColumnsDeps {
onUserClick: (userId: string, openInEditMode?: boolean) => void;
onDeleteUser: (user: UserInfo) => void;
onResetPassword: (userId: string) => void;
t: TFunction;
}
export const getUsersTableColumns = ({
@ -98,13 +97,16 @@ export const getUsersTableColumns = ({
onUserClick,
onDeleteUser,
onResetPassword,
t,
}: UsersTableColumnsDeps): ColumnDef<UserInfo>[] => {
const baseColumns: ColumnDef<UserInfo>[] = [
{
id: "user_id",
accessorKey: "user_id",
meta: { title: "User ID" },
header: ({ column }) => <DataTableSortHeader column={column} title="User ID" variant="header-cycle" />,
meta: { title: t("users.table.columns.userId") },
header: ({ column }) => (
<DataTableSortHeader column={column} title={t("users.table.columns.userId")} variant="header-cycle" />
),
size: 220,
enableSorting: true,
cell: ({ row }) => (
@ -118,8 +120,10 @@ export const getUsersTableColumns = ({
{
id: "user_email",
accessorKey: "user_email",
meta: { title: "Email" },
header: ({ column }) => <DataTableSortHeader column={column} title="Email" variant="header-cycle" />,
meta: { title: t("users.table.columns.email") },
header: ({ column }) => (
<DataTableSortHeader column={column} title={t("users.table.columns.email")} variant="header-cycle" />
),
size: 220,
enableSorting: true,
cell: ({ row }) => (
@ -131,8 +135,8 @@ export const getUsersTableColumns = ({
{
id: "status",
accessorFn: (user) => (isScimInactive(user) ? "Inactive" : "Active"),
meta: { title: "Status", skeleton: "badge" },
header: ({ column }) => <DataTableSortHeader column={column} title="Status" />,
meta: { title: t("users.table.columns.status"), skeleton: "badge" },
header: ({ column }) => <DataTableSortHeader column={column} title={t("users.table.columns.status")} />,
sortDescFirst: false,
size: 110,
enableSorting: true,
@ -141,20 +145,28 @@ export const getUsersTableColumns = ({
return (
<StatusBadge
tone="error"
label="Inactive"
tooltip={SCIM_INACTIVE_HINT}
label={t("users.table.status.inactive")}
tooltip={t("users.table.status.scimInactiveHint")}
dataTestId={`user-status-${row.original.user_id}`}
/>
);
}
return <StatusBadge tone="success" label="Active" dataTestId={`user-status-${row.original.user_id}`} />;
return (
<StatusBadge
tone="success"
label={t("users.table.status.active")}
dataTestId={`user-status-${row.original.user_id}`}
/>
);
},
},
{
id: "user_role",
accessorKey: "user_role",
meta: { title: "Global Proxy Role" },
header: ({ column }) => <DataTableSortHeader column={column} title="Global Proxy Role" variant="header-cycle" />,
meta: { title: t("users.table.columns.globalProxyRole") },
header: ({ column }) => (
<DataTableSortHeader column={column} title={t("users.table.columns.globalProxyRole")} variant="header-cycle" />
),
size: 160,
enableSorting: true,
cell: ({ row }) => (
@ -166,8 +178,8 @@ export const getUsersTableColumns = ({
{
id: "user_alias",
accessorKey: "user_alias",
meta: { title: "User Alias" },
header: ({ column }) => <DataTableSortHeader column={column} title="User Alias" />,
meta: { title: t("users.table.columns.userAlias") },
header: ({ column }) => <DataTableSortHeader column={column} title={t("users.table.columns.userAlias")} />,
sortDescFirst: false,
size: 150,
enableSorting: true,
@ -180,8 +192,10 @@ export const getUsersTableColumns = ({
{
id: "spend",
accessorKey: "spend",
meta: { title: "Spend (USD)", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="Spend (USD)" variant="header-cycle" />,
meta: { title: t("users.table.columns.spend"), numeric: true },
header: ({ column }) => (
<DataTableSortHeader column={column} title={t("users.table.columns.spend")} variant="header-cycle" />
),
size: 130,
enableSorting: true,
cell: ({ row }) => <MoneyCell value={row.original.spend} decimals={2} />,
@ -189,23 +203,27 @@ export const getUsersTableColumns = ({
{
id: "max_budget",
accessorKey: "max_budget",
meta: { title: "Budget (USD)", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="Budget (USD)" />,
meta: { title: t("users.table.columns.budget"), numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title={t("users.table.columns.budget")} />,
sortDescFirst: true,
size: 130,
enableSorting: true,
cell: ({ row }) => <MoneyCell value={row.original.max_budget} decimals={2} emptyText="Unlimited" showZero />,
cell: ({ row }) => (
<MoneyCell value={row.original.max_budget} decimals={2} emptyText={t("users.table.unlimitedBudget")} showZero />
),
},
{
id: "sso_user_id",
accessorKey: "sso_user_id",
meta: { title: "SSO ID" },
meta: { title: t("users.table.columns.ssoId") },
header: ({ column }) => (
<span className="flex items-center gap-1.5">
<DataTableSortHeader column={column} title="SSO ID" />
<DataTableSortHeader column={column} title={t("users.table.columns.ssoId")} />
<CellTooltip
content={SSO_ID_HINT}
trigger={<Info className="size-3.5 shrink-0 text-muted-foreground" aria-label="About SSO ID" />}
content={t("users.table.ssoIdHint")}
trigger={
<Info className="size-3.5 shrink-0 text-muted-foreground" aria-label={t("users.table.ssoIdHintLabel")} />
}
/>
</span>
),
@ -221,8 +239,8 @@ export const getUsersTableColumns = ({
{
id: "key_count",
accessorKey: "key_count",
meta: { title: "Virtual Keys", skeleton: "badge" },
header: ({ column }) => <DataTableSortHeader column={column} title="Virtual Keys" />,
meta: { title: t("users.table.columns.virtualKeys"), skeleton: "badge" },
header: ({ column }) => <DataTableSortHeader column={column} title={t("users.table.columns.virtualKeys")} />,
sortDescFirst: true,
size: 120,
enableSorting: true,
@ -234,7 +252,7 @@ export const getUsersTableColumns = ({
variant="outline"
className="whitespace-nowrap border-indigo-200 bg-indigo-50 font-normal text-indigo-600 dark:border-indigo-800 dark:bg-indigo-950 dark:text-indigo-300"
>
{keyCount} {keyCount === 1 ? "Key" : "Keys"}
{keyCount} {t("users.table.keyCount", { count: keyCount })}
</Badge>
);
}
@ -243,7 +261,7 @@ export const getUsersTableColumns = ({
variant="outline"
className="whitespace-nowrap border-border bg-muted font-normal text-muted-foreground"
>
No Keys
{t("users.table.noKeys")}
</Badge>
);
},
@ -251,8 +269,10 @@ export const getUsersTableColumns = ({
{
id: "created_at",
accessorKey: "created_at",
meta: { title: "Created At" },
header: ({ column }) => <DataTableSortHeader column={column} title="Created At" variant="header-cycle" />,
meta: { title: t("users.table.columns.createdAt") },
header: ({ column }) => (
<DataTableSortHeader column={column} title={t("users.table.columns.createdAt")} variant="header-cycle" />
),
size: 130,
enableSorting: true,
cell: ({ row }) => <DateCell value={row.original.created_at} precision="date" />,
@ -260,8 +280,8 @@ export const getUsersTableColumns = ({
{
id: "updated_at",
accessorKey: "updated_at",
meta: { title: "Updated At" },
header: ({ column }) => <DataTableSortHeader column={column} title="Updated At" />,
meta: { title: t("users.table.columns.updatedAt") },
header: ({ column }) => <DataTableSortHeader column={column} title={t("users.table.columns.updatedAt")} />,
sortDescFirst: false,
size: 130,
enableSorting: true,
@ -269,8 +289,8 @@ export const getUsersTableColumns = ({
},
{
id: "actions",
meta: { title: "Actions", className: "text-right", headerClassName: "text-right" },
header: () => <span className="sr-only">Actions</span>,
meta: { title: t("users.table.columns.actions"), className: "text-right", headerClassName: "text-right" },
header: () => <span className="sr-only">{t("users.table.columns.actions")}</span>,
size: 60,
enableSorting: false,
enableHiding: false,
@ -294,7 +314,7 @@ export const getUsersTableColumns = ({
return [
createSelectionColumn<UserInfo>({
rowAriaLabel: (row) => `Select ${row.original.user_email || row.original.user_id}`,
rowAriaLabel: (row) => t("users.table.selectRow", { user: row.original.user_email || row.original.user_id }),
}),
...baseColumns,
];

View file

@ -1,4 +1,6 @@
import type { ParseKeys } from "i18next";
import React, { useState } from "react";
import { useTranslation } from "react-i18next";
import { Button } from "@/components/ui/button";
import { Card } from "@/components/ui/card";
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
@ -74,10 +76,10 @@ interface TeamOption {
team_alias: string;
}
const MEMBER_ROLE_OPTIONS = [
{ value: "user", hint: "Can view team info, but not manage it" },
{ value: "admin", hint: "Can create team keys, add members, and manage settings" },
] as const;
const MEMBER_ROLE_OPTIONS: ReadonlyArray<{ value: string; hintKey: ParseKeys }> = [
{ value: "user", hintKey: "users.userInfo.memberRoles.userHint" },
{ value: "admin", hintKey: "users.userInfo.memberRoles.adminHint" },
];
export default function UserInfoView({
userId,
@ -89,6 +91,7 @@ export default function UserInfoView({
initialTab = 0,
startInEditMode = false,
}: UserInfoViewProps) {
const { t } = useTranslation();
const { premiumUser, userId: signedInUserId } = useAuthorized();
const canViewAutoRouterUsage = hasProxyWideSpendView(userRole);
const canViewSavings = canViewAutoRouterUsage || (Boolean(userId.trim()) && userId === signedInUserId);
@ -159,14 +162,14 @@ export default function UserInfoView({
setUserModels(availableModels);
} catch (error) {
console.error("Error fetching user data:", error);
toast.fromError("Failed to fetch user data");
toast.fromError(t("users.userInfo.toasts.fetchFailed"));
} finally {
setIsLoading(false);
}
};
fetchData();
}, [accessToken, userId, userRole]);
}, [accessToken, userId, userRole, t]);
const isProxyAdmin = userRole === "proxy_admin" || userRole === "Admin";
@ -204,7 +207,7 @@ export default function UserInfoView({
user_id: userId,
};
await teamMemberAddCall(accessToken, selectedTeamId, member);
toast.success("User added to team successfully");
toast.success(t("users.userInfo.toasts.addedToTeam"));
setIsAddTeamModalOpen(false);
// Re-fetch user data to refresh teams
const data = await userGetInfoV2(accessToken, userId);
@ -224,7 +227,7 @@ export default function UserInfoView({
}
} catch (error: any) {
console.error("Error adding user to team:", error);
toast.fromError(error?.message || "Failed to add user to team");
toast.fromError(error?.message || t("users.userInfo.toasts.addToTeamFailed"));
} finally {
setIsAddingTeam(false);
}
@ -244,7 +247,7 @@ export default function UserInfoView({
user_id: userId,
};
await teamMemberDeleteCall(accessToken, teamToRemove.team_id, member);
toast.success("User removed from team successfully");
toast.success(t("users.userInfo.toasts.removedFromTeam"));
setIsRemoveTeamModalOpen(false);
setTeamToRemove(null);
// Re-fetch user data to refresh teams
@ -265,7 +268,7 @@ export default function UserInfoView({
}
} catch (error: any) {
console.error("Error removing user from team:", error);
toast.fromError(error?.message || "Failed to remove user from team");
toast.fromError(error?.message || t("users.userInfo.toasts.removeFromTeamFailed"));
} finally {
setIsRemovingTeam(false);
}
@ -282,16 +285,16 @@ export default function UserInfoView({
const handleResetPassword = async () => {
if (!accessToken) {
toast.fromError("Access token not found");
toast.fromError(t("users.userInfo.toasts.accessTokenMissing"));
return;
}
try {
toast.success("Generating password reset link...");
toast.success(t("users.userInfo.toasts.generatingResetLink"));
const data = await invitationCreateCall(accessToken, userId);
setInvitationLinkData(data);
setIsInvitationLinkModalVisible(true);
} catch (error) {
toast.fromError("Failed to generate password reset link");
toast.fromError(t("users.userInfo.toasts.resetLinkFailed"));
}
};
@ -300,14 +303,14 @@ export default function UserInfoView({
if (!accessToken) return;
setIsDeletingUser(true);
await userDeleteCall(accessToken, [userId]);
toast.success("User deleted successfully");
toast.success(t("users.userInfo.toasts.deleted"));
if (onDelete) {
onDelete();
}
onClose();
} catch (error) {
console.error("Error deleting user:", error);
toast.fromError("Failed to delete user");
toast.fromError(t("users.userInfo.toasts.deleteFailed"));
} finally {
setIsDeleteModalOpen(false);
setIsDeletingUser(false);
@ -353,11 +356,11 @@ export default function UserInfoView({
: userData.object_permission,
});
toast.success("User updated successfully");
toast.success(t("users.userInfo.toasts.updated"));
setIsEditing(false);
} catch (error) {
console.error("Error updating user:", error);
toast.fromError("Failed to update user");
toast.fromError(t("users.userInfo.toasts.updateFailed"));
}
};
@ -366,9 +369,9 @@ export default function UserInfoView({
<div className="p-4">
<Button variant="ghost" onClick={onClose} className="mb-4">
<ArrowLeft />
Back to Users
{t("users.userInfo.backToUsers")}
</Button>
<p className="text-sm">Loading user data...</p>
<p className="text-sm">{t("users.userInfo.loading")}</p>
</div>
);
}
@ -378,9 +381,9 @@ export default function UserInfoView({
<div className="p-4">
<Button variant="ghost" onClick={onClose} className="mb-4">
<ArrowLeft />
Back to Users
{t("users.userInfo.backToUsers")}
</Button>
<p className="text-sm">User not found</p>
<p className="text-sm">{t("users.userInfo.notFound")}</p>
</div>
);
}
@ -395,6 +398,12 @@ export default function UserInfoView({
}
};
const unknownTimestamp = t("users.userInfo.details.unknown");
const unlimitedLabel = t("users.userInfo.unlimited");
const budgetLimit =
userData.max_budget !== null ? `$${formatNumberWithCommas(userData.max_budget, 2)}` : unlimitedLabel;
const teamPickerPlaceholder = t("users.userInfo.addTeamDialog.teamPlaceholder");
// Build a legacy-compatible shape for UserEditView
const userDataForEdit = {
user_id: userData.user_id,
@ -421,9 +430,9 @@ export default function UserInfoView({
<div>
<Button variant="ghost" onClick={onClose} className="mb-4">
<ArrowLeft />
Back to Users
{t("users.userInfo.backToUsers")}
</Button>
<h2 className="text-xl font-semibold">{userData.user_email || "User"}</h2>
<h2 className="text-xl font-semibold">{userData.user_email || t("users.userInfo.fallbackTitle")}</h2>
<div className="flex items-center cursor-pointer">
<span className="text-sm text-muted-foreground font-mono">{userData.user_id}</span>
<Button
@ -444,7 +453,7 @@ export default function UserInfoView({
<div className="flex items-center space-x-2">
<Button variant="secondary" onClick={handleResetPassword} className="flex items-center">
<RefreshCw />
Reset Password
{t("users.userInfo.actions.resetPassword")}
</Button>
<Button
variant="secondary"
@ -452,7 +461,7 @@ export default function UserInfoView({
className="flex items-center text-destructive border-destructive hover:bg-destructive/10"
>
<Trash2 />
Delete User
{t("users.userInfo.actions.deleteUser")}
</Button>
</div>
)}
@ -460,18 +469,18 @@ export default function UserInfoView({
<DeleteResourceModal
isOpen={isDeleteModalOpen}
title="Delete User?"
message="Are you sure you want to delete this user? This action cannot be undone."
resourceInformationTitle="User Information"
title={t("users.userInfo.deleteModal.title")}
message={t("users.userInfo.deleteModal.message")}
resourceInformationTitle={t("users.userInfo.deleteModal.informationTitle")}
resourceInformation={[
{ label: "Email", value: userData.user_email },
{ label: "User ID", value: userData.user_id, code: true },
{ label: t("users.userInfo.deleteModal.email"), value: userData.user_email },
{ label: t("users.userInfo.deleteModal.userId"), value: userData.user_id, code: true },
{
label: "Global Proxy Role",
label: t("users.userInfo.deleteModal.globalProxyRole"),
value: (userData.user_role && possibleUIRoles?.[userData.user_role]?.ui_label) || userData.user_role || "-",
},
{
label: "Total Spend (USD)",
label: t("users.userInfo.deleteModal.totalSpend"),
value: userData.spend !== null && userData.spend !== undefined ? userData.spend.toFixed(2) : undefined,
},
]}
@ -487,19 +496,19 @@ export default function UserInfoView({
>
<TabsList variant="line" className="mb-4">
<TabsTrigger value="overview" className="flex-none data-active:text-primary after:bg-primary">
Overview
{t("users.userInfo.tabs.overview")}
</TabsTrigger>
<TabsTrigger value="details" className="flex-none data-active:text-primary after:bg-primary">
Details
{t("users.userInfo.tabs.details")}
</TabsTrigger>
{canViewSavings && (
<TabsTrigger value="savings" className="flex-none data-active:text-primary after:bg-primary">
Savings
{t("users.userInfo.tabs.savings")}
</TabsTrigger>
)}
{canViewAutoRouterUsage && (
<TabsTrigger value="auto-router-usage" className="flex-none data-active:text-primary after:bg-primary">
Auto-router usage
{t("users.userInfo.tabs.autoRouterUsage")}
</TabsTrigger>
)}
</TabsList>
@ -509,7 +518,7 @@ export default function UserInfoView({
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-6">
<Card className="block p-6">
<p>
Spend
{t("users.userInfo.overview.spend")}
<BudgetPeriodTooltip
spend={userData.spend}
maxBudget={userData.max_budget}
@ -519,19 +528,17 @@ export default function UserInfoView({
</p>
<div className="mt-2">
<h3 className="text-lg font-medium">${formatNumberWithCommas(userData.spend || 0, 2)}</h3>
<p>
of {userData.max_budget !== null ? `$${formatNumberWithCommas(userData.max_budget, 2)}` : "Unlimited"}
</p>
<p>{t("users.userInfo.overview.budgetOf", { budget: budgetLimit })}</p>
</div>
</Card>
<Card className="block p-6">
<div className="flex justify-between items-center mb-2">
<p>Teams</p>
<p>{t("users.userInfo.overview.teams")}</p>
{isProxyAdmin && (
<Button variant="ghost" size="sm" onClick={handleOpenAddTeamModal}>
<Plus />
Add Team
{t("users.userInfo.overview.addTeam")}
</Button>
)}
</div>
@ -541,8 +548,10 @@ export default function UserInfoView({
<Table>
<TableHeader>
<TableRow>
<TableHead>Team Name</TableHead>
{isProxyAdmin && <TableHead className="text-right">Actions</TableHead>}
<TableHead>{t("users.userInfo.overview.teamName")}</TableHead>
{isProxyAdmin && (
<TableHead className="text-right">{t("users.userInfo.overview.actions")}</TableHead>
)}
</TableRow>
</TableHeader>
<TableBody>
@ -558,7 +567,9 @@ export default function UserInfoView({
<Button
variant="ghost"
size="icon-sm"
aria-label={`Remove from ${team.team_alias || team.team_id}`}
aria-label={t("users.userInfo.overview.removeFromTeam", {
team: team.team_alias || team.team_id,
})}
onClick={() => handleOpenRemoveTeamModal(team)}
className="text-destructive"
>
@ -572,28 +583,28 @@ export default function UserInfoView({
</Table>
</div>
) : (
<p>No teams</p>
<p>{t("users.userInfo.overview.noTeams")}</p>
)}
{!isTeamsExpanded && teamDetails.length > 20 && (
<Button variant="ghost" size="sm" className="mt-2" onClick={() => setIsTeamsExpanded(true)}>
+{teamDetails.length - 20} more
{t("users.userInfo.overview.moreTeams", { count: teamDetails.length - 20 })}
</Button>
)}
{isTeamsExpanded && teamDetails.length > 20 && (
<Button variant="ghost" size="sm" className="mt-2" onClick={() => setIsTeamsExpanded(false)}>
Show Less
{t("users.userInfo.overview.showLess")}
</Button>
)}
</div>
</Card>
<Card className="block p-6">
<p>Personal Models</p>
<p>{t("users.userInfo.personalModels")}</p>
<div className="mt-2">
{userData.models?.length && userData.models?.length > 0 ? (
userData.models?.map((model, index) => <p key={index}>{model}</p>)
) : (
<p>All proxy models</p>
<p>{t("users.userInfo.allProxyModels")}</p>
)}
</div>
</Card>
@ -604,9 +615,9 @@ export default function UserInfoView({
<TabsContent value="details" keepMounted>
<Card className="block p-6">
<div className="flex justify-between items-center mb-4">
<h3 className="text-lg font-medium">User Settings</h3>
<h3 className="text-lg font-medium">{t("users.userInfo.details.title")}</h3>
{!isEditing && userRole && rolesWithWriteAccess.includes(userRole) && (
<Button onClick={() => setIsEditing(true)}>Edit Settings</Button>
<Button onClick={() => setIsEditing(true)}>{t("users.userInfo.details.editSettings")}</Button>
)}
</div>
@ -627,7 +638,7 @@ export default function UserInfoView({
) : (
<div className="space-y-4">
<div>
<p className="font-medium">User ID</p>
<p className="font-medium">{t("users.userInfo.details.userId")}</p>
<div className="flex items-center cursor-pointer">
<span className="font-mono">{userData.user_id}</span>
<Button
@ -646,32 +657,32 @@ export default function UserInfoView({
</div>
<div>
<p className="font-medium">Email</p>
<p>{userData.user_email || "Not Set"}</p>
<p className="font-medium">{t("users.userInfo.details.email")}</p>
<p>{userData.user_email || t("users.userInfo.details.notSet")}</p>
</div>
<div>
<p className="font-medium">User Alias</p>
<p>{userData.user_alias || "Not Set"}</p>
<p className="font-medium">{t("users.userInfo.details.userAlias")}</p>
<p>{userData.user_alias || t("users.userInfo.details.notSet")}</p>
</div>
<div>
<p className="font-medium">Global Proxy Role</p>
<p>{userData.user_role || "Not Set"}</p>
<p className="font-medium">{t("users.userInfo.details.globalProxyRole")}</p>
<p>{userData.user_role || t("users.userInfo.details.notSet")}</p>
</div>
<div>
<p className="font-medium">Created</p>
<p>{userData.created_at ? new Date(userData.created_at).toLocaleString() : "Unknown"}</p>
<p className="font-medium">{t("users.userInfo.details.created")}</p>
<p>{userData.created_at ? new Date(userData.created_at).toLocaleString() : unknownTimestamp}</p>
</div>
<div>
<p className="font-medium">Last Updated</p>
<p>{userData.updated_at ? new Date(userData.updated_at).toLocaleString() : "Unknown"}</p>
<p className="font-medium">{t("users.userInfo.details.lastUpdated")}</p>
<p>{userData.updated_at ? new Date(userData.updated_at).toLocaleString() : unknownTimestamp}</p>
</div>
<div>
<p className="font-medium">Personal Models</p>
<p className="font-medium">{t("users.userInfo.personalModels")}</p>
<div className="flex flex-wrap gap-2 mt-1">
{userData.models?.length && userData.models?.length > 0 ? (
userData.models?.map((model, index) => (
@ -680,34 +691,34 @@ export default function UserInfoView({
</span>
))
) : (
<p>All proxy models</p>
<p>{t("users.userInfo.allProxyModels")}</p>
)}
</div>
</div>
<div>
<p className="font-medium">Max Budget</p>
<p className="font-medium">{t("users.userInfo.details.maxBudget")}</p>
<p>
{userData.max_budget !== null && userData.max_budget !== undefined
? `$${formatNumberWithCommas(userData.max_budget, 4)}`
: "Unlimited"}
: unlimitedLabel}
</p>
</div>
<div>
<p className="font-medium">Budget Reset</p>
<p className="font-medium">{t("users.userInfo.details.budgetReset")}</p>
<p>{getBudgetDurationLabel(userData.budget_duration ?? null)}</p>
</div>
<div>
<p className="font-medium">Metadata</p>
<p className="font-medium">{t("users.userInfo.details.metadata")}</p>
<pre className="bg-muted p-2 rounded-sm text-xs overflow-auto mt-1">
{JSON.stringify(userData.metadata || {}, null, 2)}
</pre>
</div>
<div>
<p className="font-medium mb-2">MCP Permissions</p>
<p className="font-medium mb-2">{t("users.userInfo.details.mcpPermissions")}</p>
<MCPServerPermissions
mcpServers={userData.object_permission?.mcp_servers || []}
mcpAccessGroups={userData.object_permission?.mcp_access_groups || []}
@ -730,10 +741,10 @@ export default function UserInfoView({
scope={{ userId }}
activity={activityDateRange}
entityType="user"
scopeNote="Savings for this user across API keys and JWT-authenticated requests."
scopeNote={t("users.userInfo.savings.scopeNote")}
/>
) : (
<p role="alert">Savings are unavailable because this user has no ID.</p>
<p role="alert">{t("users.userInfo.savings.unavailable")}</p>
))}
</TabsContent>
)}
@ -748,7 +759,7 @@ export default function UserInfoView({
activity={activityDateRange}
/>
) : (
<p role="alert">Auto-router usage is unavailable because this user has no ID.</p>
<p role="alert">{t("users.userInfo.autoRouter.unavailable")}</p>
))}
</TabsContent>
)}
@ -764,14 +775,14 @@ export default function UserInfoView({
{/* Delete Team Member Modal */}
<DeleteResourceModal
isOpen={isRemoveTeamModalOpen}
title="Remove from Team"
alertMessage="Removing this user from the team will also delete any keys the user created for this team."
message="Are you sure you want to remove this user from the team? This action cannot be undone."
resourceInformationTitle="Team Membership"
title={t("users.userInfo.removeTeamModal.title")}
alertMessage={t("users.userInfo.removeTeamModal.alert")}
message={t("users.userInfo.removeTeamModal.message")}
resourceInformationTitle={t("users.userInfo.removeTeamModal.informationTitle")}
resourceInformation={[
{ label: "Team", value: teamToRemove?.team_alias || teamToRemove?.team_id },
{ label: "User ID", value: userData?.user_id, code: true },
{ label: "Email", value: userData?.user_email },
{ label: t("users.userInfo.removeTeamModal.team"), value: teamToRemove?.team_alias || teamToRemove?.team_id },
{ label: t("users.userInfo.removeTeamModal.userId"), value: userData?.user_id, code: true },
{ label: t("users.userInfo.removeTeamModal.email"), value: userData?.user_email },
]}
onCancel={handleRemoveTeamCancel}
onOk={handleRemoveTeamConfirm}
@ -786,7 +797,7 @@ export default function UserInfoView({
>
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[500px]">
<DialogHeader>
<DialogTitle>Add User to Team</DialogTitle>
<DialogTitle>{t("users.userInfo.addTeamDialog.title")}</DialogTitle>
</DialogHeader>
<form
onSubmit={(event) => {
@ -796,7 +807,7 @@ export default function UserInfoView({
>
<FieldGroup>
<Field>
<FieldLabel htmlFor={ADD_TEAM_FIELD_ID}>Team</FieldLabel>
<FieldLabel htmlFor={ADD_TEAM_FIELD_ID}>{t("users.userInfo.addTeamDialog.team")}</FieldLabel>
<Combobox
items={availableTeamsForAdd}
value={selectedTeamOption}
@ -804,9 +815,9 @@ export default function UserInfoView({
itemToStringLabel={(team: TeamOption) => team.team_alias}
isItemEqualToValue={(team: TeamOption, value: TeamOption) => team.team_id === value.team_id}
>
<ComboboxInput id={ADD_TEAM_FIELD_ID} placeholder="Select a team" className="w-full" />
<ComboboxInput id={ADD_TEAM_FIELD_ID} placeholder={teamPickerPlaceholder} className="w-full" />
<ComboboxContent>
<ComboboxEmpty>No teams found</ComboboxEmpty>
<ComboboxEmpty>{t("users.userInfo.addTeamDialog.noTeams")}</ComboboxEmpty>
<ComboboxList>
{(team: TeamOption) => (
<ComboboxItem key={team.team_id} value={team} title={team.team_alias}>
@ -819,7 +830,7 @@ export default function UserInfoView({
</Field>
<Field>
<FieldLabel htmlFor={ADD_TEAM_ROLE_FIELD_ID}>Member Role</FieldLabel>
<FieldLabel htmlFor={ADD_TEAM_ROLE_FIELD_ID}>{t("users.userInfo.addTeamDialog.memberRole")}</FieldLabel>
<Select value={selectedRole} onValueChange={(value) => value !== null && setSelectedRole(value)}>
<SelectTrigger id={ADD_TEAM_ROLE_FIELD_ID} className="w-full">
<SelectValue />
@ -827,9 +838,9 @@ export default function UserInfoView({
<SelectContent>
{MEMBER_ROLE_OPTIONS.map((option) => (
<SelectItem key={option.value} value={option.value} title={option.value}>
<SimpleTooltip content={option.hint}>
<SimpleTooltip content={t(option.hintKey)}>
<span className="font-medium">{option.value}</span>
<span className="ml-2 text-muted-foreground text-sm">- {option.hint}</span>
<span className="ml-2 text-muted-foreground text-sm">- {t(option.hintKey)}</span>
</SimpleTooltip>
</SelectItem>
))}
@ -840,7 +851,7 @@ export default function UserInfoView({
<div className="text-right mt-4">
<Button type="submit" disabled={isAddingTeam || !selectedTeamId} aria-busy={isAddingTeam}>
{isAddingTeam ? "Adding..." : "Add to Team"}
{isAddingTeam ? t("users.userInfo.addTeamDialog.submitting") : t("users.userInfo.addTeamDialog.submit")}
</Button>
</div>
</form>

View file

@ -15,6 +15,7 @@ import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { ChevronRight, CircleHelp, Info, UserPlus } from "lucide-react";
import React, { useEffect, useState } from "react";
import { useForm } from "react-hook-form";
import { useTranslation } from "react-i18next";
import TeamDropdown from "./common_components/team_dropdown";
import { getModelDisplayName } from "./key_team_helpers/fetch_available_models_team_key";
import { toast } from "@/lib/toast";
@ -134,18 +135,21 @@ const labelWithHint = (label: string, hint: string): React.ReactNode => (
</>
);
const EmailInvitationsNotice: React.FC = () => (
<Alert variant="info" className="mb-4">
<Info />
<AlertTitle>Email invitations</AlertTitle>
<AlertDescription>
New users receive an email invite only when an email integration (SMTP, Resend, or SendGrid) is configured.{" "}
<a href="https://docs.litellm.ai/docs/proxy/email" target="_blank" rel="noreferrer">
Learn how to set up email notifications
</a>
</AlertDescription>
</Alert>
);
const EmailInvitationsNotice: React.FC = () => {
const { t } = useTranslation();
return (
<Alert variant="info" className="mb-4">
<Info />
<AlertTitle>{t("users.createUser.emailNotice.title")}</AlertTitle>
<AlertDescription>
{t("users.createUser.emailNotice.body")}{" "}
<a href="https://docs.litellm.ai/docs/proxy/email" target="_blank" rel="noreferrer">
{t("users.createUser.emailNotice.docsLink")}
</a>
</AlertDescription>
</Alert>
);
};
export const CreateUserButton: React.FC<CreateuserProps> = ({
userID,
@ -154,6 +158,7 @@ export const CreateUserButton: React.FC<CreateuserProps> = ({
onUserCreated,
isEmbedded = false,
}) => {
const { t } = useTranslation();
const queryClient = useQueryClient();
const [uiSettings, setUISettings] = useState<UISettings | null>(null);
const defaultValues = isEmbedded ? EMBEDDED_DEFAULTS : STANDALONE_DEFAULTS;
@ -201,7 +206,7 @@ export const CreateUserButton: React.FC<CreateuserProps> = ({
const handleCreate = async (formValues: CreateUserFormValues) => {
try {
toast.info("Making API Call");
toast.info(t("users.createUser.toasts.makingApiCall"));
if (!isEmbedded) {
setIsModalVisible(true);
}
@ -228,11 +233,11 @@ export const CreateUserButton: React.FC<CreateuserProps> = ({
setIsInvitationLinkModalVisible(true);
}
toast.success("API user Created");
toast.success(t("users.createUser.toasts.created"));
form.reset(defaultValues);
localStorage.removeItem("userData" + userID);
} catch (error: any) {
const errorMessage = error.response?.data?.detail || error?.message || "Error creating the user";
const errorMessage = error.response?.data?.detail || error?.message || t("users.createUser.toasts.createFailed");
toast.fromError(errorMessage);
console.error("Error creating the user:", error);
}
@ -245,7 +250,7 @@ export const CreateUserButton: React.FC<CreateuserProps> = ({
}));
const userEmailField = (
<FormField control={form.control} name="user_email" label="User Email">
<FormField control={form.control} name="user_email" label={t("users.createUser.fields.userEmail")}>
{({ ref, value, ...control }) => <Input {...control} ref={ref} value={value ?? ""} />}
</FormField>
);
@ -254,23 +259,34 @@ export const CreateUserButton: React.FC<CreateuserProps> = ({
<FormField
control={form.control}
name="team_id"
label="Team"
description="If selected, user will be added as a 'user' role to the team."
label={t("users.createUser.fields.team")}
description={t("users.createUser.fields.teamDescription")}
>
{({ id, value, onChange }) => <TeamDropdown id={id} value={value} onChange={onChange} />}
</FormField>
);
const metadataField = (
<FormField control={form.control} name="metadata" label="Metadata">
<FormField control={form.control} name="metadata" label={t("users.createUser.fields.metadata")}>
{({ ref, value, ...control }) => (
<Textarea {...control} ref={ref} value={value ?? ""} rows={4} placeholder="Enter metadata as JSON" />
<Textarea
{...control}
ref={ref}
value={value ?? ""}
rows={4}
placeholder={t("users.createUser.fields.metadataPlaceholder")}
/>
)}
</FormField>
);
const sendInviteEmailField = (
<FormField control={form.control} name="send_invite_email" label="Send invitation email" orientation="horizontal">
<FormField
control={form.control}
name="send_invite_email"
label={t("users.createUser.fields.sendInviteEmail")}
orientation="horizontal"
>
{({ id, value, onChange, onBlur }) => (
<Checkbox id={id} checked={value} onCheckedChange={onChange} onBlur={onBlur} />
)}
@ -309,13 +325,13 @@ export const CreateUserButton: React.FC<CreateuserProps> = ({
<EmailInvitationsNotice />
<FieldGroup>
{userEmailField}
{roleField("User Role")}
{roleField(t("users.createUser.fields.userRole"))}
{teamField}
{metadataField}
{sendInviteEmailField}
</FieldGroup>
<div className="mt-4 text-right">
<Button type="submit">Create User</Button>
<Button type="submit">{t("users.createUser.createButton")}</Button>
</div>
</form>
</TooltipProvider>
@ -326,15 +342,15 @@ export const CreateUserButton: React.FC<CreateuserProps> = ({
return (
<>
<Button type="button" onClick={() => setIsModalVisible(true)}>
+ Invite User
{t("users.createUser.openButton")}
</Button>
<Dialog open={isModalVisible} onOpenChange={(open) => !open && handleCancel()}>
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[800px]">
<DialogHeader>
<DialogTitle>Invite User</DialogTitle>
<DialogTitle>{t("users.createUser.title")}</DialogTitle>
</DialogHeader>
<div className="flex flex-col gap-3">
<p className="mb-1 text-sm text-foreground">Create a User who can own keys</p>
<p className="mb-1 text-sm text-foreground">{t("users.createUser.subtitle")}</p>
<EmailInvitationsNotice />
</div>
<TooltipProvider>
@ -342,18 +358,15 @@ export const CreateUserButton: React.FC<CreateuserProps> = ({
<FieldGroup>
{userEmailField}
{roleField(
labelWithHint(
"Global Proxy Role",
"This role is independent of any team/org specific roles. Configure Team / Organization Admins in the Settings",
),
labelWithHint(t("users.createUser.fields.globalRole"), t("users.createUser.fields.globalRoleHint")),
)}
{teamField}
<FormField
control={form.control}
name="organization_ids"
label="Organization"
description="The user will be added to the selected organization(s)."
label={t("users.createUser.fields.organization")}
description={t("users.createUser.fields.organizationDescription")}
>
{({ id, value, onChange }) => (
<Select
@ -363,10 +376,10 @@ export const CreateUserButton: React.FC<CreateuserProps> = ({
onValueChange={(selected: string[]) => onChange(selected.length === 0 ? undefined : selected)}
>
<SelectTrigger id={id} className="w-full">
<SelectValue placeholder="Select Organization">
<SelectValue placeholder={t("users.createUser.fields.organizationPlaceholder")}>
{(selected: string[]) =>
selected.length === 0
? "Select Organization"
? t("users.createUser.fields.organizationPlaceholder")
: organizationOptions
.filter((option) => selected.includes(option.value))
.map((option) => option.label)
@ -394,25 +407,28 @@ export const CreateUserButton: React.FC<CreateuserProps> = ({
className={`size-4 transition-transform ${isPersonalKeyOpen ? "rotate-90" : ""}`}
aria-hidden
/>
Personal Key Creation
{t("users.createUser.fields.personalKeyCreation")}
</CollapsibleTrigger>
<CollapsibleContent className="pt-4">
<FormField
control={form.control}
name="models"
label={labelWithHint("Models", "Models user has access to, outside of team scope.")}
description="Models user has access to, outside of team scope."
label={labelWithHint(
t("users.createUser.fields.models"),
t("users.createUser.fields.modelsHint"),
)}
description={t("users.createUser.fields.modelsDescription")}
>
{({ value, onChange }) => (
<MultiSelect
options={[
{ label: "All Proxy Models", value: "all-proxy-models" },
{ label: "No Default Models", value: "no-default-models" },
{ label: t("users.createUser.fields.allProxyModels"), value: "all-proxy-models" },
{ label: t("users.createUser.fields.noDefaultModels"), value: "no-default-models" },
...userModels.map((model) => ({ label: getModelDisplayName(model), value: model })),
]}
value={value ?? []}
onValueChange={onChange}
placeholder="Select models"
placeholder={t("users.createUser.fields.modelsPlaceholder")}
/>
)}
</FormField>
@ -423,7 +439,7 @@ export const CreateUserButton: React.FC<CreateuserProps> = ({
<div className="mt-4 text-right">
<Button type="submit">
<UserPlus />
Invite User
{t("users.createUser.submitButton")}
</Button>
</div>
</form>

View file

@ -17,18 +17,20 @@ import {
ComboboxValue,
useComboboxAnchor,
} from "@/components/ui/combobox";
import type { TFunction } from "i18next";
import { useTranslation } from "react-i18next";
import { Skeleton } from "@/components/ui/skeleton";
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
import type { Team } from "@/components/key_team_helpers/key_list";
import { splitWildcardModels } from "./modelUtils";
const MODEL_SELECT_ALL_PROXY_MODELS_SPECIAL_VALUE = {
label: "All Proxy Models",
labelKey: "modelSelect.specialOptions.allProxyModels",
value: "all-proxy-models",
} as const;
const MODEL_SELECT_NO_DEFAULT_MODELS_SPECIAL_VALUE = {
label: "No Default Models",
labelKey: "modelSelect.specialOptions.noDefaultModels",
value: "no-default-models",
} as const;
@ -137,14 +139,16 @@ const unavailableGroups = (
selectedOptions: ModelOption[],
offeredByValue: Map<string, ModelOption>,
offeredListKnown: boolean,
t: TFunction,
): ModelOptionGroup[] => {
if (!offeredListKnown) return [];
const items = selectedOptions.filter((option) => !offeredByValue.has(option.value));
if (items.length === 0) return [];
return [{ label: "Unavailable", items }];
return [{ label: t("modelSelect.groups.unavailable"), items }];
};
export const ModelSelect = (props: ModelSelectProps) => {
const { t } = useTranslation();
const anchor = useComboboxAnchor();
const { id, teamID, organizationID, options, context, dataTestId, value = [], onChange, style } = props;
const { showAllProxyModelsOverride, includeSpecialOptions } = options || {};
@ -186,12 +190,12 @@ export const ModelSelect = (props: ModelSelectProps) => {
...(includeSpecialOptions
? [
{
label: "Special Options",
label: t("modelSelect.groups.special"),
items: [
...(shouldShowAllProxyModels
? [
{
label: MODEL_SELECT_ALL_PROXY_MODELS_SPECIAL_VALUE.label,
label: t(MODEL_SELECT_ALL_PROXY_MODELS_SPECIAL_VALUE.labelKey),
value: MODEL_SELECT_ALL_PROXY_MODELS_SPECIAL_VALUE.value,
disabled:
value.length > 0 &&
@ -202,7 +206,7 @@ export const ModelSelect = (props: ModelSelectProps) => {
]
: []),
{
label: MODEL_SELECT_NO_DEFAULT_MODELS_SPECIAL_VALUE.label,
label: t(MODEL_SELECT_NO_DEFAULT_MODELS_SPECIAL_VALUE.labelKey),
value: MODEL_SELECT_NO_DEFAULT_MODELS_SPECIAL_VALUE.value,
disabled:
value.length > 0 &&
@ -215,13 +219,13 @@ export const ModelSelect = (props: ModelSelectProps) => {
...(wildcard.length > 0
? [
{
label: "Wildcard Options",
label: t("modelSelect.groups.wildcard"),
items: wildcard.map((model) => {
const provider = model.replace("/*", "");
const capitalizedProvider = provider.charAt(0).toUpperCase() + provider.slice(1);
return {
label: `All ${capitalizedProvider} models`,
label: t("modelSelect.wildcardOption", { provider: capitalizedProvider }),
value: model,
disabled: hasSpecialOptionSelected,
};
@ -230,7 +234,7 @@ export const ModelSelect = (props: ModelSelectProps) => {
]
: []),
{
label: "Models",
label: t("modelSelect.groups.models"),
items: regular.map((model) => ({
label: model,
value: model,
@ -240,15 +244,17 @@ export const ModelSelect = (props: ModelSelectProps) => {
];
const offeredByValue = new Map(offeredGroups.flatMap((group) => group.items).map((option) => [option.value, option]));
const selectedOptions = value.map(
(v) =>
offeredByValue.get(v) ?? MODEL_SENTINEL_OPTIONS.find((option) => option.value === v) ?? { label: v, value: v },
);
const sentinelOption = (v: string): ModelOption | undefined => {
const sentinel = MODEL_SENTINEL_OPTIONS.find((option) => option.value === v);
return sentinel === undefined ? undefined : { label: t(sentinel.labelKey), value: sentinel.value };
};
const selectedOptions = value.map((v) => offeredByValue.get(v) ?? sentinelOption(v) ?? { label: v, value: v });
const groups: ModelOptionGroup[] = [
...unavailableGroups(
selectedOptions,
offeredByValue,
isOfferedListKnown(allProxyModels !== undefined, context, organizationID, organizationModels),
t,
),
...offeredGroups,
];
@ -275,19 +281,24 @@ export const ModelSelect = (props: ModelSelectProps) => {
))}
{overflowOptions.length > 0 && (
<Tooltip>
<TooltipTrigger
render={<span className="px-1 text-xs text-muted-foreground" />}
>{`+${overflowOptions.length} more`}</TooltipTrigger>
<TooltipTrigger render={<span className="px-1 text-xs text-muted-foreground" />}>
{t("modelSelect.moreSelected", { count: overflowOptions.length })}
</TooltipTrigger>
<TooltipContent>{overflowOptions.map((option) => option.value).join(", ")}</TooltipContent>
</Tooltip>
)}
</>
)}
</ComboboxValue>
<ComboboxChipsInput id={id} placeholder="Select Models" aria-label="Select Models" className="min-w-24" />
<ComboboxChipsInput
id={id}
placeholder={t("modelSelect.placeholder")}
aria-label={t("modelSelect.placeholder")}
className="min-w-24"
/>
</ComboboxChips>
<ComboboxContent anchor={anchor}>
<ComboboxEmpty>No models found</ComboboxEmpty>
<ComboboxEmpty>{t("modelSelect.empty")}</ComboboxEmpty>
<ComboboxList>
{(group: ModelOptionGroup) => (
<ComboboxGroup key={group.label} items={group.items}>

View file

@ -1,5 +1,6 @@
import { Edit, Save } from "lucide-react";
import React, { useEffect, useState } from "react";
import { useTranslation } from "react-i18next";
import { useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrganizations";
import { Badge } from "@/components/ui/badge";
@ -70,7 +71,10 @@ const SettingRow: React.FC<SettingRowProps> = ({ label, description, isEditing,
</div>
);
const NotSet = () => <span className="italic text-muted-foreground">Not set</span>;
const NotSet = () => {
const { t } = useTranslation();
return <span className="italic text-muted-foreground">{t("teams.ssoSettings.notSet")}</span>;
};
const renderTags = (values: string[], displayFn?: (v: string) => string) => {
if (!values || values.length === 0) return <NotSet />;
@ -111,6 +115,7 @@ const DEFAULT_VALUES: SettingsValues = {
};
const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken }) => {
const { t } = useTranslation();
const anchor = useComboboxAnchor();
const [loading, setLoading] = useState<boolean>(true);
const [values, setValues] = useState<SettingsValues>(DEFAULT_VALUES);
@ -135,14 +140,14 @@ const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken }) => {
} catch (error) {
console.error("Error fetching team SSO settings:", error);
setFetchError(true);
toast.fromError("Failed to fetch team settings");
toast.fromError(t("teams.ssoSettings.toasts.fetchFailed"));
} finally {
setLoading(false);
}
};
fetchSettings();
}, [accessToken]);
}, [accessToken, t]);
const handleSave = async () => {
if (!accessToken) return;
@ -154,10 +159,10 @@ const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken }) => {
setValues(newValues);
setEditedValues(newValues);
setIsEditing(false);
toast.success("Default team settings updated successfully");
toast.success(t("teams.ssoSettings.toasts.updated"));
} catch (error) {
console.error("Error updating team settings:", error);
toast.fromError("Failed to update team settings");
toast.fromError(t("teams.ssoSettings.toasts.updateFailed"));
} finally {
setSaving(false);
}
@ -175,7 +180,7 @@ const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken }) => {
if (loading) {
return (
<div className="flex h-64 items-center justify-center" aria-busy="true">
<UiLoadingSpinner aria-label="Loading default team settings" />
<UiLoadingSpinner aria-label={t("teams.ssoSettings.loading")} />
</div>
);
}
@ -184,7 +189,7 @@ const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken }) => {
return (
<Card>
<CardContent>
<p>No team settings available or you do not have permission to view them.</p>
<p>{t("teams.ssoSettings.noAccess")}</p>
</CardContent>
</Card>
);
@ -195,17 +200,15 @@ const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken }) => {
<CardHeader className="gap-4 border-b border-border pb-6">
<div>
<CardTitle>
<h3 className="text-lg font-semibold text-foreground">Default Team Settings</h3>
<h3 className="text-lg font-semibold text-foreground">{t("teams.ssoSettings.title")}</h3>
</CardTitle>
<CardDescription className="mt-1">
These settings will be applied by default when creating new teams.
</CardDescription>
<CardDescription className="mt-1">{t("teams.ssoSettings.description")}</CardDescription>
</div>
<CardAction>
{isEditing ? (
<div className="flex gap-3">
<Button type="button" variant="outline" onClick={handleCancel} disabled={saving}>
Cancel
{t("teams.ssoSettings.actions.cancel")}
</Button>
<Button type="button" onClick={handleSave} disabled={saving}>
{saving ? (
@ -213,13 +216,13 @@ const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken }) => {
) : (
<Save data-icon="inline-start" />
)}
Save Changes
{t("teams.ssoSettings.actions.save")}
</Button>
</div>
) : (
<Button type="button" variant="outline" onClick={() => setIsEditing(true)}>
<Edit data-icon="inline-start" />
Edit Settings
{t("teams.ssoSettings.actions.edit")}
</Button>
)}
</CardAction>
@ -228,12 +231,12 @@ const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken }) => {
<CardContent className="pt-8">
<section className="mb-8">
<h4 className="mb-2 text-xs font-bold tracking-wider text-muted-foreground uppercase">
Budget & Rate Limits
{t("teams.ssoSettings.sections.budget")}
</h4>
<div className="border-t border-border">
<SettingRow
label="Max Budget"
description="Maximum budget (in USD) for new automatically created teams."
label={t("teams.ssoSettings.fields.maxBudget.label")}
description={t("teams.ssoSettings.fields.maxBudget.description")}
isEditing={isEditing}
viewContent={
values.max_budget != null ? <span>${Number(values.max_budget).toLocaleString()}</span> : <NotSet />
@ -249,16 +252,16 @@ const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken }) => {
onChange={(event) =>
update("max_budget", event.target.value === "" ? null : Number(event.target.value))
}
placeholder="Not set"
aria-label="Max Budget"
placeholder={t("teams.ssoSettings.notSet")}
aria-label={t("teams.ssoSettings.fields.maxBudget.label")}
/>
</InputGroup>
}
/>
<SettingRow
label="Budget Duration"
description="How frequently the team's budget resets."
label={t("teams.ssoSettings.fields.budgetDuration.label")}
description={t("teams.ssoSettings.fields.budgetDuration.description")}
isEditing={isEditing}
viewContent={
values.budget_duration ? <span>{getBudgetDurationLabel(values.budget_duration)}</span> : <NotSet />
@ -273,8 +276,8 @@ const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken }) => {
/>
<SettingRow
label="TPM Limit"
description="Maximum tokens per minute allowed across all models."
label={t("teams.ssoSettings.fields.tpmLimit.label")}
description={t("teams.ssoSettings.fields.tpmLimit.description")}
isEditing={isEditing}
viewContent={values.tpm_limit != null ? <span>{values.tpm_limit.toLocaleString()}</span> : <NotSet />}
editContent={
@ -286,16 +289,16 @@ const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken }) => {
onChange={(event) =>
update("tpm_limit", event.target.value === "" ? null : Number(event.target.value))
}
placeholder="Not set"
placeholder={t("teams.ssoSettings.notSet")}
min={0}
aria-label="TPM Limit"
aria-label={t("teams.ssoSettings.fields.tpmLimit.label")}
/>
}
/>
<SettingRow
label="RPM Limit"
description="Maximum requests per minute allowed across all models."
label={t("teams.ssoSettings.fields.rpmLimit.label")}
description={t("teams.ssoSettings.fields.rpmLimit.description")}
isEditing={isEditing}
viewContent={values.rpm_limit != null ? <span>{values.rpm_limit.toLocaleString()}</span> : <NotSet />}
editContent={
@ -307,9 +310,9 @@ const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken }) => {
onChange={(event) =>
update("rpm_limit", event.target.value === "" ? null : Number(event.target.value))
}
placeholder="Not set"
placeholder={t("teams.ssoSettings.notSet")}
min={0}
aria-label="RPM Limit"
aria-label={t("teams.ssoSettings.fields.rpmLimit.label")}
/>
}
/>
@ -318,12 +321,12 @@ const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken }) => {
<section>
<h4 className="mb-2 text-xs font-bold tracking-wider text-muted-foreground uppercase">
Access & Permissions
{t("teams.ssoSettings.sections.access")}
</h4>
<div className="border-t border-border">
<SettingRow
label="Default Organization"
description="Teams created without an explicit organization are assigned to this organization."
label={t("teams.ssoSettings.fields.organization.label")}
description={t("teams.ssoSettings.fields.organization.description")}
isEditing={isEditing}
viewContent={
values.organization_id ? (
@ -339,15 +342,15 @@ const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken }) => {
loading={isOrganizationsLoading}
value={editedValues.organization_id ?? undefined}
onChange={(organizationId) => update("organization_id", organizationId || null)}
placeholder="Select an organization"
placeholder={t("teams.ssoSettings.fields.organization.placeholder")}
/>
</div>
}
/>
<SettingRow
label="Models"
description="Default list of models that new teams can access."
label={t("teams.ssoSettings.fields.models.label")}
description={t("teams.ssoSettings.fields.models.description")}
isEditing={isEditing}
viewContent={renderTags(values.models, getModelDisplayName)}
editContent={
@ -363,8 +366,8 @@ const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken }) => {
/>
<SettingRow
label="Team Member Permissions"
description="Default permissions granted to members of newly created teams. /key/info and /key/health are always included."
label={t("teams.ssoSettings.fields.permissions.label")}
description={t("teams.ssoSettings.fields.permissions.description")}
isEditing={isEditing}
viewContent={renderTags(values.team_member_permissions)}
editContent={
@ -384,7 +387,10 @@ const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken }) => {
))
}
</ComboboxValue>
<ComboboxChipsInput placeholder="Select permissions" aria-label="Team Member Permissions" />
<ComboboxChipsInput
placeholder={t("teams.ssoSettings.fields.permissions.placeholder")}
aria-label={t("teams.ssoSettings.fields.permissions.label")}
/>
</ComboboxChips>
<ComboboxContent anchor={anchor}>
<ComboboxList>

View file

@ -18,7 +18,9 @@ import { useZodForm } from "@/lib/forms/useZodForm";
import { TagsInput } from "@/app/(dashboard)/guardrails/_components/content_filter/TagsInput";
import { TabsContent } from "@/components/ui/tabs";
import { ChevronDown, Plus, Users } from "lucide-react";
import type { TFunction } from "i18next";
import React, { useEffect, useMemo, useState } from "react";
import { useTranslation } from "react-i18next";
import { z } from "zod";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import { PageHeader, PageHeaderControls, PageHeaderDescription, PageHeaderTitle } from "@/components/shared/PageHeader";
@ -74,43 +76,44 @@ const SUPPRESSED_BY_DESCRIPTION = "";
const numericInputSchema = z.union([z.string(), z.number()]).optional();
const teamCreateFieldsSchema = z.object({
team_alias: z.string().min(1, "Please input a team name"),
organization_id: z.string().nullish(),
models: z.array(z.string()).optional(),
max_budget: numericInputSchema,
budget_duration: z.string().nullish(),
tpm_limit: numericInputSchema,
rpm_limit: numericInputSchema,
tpd_limit: numericInputSchema,
metadata: metadataPairsSchema.optional(),
team_id: z.string().optional(),
team_member_budget: z.number().optional(),
team_member_key_duration: z.string().optional(),
team_member_rpm_limit: numericInputSchema,
team_member_tpm_limit: numericInputSchema,
secret_manager_settings: z.string().optional(),
guardrails: z.array(z.string()).optional(),
disable_global_guardrails: z.boolean().optional(),
require_trace_id: z.boolean().optional(),
policies: z.array(z.string()).optional(),
access_group_ids: z.array(z.string()).optional(),
allowed_vector_store_ids: z.array(z.string()).optional(),
allowed_passthrough_routes: z.array(z.string()).optional(),
allowed_mcp_servers_and_groups: z
.object({
servers: z.array(z.string()),
accessGroups: z.array(z.string()),
toolsets: z.array(z.string()).optional(),
})
.optional(),
mcp_tool_permissions: z.record(z.string(), z.array(z.string())).optional(),
allowed_agents_and_groups: z.object({ agents: z.array(z.string()), accessGroups: z.array(z.string()) }).optional(),
object_permission_search_tools: z.array(z.string()).optional(),
object_permission_skills: z.array(z.string()).optional(),
});
const buildTeamCreateFieldsSchema = (t: TFunction) =>
z.object({
team_alias: z.string().min(1, t("teams.page.create.errors.teamNameRequired")),
organization_id: z.string().nullish(),
models: z.array(z.string()).optional(),
max_budget: numericInputSchema,
budget_duration: z.string().nullish(),
tpm_limit: numericInputSchema,
rpm_limit: numericInputSchema,
tpd_limit: numericInputSchema,
metadata: metadataPairsSchema.optional(),
team_id: z.string().optional(),
team_member_budget: z.number().optional(),
team_member_key_duration: z.string().optional(),
team_member_rpm_limit: numericInputSchema,
team_member_tpm_limit: numericInputSchema,
secret_manager_settings: z.string().optional(),
guardrails: z.array(z.string()).optional(),
disable_global_guardrails: z.boolean().optional(),
require_trace_id: z.boolean().optional(),
policies: z.array(z.string()).optional(),
access_group_ids: z.array(z.string()).optional(),
allowed_vector_store_ids: z.array(z.string()).optional(),
allowed_passthrough_routes: z.array(z.string()).optional(),
allowed_mcp_servers_and_groups: z
.object({
servers: z.array(z.string()),
accessGroups: z.array(z.string()),
toolsets: z.array(z.string()).optional(),
})
.optional(),
mcp_tool_permissions: z.record(z.string(), z.array(z.string())).optional(),
allowed_agents_and_groups: z.object({ agents: z.array(z.string()), accessGroups: z.array(z.string()) }).optional(),
object_permission_search_tools: z.array(z.string()).optional(),
object_permission_skills: z.array(z.string()).optional(),
});
type TeamCreateFormValues = z.infer<typeof teamCreateFieldsSchema>;
type TeamCreateFormValues = z.infer<ReturnType<typeof buildTeamCreateFieldsSchema>>;
const EMPTY_TEAM_CREATE_VALUES: TeamCreateFormValues = {
team_alias: "",
@ -216,6 +219,7 @@ const getAdminOrganizations = (
// @deprecated
const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly = false, premiumUser = false }) => {
const { t } = useTranslation();
const { data: organizationsData } = useOrganizations();
const organizations = organizationsData ?? null;
const { data: teamMetadataSchemaFields = [], isLoading: isTeamMetadataSchemaLoading } = useTeamMetadataSchema();
@ -237,7 +241,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
const teamCreateSchema = useMemo(
() =>
teamCreateFieldsSchema.superRefine((values, ctx) => {
buildTeamCreateFieldsSchema(t).superRefine((values, ctx) => {
if (isOrgAdmin && !values.organization_id) {
ctx.addIssue({ code: "custom", message: SUPPRESSED_BY_DESCRIPTION, path: ["organization_id"] });
}
@ -248,7 +252,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
if (!organizationIsStillPickable) {
ctx.addIssue({
code: "custom",
message: "You can no longer create teams in this organization",
message: t("teams.page.create.errors.organizationNotPickable"),
path: ["organization_id"],
});
}
@ -256,7 +260,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
ctx.addIssue({ code: "custom", message: SUPPRESSED_BY_DESCRIPTION, path: ["secret_manager_settings"] });
}
}),
[isOrgAdmin, additionalSettingsOpen, adminOrgs, organizations],
[isOrgAdmin, additionalSettingsOpen, adminOrgs, organizations, t],
);
const form = useZodForm(teamCreateSchema, { defaultValues: EMPTY_TEAM_CREATE_VALUES });
@ -294,7 +298,10 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
});
const defaultBudgetDuration: string | undefined = defaultTeamSettings?.values?.budget_duration ?? undefined;
const budgetDurationPlaceholder = defaultBudgetDuration
? `Default: ${getBudgetDurationLabel(defaultBudgetDuration)} (${defaultBudgetDuration})`
? t("teams.page.create.fields.budgetDuration.defaultPlaceholder", {
label: getBudgetDurationLabel(defaultBudgetDuration),
duration: defaultBudgetDuration,
})
: undefined;
// Add this useEffect to fetch guardrails
@ -385,9 +392,9 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
setIsTeamDeleting(true);
await teamDeleteCall(accessToken, teamToDelete.team_id);
await refreshTeams();
toast.success("Team deleted successfully");
toast.success(t("teams.page.toasts.deleted"));
} catch (error) {
toast.fromError("Error deleting the team: " + error);
toast.fromError(t("teams.page.toasts.deleteFailed", { error: String(error) }));
} finally {
setIsTeamDeleting(false);
setIsDeleteModalOpen(false);
@ -432,7 +439,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
formValues.budget_duration = null;
}
toast.info("Creating Team");
toast.info(t("teams.page.toasts.creating"));
const metadataObject = {
...metadataPairsToObject(formValues.metadata),
@ -561,14 +568,14 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
...(teamMemberModelBudgets !== undefined ? { team_member_model_max_budget: teamMemberModelBudgets } : {}),
models: normalizeTeamModelSelection(formValues.models),
});
toast.success("Team created");
toast.success(t("teams.page.toasts.created"));
await refreshTeams();
resetCreateForm();
setIsTeamModalVisible(false);
}
} catch (error) {
console.error("Error creating the team:", error);
toast.fromError("Error creating the team: " + extractProxyErrorMessage(error));
toast.fromError(t("teams.page.toasts.createFailed", { error: extractProxyErrorMessage(error) }));
}
};
@ -602,7 +609,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
const tabItems = [
{
key: "your-teams",
label: "Your Teams",
label: t("teams.page.tabs.yourTeams"),
className: "flex min-h-0 flex-1 flex-col",
children: (
<>
@ -624,23 +631,23 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
<DeleteResourceModal
isOpen={isDeleteModalOpen}
title="Delete Team?"
title={t("teams.page.deleteModal.title")}
alertMessage={(() => {
const deleteKeyCount = teamToDelete?.keys_count ?? teamToDelete?.keys?.length ?? 0;
return deleteKeyCount === 0
? undefined
: `Warning: This team has ${deleteKeyCount} keys associated with it. Deleting the team will also delete all associated keys, along with any models created for this team. This action is irreversible.`;
: t("teams.page.deleteModal.keysWarning", { count: deleteKeyCount });
})()}
message="Are you sure you want to delete this team, all its keys, and any models created for it? This action cannot be undone."
resourceInformationTitle="Team Information"
message={t("teams.page.deleteModal.message")}
resourceInformationTitle={t("teams.page.deleteModal.informationTitle")}
resourceInformation={[
{ label: "Team ID", value: teamToDelete?.team_id, code: true },
{ label: "Team Name", value: teamToDelete?.team_alias },
{ label: t("teams.page.deleteModal.teamId"), value: teamToDelete?.team_id, code: true },
{ label: t("teams.page.deleteModal.teamName"), value: teamToDelete?.team_alias },
{
label: "Keys",
label: t("teams.page.deleteModal.keys"),
value: teamToDelete?.keys_count ?? teamToDelete?.keys?.length ?? 0,
},
{ label: "Members", value: teamToDelete?.members_with_roles?.length },
{ label: t("teams.page.deleteModal.members"), value: teamToDelete?.members_with_roles?.length },
]}
requiredConfirmation={teamToDelete?.team_alias}
onCancel={cancelDelete}
@ -652,7 +659,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
},
{
key: "available-teams",
label: "Available Teams",
label: t("teams.page.tabs.availableTeams"),
className: "min-h-0 flex-1 overflow-y-auto",
children: <AvailableTeamsPanel accessToken={accessToken} userID={userID} />,
},
@ -660,7 +667,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
? [
{
key: "default-settings",
label: "Default Team Settings",
label: t("teams.page.tabs.defaultSettings"),
className: "min-h-0 flex-1 overflow-y-auto",
children: <TeamSSOSettings accessToken={accessToken} userID={userID || ""} userRole={userRole || ""} />,
},
@ -693,16 +700,16 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
<PageHeader>
<PageHeaderTitle>
<Users />
Teams
{t("teams.page.header.title")}
</PageHeaderTitle>
<PageHeaderDescription>Manage teams, members, and their access to models and budgets</PageHeaderDescription>
<PageHeaderDescription>{t("teams.page.header.description")}</PageHeaderDescription>
<PageHeaderControls>
<PageTabsList>
{canCreateOrManageTeams(userRole, userID, organizations) && (
<>
<UIButton onClick={openCreateTeamModal} data-testid="create-team-button">
<Plus className="size-4" />
Create Team
{t("teams.page.header.createTeam")}
</UIButton>
<ToolbarSeparator className="mx-0 h-6" />
</>
@ -727,12 +734,12 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
<Dialog open={isTeamModalVisible} onOpenChange={(open) => !open && handleCancel()}>
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[1000px]">
<DialogHeader>
<DialogTitle>Create Team</DialogTitle>
<DialogTitle>{t("teams.page.create.title")}</DialogTitle>
</DialogHeader>
<TooltipProvider>
<form onSubmit={form.handleSubmit(onCreateSubmit)}>
<FieldGroup>
<FormField control={form.control} name="team_alias" label="Team Name">
<FormField control={form.control} name="team_alias" label={t("teams.page.create.fields.teamName")}>
{({ ref, value, ...field }) => (
<UIInput {...field} ref={ref} value={value ?? ""} data-testid="team-name-input" />
)}
@ -749,15 +756,15 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
name="organization_id"
className="mt-8"
label={labelWithDocsHint(
"Organization",
"Organizations can have multiple teams. Learn more about the user management hierarchy",
t("teams.page.create.fields.organization.label"),
t("teams.page.create.fields.organization.hint"),
"https://docs.litellm.ai/docs/proxy/user_management_heirarchy",
)}
description={
isOrgAdmin && isSingleOrg
? "You can only create teams within this organization"
? t("teams.page.create.fields.organization.singleOrgDescription")
: isOrgAdmin
? "required"
? t("teams.page.create.fields.organization.requiredDescription")
: undefined
}
>
@ -773,9 +780,11 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
disabled={isOrgAdmin && soleOrganizationId !== null && value === soleOrganizationId}
allowClear={!isOrgAdmin}
placeholder={
hasNoOrgs ? "No organizations available" : "Search or select an Organization"
hasNoOrgs
? t("teams.page.create.fields.organization.noOrganizations")
: t("teams.page.create.fields.organization.placeholder")
}
emptyText="No organizations available"
emptyText={t("teams.page.create.fields.organization.noOrganizations")}
onValueChange={(next) => selectCreateTeamOrganization(next, value ?? null, onChange)}
/>
)}
@ -784,8 +793,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
{isOrgAdmin && !isSingleOrg && adminOrgs.length > 1 && (
<div className="mb-8 rounded-md border border-info/20 bg-info/10 p-4">
<span className="text-sm text-info">
Please select an organization to create a team for. You can only create teams within
organizations where you are an admin.
{t("teams.page.create.fields.organization.selectPrompt")}
</span>
</div>
)}
@ -796,8 +804,8 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
control={form.control}
name="models"
label={labelWithHint(
"Models",
"These are the models that your selected team has access to. Leave empty to grant no models directly, e.g. when the team gets its models from access groups",
t("teams.page.create.fields.models.label"),
t("teams.page.create.fields.models.hint"),
)}
>
{({ id, value, onChange }) => (
@ -816,12 +824,17 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
)}
</FormField>
<FormField control={form.control} name="max_budget" label="Max Budget (USD)">
<FormField control={form.control} name="max_budget" label={t("teams.page.create.fields.maxBudget")}>
{({ ref, value, ...field }) => (
<NumericalInput {...field} ref={ref} value={value ?? ""} step={0.01} precision={2} width={200} />
)}
</FormField>
<FormField control={form.control} name="budget_duration" className="mt-8" label="Reset Budget">
<FormField
control={form.control}
name="budget_duration"
className="mt-8"
label={t("teams.page.create.fields.budgetDuration.label")}
>
{({ id, value, onChange }) => (
<BudgetDurationDropdown
id={id}
@ -838,14 +851,14 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
value={modelMaxBudget}
onChange={setModelMaxBudget}
availableModels={userModels}
hint="Cap this team's spend on individual models, each with its own reset window. Every key on the team shares the cap unless the key sets its own budget for that model."
hint={t("teams.page.create.fields.modelMaxBudgetHint")}
/>
<FormField control={form.control} name="tpm_limit" label="Tokens per minute Limit (TPM)">
<FormField control={form.control} name="tpm_limit" label={t("teams.page.create.fields.tpmLimit")}>
{({ ref, value, ...field }) => (
<NumericalInput {...field} ref={ref} value={value ?? ""} step={1} width={400} />
)}
</FormField>
<FormField control={form.control} name="rpm_limit" label="Requests per minute Limit (RPM)">
<FormField control={form.control} name="rpm_limit" label={t("teams.page.create.fields.rpmLimit")}>
{({ ref, value, ...field }) => (
<NumericalInput {...field} ref={ref} value={value ?? ""} step={1} width={400} />
)}
@ -854,8 +867,8 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
control={form.control}
name="tpd_limit"
label={labelWithHint(
"Tokens per day Limit (TPD)",
"Daily token budget for batch submissions (/v1/batches). When set, batch input files are charged against this 24h window instead of the team's TPM/RPM limits. Online requests keep using TPM/RPM.",
t("teams.page.create.fields.tpdLimit.label"),
t("teams.page.create.fields.tpdLimit.hint"),
)}
>
{({ ref, value, ...field }) => (
@ -863,7 +876,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
)}
</FormField>
<Field>
<FieldLabel>Metadata</FieldLabel>
<FieldLabel>{t("teams.page.create.fields.metadata.label")}</FieldLabel>
<MetadataKeyValueFields
control={form.control}
getValues={form.getValues}
@ -872,7 +885,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
schemaLoading={isTeamMetadataSchemaLoading}
/>
<FieldDescription>
Values are saved as text. Enter JSON for typed values, e.g. 3, true, or {'{"region": "us"}'}.
{t("teams.page.create.fields.metadata.description", { example: '{"region": "us"}' })}
</FieldDescription>
</Field>
@ -882,7 +895,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
className="mt-20 mb-8 overflow-hidden rounded-lg border"
>
<CollapsibleTrigger className="group/section flex w-full items-center justify-between px-4 py-3 text-left">
<b>Additional Settings</b>
<b>{t("teams.page.create.sections.additional")}</b>
<ChevronDown className="size-5 shrink-0 text-muted-foreground transition-transform group-data-[panel-open]/section:rotate-180" />
</CollapsibleTrigger>
<CollapsibleContent className="px-4 pb-3">
@ -890,8 +903,8 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
<FormField
control={form.control}
name="team_id"
label="Team ID"
description="ID of the team you want to create. If not provided, it will be generated automatically."
label={t("teams.page.create.fields.teamId.label")}
description={t("teams.page.create.fields.teamId.description")}
>
{({ ref, value, ...field }) => <UIInput {...field} ref={ref} value={value ?? ""} />}
</FormField>
@ -899,8 +912,8 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
control={form.control}
name="team_member_budget"
label={labelWithHint(
"Team Member Budget (USD)",
"This is the individual budget for a user in the team.",
t("teams.page.create.fields.teamMemberBudget.label"),
t("teams.page.create.fields.teamMemberBudget.hint"),
)}
>
{({ ref, value, onChange, ...field }) => (
@ -919,31 +932,36 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
</FormField>
<ModelMaxBudgetField
key={`team-member-model-max-budget-${routerSettingsKey}`}
label="Default Per-Model Budget"
label={t("teams.page.create.fields.teamMemberModelMaxBudget.label")}
premiumUser={premiumUser}
value={teamMemberModelMaxBudget}
onChange={setTeamMemberModelMaxBudget}
availableModels={userModels}
hint="Set a default per-model spend cap for each team member, with an independent reset window."
hint={t("teams.page.create.fields.teamMemberModelMaxBudget.hint")}
/>
<FormField
control={form.control}
name="team_member_key_duration"
label={labelWithHint(
"Team Member Key Duration (eg: 1d, 1mo)",
"Set a limit to the duration of a team member's key. Format: 30s (seconds), 30m (minutes), 30h (hours), 30d (days), 1mo (month)",
t("teams.page.create.fields.teamMemberKeyDuration.label"),
t("teams.page.create.fields.teamMemberKeyDuration.hint"),
)}
>
{({ ref, value, ...field }) => (
<UIInput {...field} ref={ref} value={value ?? ""} placeholder="e.g., 30d" />
<UIInput
{...field}
ref={ref}
value={value ?? ""}
placeholder={t("teams.page.create.fields.teamMemberKeyDuration.placeholder")}
/>
)}
</FormField>
<FormField
control={form.control}
name="team_member_rpm_limit"
label={labelWithHint(
"Team Member RPM Limit",
"The RPM (Requests Per Minute) limit for individual team members",
t("teams.page.create.fields.teamMemberRpmLimit.label"),
t("teams.page.create.fields.teamMemberRpmLimit.hint"),
)}
>
{({ ref, value, ...field }) => (
@ -954,8 +972,8 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
control={form.control}
name="team_member_tpm_limit"
label={labelWithHint(
"Team Member TPM Limit",
"The TPM (Tokens Per Minute) limit for individual team members",
t("teams.page.create.fields.teamMemberTpmLimit.label"),
t("teams.page.create.fields.teamMemberTpmLimit.hint"),
)}
>
{({ ref, value, ...field }) => (
@ -965,11 +983,11 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
<FormField
control={form.control}
name="secret_manager_settings"
label="Secret Manager Settings"
label={t("teams.page.create.fields.secretManagerSettings.label")}
description={
premiumUser
? "Enter secret manager configuration as a JSON object."
: "Premium feature - Upgrade to manage secret manager settings."
? t("teams.page.create.fields.secretManagerSettings.premiumDescription")
: t("teams.page.create.fields.secretManagerSettings.upgradeDescription")
}
>
{({ ref, value, ...field }) => (
@ -988,11 +1006,11 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
name="guardrails"
className="mt-8"
label={labelWithDocsHint(
"Guardrails",
"Setup your first guardrail",
t("teams.page.create.fields.guardrails.label"),
t("teams.page.create.fields.guardrails.hint"),
"https://docs.litellm.ai/docs/proxy/guardrails/quick_start",
)}
description="Select existing guardrails or enter new ones"
description={t("teams.page.create.fields.guardrails.description")}
>
{({ id, value, onChange }) => (
<TagsInput
@ -1000,7 +1018,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
value={value ?? []}
onValueChange={onChange}
options={guardrailsList.map((name) => ({ value: name, label: name }))}
placeholder="Select or enter guardrails"
placeholder={t("teams.page.create.fields.guardrails.placeholder")}
/>
)}
</FormField>
@ -1010,13 +1028,13 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
name="disable_global_guardrails"
className="mt-4"
label={labelWithHint(
"Disable Global Guardrails",
"When enabled, this team will bypass any guardrails configured to run on every request (global guardrails)",
t("teams.page.create.fields.disableGlobalGuardrails.label"),
t("teams.page.create.fields.disableGlobalGuardrails.hint"),
)}
description={
premiumUser
? "Bypass global guardrails for this team"
: "Premium feature - Upgrade to disable global guardrails by team"
? t("teams.page.create.fields.disableGlobalGuardrails.premiumDescription")
: t("teams.page.create.fields.disableGlobalGuardrails.upgradeDescription")
}
>
{({ id, value, onChange }) => (
@ -1035,8 +1053,8 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
name="require_trace_id"
className="mt-4"
label={labelWithHint(
"Require Trace ID",
"Reject LLM, MCP and agent requests from this team that do not send a trace ID (x-litellm-trace-id header, LLM requests can also use traceparent or metadata.trace_id)",
t("teams.page.create.fields.requireTraceId.label"),
t("teams.page.create.fields.requireTraceId.hint"),
)}
>
{({ id, value, onChange }) => (
@ -1050,11 +1068,11 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
name="policies"
className="mt-8"
label={labelWithDocsHint(
"Policies",
"Apply policies to this team to control guardrails and other settings",
t("teams.page.create.fields.policies.label"),
t("teams.page.create.fields.policies.hint"),
"https://docs.litellm.ai/docs/proxy/guardrails/guardrail_policies",
)}
description="Select existing policies or enter new ones"
description={t("teams.page.create.fields.policies.description")}
>
{({ id, value, onChange }) => (
<TagsInput
@ -1062,7 +1080,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
value={value ?? []}
onValueChange={onChange}
options={policiesList.map((name) => ({ value: name, label: name }))}
placeholder="Select or enter policies"
placeholder={t("teams.page.create.fields.policies.placeholder")}
/>
)}
</FormField>
@ -1072,16 +1090,16 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
name="access_group_ids"
className="mt-8"
label={labelWithHint(
"Access Groups",
"Assign access groups to this team. Access groups control which models, MCP servers, and agents this team can use",
t("teams.page.create.fields.accessGroups.label"),
t("teams.page.create.fields.accessGroups.hint"),
)}
description="Select access groups to assign to this team"
description={t("teams.page.create.fields.accessGroups.description")}
>
{({ value, onChange }) => (
<AccessGroupSelector
value={value}
onChange={onChange}
placeholder="Select access groups (optional)"
placeholder={t("teams.page.create.fields.accessGroups.placeholder")}
/>
)}
</FormField>
@ -1090,17 +1108,17 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
name="allowed_vector_store_ids"
className="mt-8"
label={labelWithHint(
"Allowed Vector Stores",
"Select which vector stores this team can access by default. Leave empty for access to all vector stores",
t("teams.page.create.fields.vectorStores.label"),
t("teams.page.create.fields.vectorStores.hint"),
)}
description="Select vector stores this team can access. Leave empty for access to all vector stores"
description={t("teams.page.create.fields.vectorStores.description")}
>
{({ value, onChange }) => (
<VectorStoreSelector
onChange={onChange}
value={value}
accessToken={accessToken || ""}
placeholder="Select vector stores (optional)"
placeholder={t("teams.page.create.fields.vectorStores.placeholder")}
/>
)}
</FormField>
@ -1111,15 +1129,15 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
label={
!premiumUser
? labelWithHint(
"Allowed Pass Through Routes",
"Premium feature - Upgrade to set allowed pass through routes",
t("teams.page.create.fields.passThroughRoutes.label"),
t("teams.page.create.fields.passThroughRoutes.upgradeHint"),
)
: !isProxyAdminRole(userRole || "")
? labelWithHint(
"Allowed Pass Through Routes",
"Only proxy admins can set allowed pass through routes",
t("teams.page.create.fields.passThroughRoutes.label"),
t("teams.page.create.fields.passThroughRoutes.proxyAdminOnlyHint"),
)
: "Allowed Pass Through Routes"
: t("teams.page.create.fields.passThroughRoutes.label")
}
>
{({ value, onChange }) => (
@ -1127,7 +1145,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
value={value}
onChange={onChange}
accessToken={accessToken || ""}
placeholder="Select pass through routes (optional)"
placeholder={t("teams.page.create.fields.passThroughRoutes.placeholder")}
disabled={!premiumUser || !isProxyAdminRole(userRole || "")}
/>
)}
@ -1142,7 +1160,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
className="mt-8 mb-8 overflow-hidden rounded-lg border"
>
<CollapsibleTrigger className="group/section flex w-full items-center justify-between px-4 py-3 text-left">
<b>MCP Settings</b>
<b>{t("teams.page.create.sections.mcp")}</b>
<ChevronDown className="size-5 shrink-0 text-muted-foreground transition-transform group-data-[panel-open]/section:rotate-180" />
</CollapsibleTrigger>
<CollapsibleContent className="px-4 pb-3">
@ -1151,17 +1169,17 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
name="allowed_mcp_servers_and_groups"
className="mt-4"
label={labelWithHint(
"Allowed MCP Servers",
"Select which MCP servers or access groups this team can access",
t("teams.page.create.fields.mcpServers.label"),
t("teams.page.create.fields.mcpServers.hint"),
)}
description="Select MCP servers or access groups this team can access"
description={t("teams.page.create.fields.mcpServers.description")}
>
{({ value, onChange }) => (
<MCPServerSelector
onChange={onChange}
value={value}
accessToken={accessToken || ""}
placeholder="Select MCP servers or access groups (optional)"
placeholder={t("teams.page.create.fields.mcpServers.placeholder")}
allowAllProxyMcpServers={isProxyAdminRole(userRole || "")}
/>
)}
@ -1186,7 +1204,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
className="mt-8 mb-8 overflow-hidden rounded-lg border"
>
<CollapsibleTrigger className="group/section flex w-full items-center justify-between px-4 py-3 text-left">
<b>Agent Settings</b>
<b>{t("teams.page.create.sections.agent")}</b>
<ChevronDown className="size-5 shrink-0 text-muted-foreground transition-transform group-data-[panel-open]/section:rotate-180" />
</CollapsibleTrigger>
<CollapsibleContent className="px-4 pb-3">
@ -1195,17 +1213,17 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
name="allowed_agents_and_groups"
className="mt-4"
label={labelWithHint(
"Allowed Agents",
"Select which agents or access groups this team can access",
t("teams.page.create.fields.agents.label"),
t("teams.page.create.fields.agents.hint"),
)}
description="Select agents or access groups this team can access"
description={t("teams.page.create.fields.agents.description")}
>
{({ value, onChange }) => (
<AgentSelector
onChange={onChange}
value={value}
accessToken={accessToken || ""}
placeholder="Select agents or access groups (optional)"
placeholder={t("teams.page.create.fields.agents.placeholder")}
/>
)}
</FormField>
@ -1218,7 +1236,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
className="mt-8 mb-8 overflow-hidden rounded-lg border"
>
<CollapsibleTrigger className="group/section flex w-full items-center justify-between px-4 py-3 text-left">
<b>Search Tool Settings</b>
<b>{t("teams.page.create.sections.searchTool")}</b>
<ChevronDown className="size-5 shrink-0 text-muted-foreground transition-transform group-data-[panel-open]/section:rotate-180" />
</CollapsibleTrigger>
<CollapsibleContent className="px-4 pb-3">
@ -1227,17 +1245,17 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
name="object_permission_search_tools"
className="mt-4"
label={labelWithHint(
"Allowed Search Tools",
"Select which search tools this team can access. Leave empty to allow all search tools.",
t("teams.page.create.fields.searchTools.label"),
t("teams.page.create.fields.searchTools.hint"),
)}
description="Restrict which configured search tools keys on this team may call."
description={t("teams.page.create.fields.searchTools.description")}
>
{({ value, onChange }) => (
<SearchToolSelector
onChange={onChange}
value={value}
accessToken={accessToken || ""}
placeholder="Select search tools (optional, empty = all allowed)"
placeholder={t("teams.page.create.fields.searchTools.placeholder")}
/>
)}
</FormField>
@ -1250,7 +1268,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
className="mt-8 mb-8 overflow-hidden rounded-lg border"
>
<CollapsibleTrigger className="group/section flex w-full items-center justify-between px-4 py-3 text-left">
<b>Skill Settings</b>
<b>{t("teams.page.create.sections.skill")}</b>
<ChevronDown className="size-5 shrink-0 text-muted-foreground transition-transform group-data-[panel-open]/section:rotate-180" />
</CollapsibleTrigger>
<CollapsibleContent className="px-4 pb-3">
@ -1259,17 +1277,17 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
name="object_permission_skills"
className="mt-4"
label={labelWithHint(
"Allowed Skills",
"Enabled skills are visible to every team. Grant disabled (private) Claude Code plugins to this team here.",
t("teams.page.create.fields.skills.label"),
t("teams.page.create.fields.skills.hint"),
)}
description="Private skills keys on this team may see in the Claude Code marketplace."
description={t("teams.page.create.fields.skills.description")}
>
{({ value, onChange }) => (
<SkillSelector
onChange={onChange}
value={value}
accessToken={accessToken || ""}
placeholder="Select skills (optional)"
placeholder={t("teams.page.create.fields.skills.placeholder")}
/>
)}
</FormField>
@ -1278,7 +1296,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
<Collapsible className="mt-8 mb-8 overflow-hidden rounded-lg border">
<CollapsibleTrigger className="group/section flex w-full items-center justify-between px-4 py-3 text-left">
<b>Logging Settings</b>
<b>{t("teams.page.create.sections.logging")}</b>
<ChevronDown className="size-5 shrink-0 text-muted-foreground transition-transform group-data-[panel-open]/section:rotate-180" />
</CollapsibleTrigger>
<CollapsibleContent className="px-4 pb-3">
@ -1297,7 +1315,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
className="mt-8 mb-8 overflow-hidden rounded-lg border"
>
<CollapsibleTrigger className="group/section flex w-full items-center justify-between px-4 py-3 text-left">
<b>Router Settings</b>
<b>{t("teams.page.create.sections.router")}</b>
<ChevronDown className="size-5 shrink-0 text-muted-foreground transition-transform group-data-[panel-open]/section:rotate-180" />
</CollapsibleTrigger>
<CollapsibleContent className="px-4 pb-3">
@ -1319,14 +1337,13 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
<Collapsible className="mt-8 mb-8 overflow-hidden rounded-lg border">
<CollapsibleTrigger className="group/section flex w-full items-center justify-between px-4 py-3 text-left">
<b>Model Aliases</b>
<b>{t("teams.page.create.sections.modelAliases")}</b>
<ChevronDown className="size-5 shrink-0 text-muted-foreground transition-transform group-data-[panel-open]/section:rotate-180" />
</CollapsibleTrigger>
<CollapsibleContent className="px-4 pb-3">
<div className="mt-4">
<p className="mb-4 block text-sm text-muted-foreground">
Create custom aliases for models that can be used by team members in API calls. This allows
you to create shortcuts for specific models.
{t("teams.page.create.fields.modelAliasesDescription")}
</p>
<ModelAliasManager
accessToken={accessToken || ""}
@ -1340,7 +1357,7 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, isViewOnly
</FieldGroup>
<div className="mt-[10px] text-right">
<UIButton type="submit" data-testid="create-team-submit">
Create Team
{t("teams.page.create.submit")}
</UIButton>
</div>
</form>

View file

@ -1,5 +1,7 @@
"use client";
import { useTranslation } from "react-i18next";
import { Meter, MeterIndicator, MeterTrack } from "@/components/shared/Meter";
import { getSpendBudgetMeterTone } from "@/components/shared/table_cells/spend_budget_cell";
import type { components } from "@/lib/http/schema";
@ -15,17 +17,19 @@ interface TeamMemberSpendBudgetCellProps {
}
interface BudgetLineProps {
label: "Team" | "My Spend";
label: string;
spend: number | null | undefined;
maxBudget: number | null | undefined;
resetAt?: string | null;
}
function BudgetLine({ label, spend, maxBudget, resetAt }: BudgetLineProps) {
const { t } = useTranslation();
const spendValue = typeof spend === "number" && Number.isFinite(spend) ? spend : 0;
const budget = typeof maxBudget === "number" && Number.isFinite(maxBudget) ? maxBudget : null;
const spendText = spendValue === 0 ? "$0.00" : getSpendString(spendValue, 2);
const budgetText = budget === null ? "Unlimited" : `$${formatNumberWithCommas(budget, 2)}`;
const budgetText =
budget === null ? t("teams.memberSpendBudgetCell.unlimited") : `$${formatNumberWithCommas(budget, 2)}`;
const resetText = formatBudgetResetUtc(resetAt);
return (
@ -38,13 +42,25 @@ function BudgetLine({ label, spend, maxBudget, resetAt }: BudgetLineProps) {
</span>
</div>
{budget !== null && budget > 0 && (
<Meter value={spendValue} max={budget} aria-valuetext={`${label} ${spendText} of ${budgetText}`}>
<Meter
value={spendValue}
max={budget}
aria-valuetext={t("teams.memberSpendBudgetCell.meterValueText", {
label,
spend: spendText,
budget: budgetText,
})}
>
<MeterTrack>
<MeterIndicator tone={getSpendBudgetMeterTone((spendValue / budget) * 100)} />
</MeterTrack>
</Meter>
)}
{resetText && <div className="text-[10px] text-muted-foreground">Resets {resetText} UTC</div>}
{resetText && (
<div className="text-[10px] text-muted-foreground">
{t("teams.memberSpendBudgetCell.resets", { date: resetText })}
</div>
)}
</div>
);
}
@ -54,12 +70,13 @@ export function TeamMemberSpendBudgetCell({
teamMaxBudget,
callerMembership,
}: TeamMemberSpendBudgetCellProps) {
const { t } = useTranslation();
return (
<div className="flex min-w-[130px] flex-col gap-2">
<BudgetLine label="Team" spend={teamSpend} maxBudget={teamMaxBudget} />
<BudgetLine label={t("teams.memberSpendBudgetCell.team")} spend={teamSpend} maxBudget={teamMaxBudget} />
{callerMembership && (
<BudgetLine
label="My Spend"
label={t("teams.memberSpendBudgetCell.mySpend")}
spend={callerMembership.spend}
maxBudget={callerMembership.max_budget}
resetAt={callerMembership.budget_reset_at}

View file

@ -15,8 +15,10 @@ import { Input } from "@/components/ui/input";
import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants";
import { useDebouncedValue } from "@tanstack/react-pacer/debouncer";
import { ColumnFiltersState, OnChangeFn, PaginationState, SortingState } from "@tanstack/react-table";
import type { ParseKeys } from "i18next";
import { Download } from "lucide-react";
import React, { useCallback, useMemo, useState } from "react";
import { useTranslation } from "react-i18next";
import { Team } from "../key_team_helpers/key_list";
import { getTeamTableColumns, TEAM_TABLE_HIDDEN_COLUMNS } from "./teamTableColumns";
@ -38,13 +40,14 @@ const toSortOrder = (sorting: SortingState): "asc" | "desc" | undefined => {
return active.desc ? "desc" : "asc";
};
const FILTER_LABELS: Record<string, string> = {
org_id: "Organization",
alias: "Team alias",
team_id: "Team ID",
const FILTER_LABEL_KEYS: Record<string, ParseKeys> = {
org_id: "teams.table.filters.organization",
alias: "teams.table.filters.teamAlias",
team_id: "teams.table.filters.teamId",
};
export function TeamsTable({ userRole, userID, onSelectTeam, onEditTeam, onDeleteTeam }: TeamsTableProps) {
const { t } = useTranslation();
const { data: fetchedOrganizations } = useOrganizations();
const organizations = useMemo(() => fetchedOrganizations ?? [], [fetchedOrganizations]);
@ -118,9 +121,14 @@ export function TeamsTable({ userRole, userID, onSelectTeam, onEditTeam, onDelet
}, [accessToken, isExporting, teamListOptions]);
const columns = useMemo(() => {
const columnDeps = { organizations, userRole, onSelectTeam, onEditTeam, onDeleteTeam };
const columnDeps = { organizations, userRole, onSelectTeam, onEditTeam, onDeleteTeam, t };
return getTeamTableColumns(columnDeps);
}, [organizations, userRole, onSelectTeam, onEditTeam, onDeleteTeam]);
}, [organizations, userRole, onSelectTeam, onEditTeam, onDeleteTeam, t]);
const filterLabels = useMemo(
() => Object.fromEntries(Object.entries(FILTER_LABEL_KEYS).map(([columnId, key]) => [columnId, t(key)])),
[t],
);
const orgOptions = useMemo(
() =>
@ -163,8 +171,8 @@ export function TeamsTable({ userRole, userID, onSelectTeam, onEditTeam, onDelet
enableColumnResizing
columnResizeMode="onChange"
isLoading={isPending || isPlaceholderData}
loadingMessage="Loading teams..."
noDataMessage="No teams found"
loadingMessage={t("teams.table.loading")}
noDataMessage={t("teams.table.empty")}
fillHeight
size="compact"
toolbar={(table) => (
@ -173,11 +181,11 @@ export function TeamsTable({ userRole, userID, onSelectTeam, onEditTeam, onDelet
table={table}
searchValue={searchInput}
onSearchChange={handleSearchChange}
searchPlaceholder="Search teams by name or ID…"
searchPlaceholder={t("teams.table.searchPlaceholder")}
onRefresh={() => refetch?.()}
isRefreshing={isFetching}
onOpenFilters={() => setFiltersOpen(true)}
filterLabels={FILTER_LABELS}
filterLabels={filterLabels}
formatFilterValue={formatFilterValue}
>
<Button
@ -188,39 +196,39 @@ export function TeamsTable({ userRole, userID, onSelectTeam, onEditTeam, onDelet
data-testid="teams-export-csv"
>
<Download />
{isExporting ? "Exporting..." : "Export CSV"}
{isExporting ? t("teams.table.exporting") : t("teams.table.exportCsv")}
</Button>
</DataTableToolbar>
<DataTableFilterDrawer
table={table}
open={filtersOpen}
onOpenChange={setFiltersOpen}
title="Filters"
description="Narrow down your teams"
title={t("teams.table.filters.title")}
description={t("teams.table.filters.description")}
>
{({ get, set }) => (
<>
<DataTableFilterField label="Organization">
<DataTableFilterField label={t("teams.table.filters.organization")}>
<SearchSelect
options={orgOptions}
value={(get("org_id") as string) || undefined}
onValueChange={(value) => set("org_id", value ?? undefined)}
placeholder="Select an organization…"
emptyText="No organizations found"
placeholder={t("teams.table.filters.organizationPlaceholder")}
emptyText={t("teams.table.filters.organizationEmpty")}
/>
</DataTableFilterField>
<DataTableFilterField label="Team alias">
<DataTableFilterField label={t("teams.table.filters.teamAlias")}>
<Input
value={(get("alias") as string) ?? ""}
onChange={(event) => set("alias", event.target.value)}
placeholder="Enter team alias…"
placeholder={t("teams.table.filters.teamAliasPlaceholder")}
/>
</DataTableFilterField>
<DataTableFilterField label="Team ID">
<DataTableFilterField label={t("teams.table.filters.teamId")}>
<Input
value={(get("team_id") as string) ?? ""}
onChange={(event) => set("team_id", event.target.value)}
placeholder="Enter team ID…"
placeholder={t("teams.table.filters.teamIdPlaceholder")}
/>
</DataTableFilterField>
</>

View file

@ -1,7 +1,9 @@
"use client";
import { ColumnDef } from "@tanstack/react-table";
import type { ParseKeys, TFunction } from "i18next";
import { Copy, KeyRound, Layers, MoreHorizontal, Pencil, Trash2, Users } from "lucide-react";
import { useTranslation } from "react-i18next";
import { DataTableMultiSortHeader, DataTableSortHeader } from "@/components/shared/DataTable";
import { DateCell, IdentityCell } from "@/components/shared/table_cells";
@ -41,11 +43,18 @@ const teamMemberCount = (team: Team): number => team.members_count ?? team.membe
const teamModelCount = (team: Team): number => team.models?.length ?? 0;
const teamKeyCount = (team: Team): number => team.keys_count ?? team.keys?.length ?? 0;
const RESOURCE_COUNT_KEYS: Record<"members" | "models" | "keys", ParseKeys> = {
members: "teams.table.resourceCounts.members",
models: "teams.table.resourceCounts.models",
keys: "teams.table.resourceCounts.keys",
};
function ResourcesCell({ team }: { team: Team }) {
const { t } = useTranslation();
const items = [
{ key: "members" as const, label: "members", count: teamMemberCount(team) },
{ key: "models" as const, label: "models", count: teamModelCount(team) },
{ key: "keys" as const, label: "keys", count: teamKeyCount(team) },
{ key: "members" as const, count: teamMemberCount(team) },
{ key: "models" as const, count: teamModelCount(team) },
{ key: "keys" as const, count: teamKeyCount(team) },
];
return (
@ -56,7 +65,7 @@ function ResourcesCell({ team }: { team: Team }) {
return (
<span
key={item.key}
title={`${item.count} ${item.label}`}
title={t(RESOURCE_COUNT_KEYS[item.key], { count: item.count })}
className={cn(
"inline-flex items-center gap-1.5 rounded-md px-2 py-1 text-xs font-medium ring-1 ring-inset [&_svg]:size-3.5",
tone.className,
@ -72,10 +81,11 @@ function ResourcesCell({ team }: { team: Team }) {
}
function RateLimitLine({ label, value }: { label: string; value: number | null }) {
const { t } = useTranslation();
return (
<div>
<span className="text-[10px] font-semibold text-muted-foreground">{label} </span>
<span className="tabular-nums">{value != null ? formatNumberWithCommas(value) : "Unlimited"}</span>
<span className="tabular-nums">{value != null ? formatNumberWithCommas(value) : t("teams.table.unlimited")}</span>
</div>
);
}
@ -88,14 +98,15 @@ interface TeamRowActionsProps {
}
function TeamRowActions({ team, canManage, onEditTeam, onDeleteTeam }: TeamRowActionsProps) {
const { t } = useTranslation();
const handleCopy = () => {
void copyToClipboard(team.team_id, "Team ID copied");
void copyToClipboard(team.team_id, t("teams.table.actions.copied"));
};
return (
<DropdownMenu>
<DropdownMenuTrigger
aria-label="Open team actions"
aria-label={t("teams.table.actions.open")}
data-testid={`team-actions-${team.team_id}`}
className={cn(buttonVariants({ variant: "ghost", size: "icon-sm" }), "text-muted-foreground")}
>
@ -105,19 +116,19 @@ function TeamRowActions({ team, canManage, onEditTeam, onDeleteTeam }: TeamRowAc
{canManage && (
<DropdownMenuItem onClick={() => onEditTeam(team)} data-testid="team-action-edit">
<Pencil />
Edit team
{t("teams.table.actions.edit")}
</DropdownMenuItem>
)}
<DropdownMenuItem onClick={handleCopy} data-testid="team-action-copy">
<Copy />
Copy team ID
{t("teams.table.actions.copyId")}
</DropdownMenuItem>
{canManage && (
<>
<DropdownMenuSeparator />
<DropdownMenuItem variant="destructive" onClick={() => onDeleteTeam(team)} data-testid="team-action-delete">
<Trash2 />
Delete team
{t("teams.table.actions.delete")}
</DropdownMenuItem>
</>
)}
@ -132,6 +143,7 @@ interface TeamTableColumnsDeps {
onSelectTeam: (team: Team) => void;
onEditTeam: (team: Team) => void;
onDeleteTeam: (team: Team) => void;
t: TFunction;
}
export const getTeamTableColumns = ({
@ -140,6 +152,7 @@ export const getTeamTableColumns = ({
onSelectTeam,
onEditTeam,
onDeleteTeam,
t,
}: TeamTableColumnsDeps): ColumnDef<Team>[] => {
const canManage = userRole === "Admin";
@ -148,7 +161,7 @@ export const getTeamTableColumns = ({
id: "team_alias",
accessorKey: "team_alias",
meta: {
title: "Team",
title: t("teams.table.columns.team"),
renderSkeleton: () => (
<div className="flex flex-col gap-2 py-1">
<Skeleton className="h-4 w-32" />
@ -156,7 +169,9 @@ export const getTeamTableColumns = ({
</div>
),
},
header: ({ column }) => <DataTableSortHeader column={column} title="Team" variant="header-cycle" />,
header: ({ column }) => (
<DataTableSortHeader column={column} title={t("teams.table.columns.team")} variant="header-cycle" />
),
size: 260,
enableSorting: true,
cell: ({ row }) => {
@ -174,8 +189,8 @@ export const getTeamTableColumns = ({
{
id: "organization_alias",
accessorKey: "organization_id",
meta: { title: "Organization" },
header: ({ column }) => <DataTableSortHeader column={column} title="Organization" />,
meta: { title: t("teams.table.columns.organization") },
header: ({ column }) => <DataTableSortHeader column={column} title={t("teams.table.columns.organization")} />,
sortDescFirst: false,
size: 160,
enableSorting: true,
@ -195,7 +210,7 @@ export const getTeamTableColumns = ({
{
id: "resources",
meta: {
title: "Resources",
title: t("teams.table.columns.resources"),
renderSkeleton: () => (
<div className="flex items-center gap-1.5">
<Skeleton className="h-6 w-12 rounded-md" />
@ -208,9 +223,9 @@ export const getTeamTableColumns = ({
<DataTableMultiSortHeader
table={table}
fields={[
{ id: "members", label: "Members" },
{ id: "models", label: "Models" },
{ id: "keys_count", label: "Keys" },
{ id: "members", label: t("teams.table.columns.members") },
{ id: "models", label: t("teams.table.columns.models") },
{ id: "keys_count", label: t("teams.table.columns.keys") },
]}
/>
),
@ -221,13 +236,13 @@ export const getTeamTableColumns = ({
{
id: "spend",
accessorKey: "spend",
meta: { title: "Spend / Budget", skeleton: "meter", numeric: true },
meta: { title: t("teams.table.columns.spendBudget"), skeleton: "meter", numeric: true },
header: ({ table }) => (
<DataTableMultiSortHeader
table={table}
fields={[
{ id: "spend", label: "Spend" },
{ id: "max_budget", label: "Budget" },
{ id: "spend", label: t("teams.table.columns.spend") },
{ id: "max_budget", label: t("teams.table.columns.budget") },
]}
/>
),
@ -244,8 +259,10 @@ export const getTeamTableColumns = ({
{
id: "created_at",
accessorKey: "created_at",
meta: { title: "Created" },
header: ({ column }) => <DataTableSortHeader column={column} title="Created" variant="header-cycle" />,
meta: { title: t("teams.table.columns.created") },
header: ({ column }) => (
<DataTableSortHeader column={column} title={t("teams.table.columns.created")} variant="header-cycle" />
),
size: 130,
enableSorting: true,
cell: (info) => <DateCell value={info.getValue() as string | null} precision="date" />,
@ -253,8 +270,8 @@ export const getTeamTableColumns = ({
{
id: "members",
accessorFn: teamMemberCount,
meta: { title: "Members", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="Members" />,
meta: { title: t("teams.table.columns.members"), numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title={t("teams.table.columns.members")} />,
sortDescFirst: true,
size: 110,
enableSorting: true,
@ -263,8 +280,8 @@ export const getTeamTableColumns = ({
{
id: "models",
accessorFn: teamModelCount,
meta: { title: "Models", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="Models" />,
meta: { title: t("teams.table.columns.models"), numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title={t("teams.table.columns.models")} />,
sortDescFirst: true,
size: 100,
enableSorting: true,
@ -272,13 +289,13 @@ export const getTeamTableColumns = ({
},
{
id: "rate_limits",
meta: { title: "Rate Limits", skeleton: "twoLine" },
meta: { title: t("teams.table.columns.rateLimits"), skeleton: "twoLine" },
header: ({ table }) => (
<DataTableMultiSortHeader
table={table}
fields={[
{ id: "tpm_limit", label: "TPM" },
{ id: "rpm_limit", label: "RPM" },
{ id: "tpm_limit", label: t("teams.table.columns.tpm") },
{ id: "rpm_limit", label: t("teams.table.columns.rpm") },
]}
/>
),
@ -286,25 +303,27 @@ export const getTeamTableColumns = ({
enableSorting: false,
cell: ({ row }) => (
<div className="text-xs leading-tight">
<RateLimitLine label="TPM" value={row.original.tpm_limit} />
<RateLimitLine label="RPM" value={row.original.rpm_limit} />
<RateLimitLine label={t("teams.table.columns.tpm")} value={row.original.tpm_limit} />
<RateLimitLine label={t("teams.table.columns.rpm")} value={row.original.rpm_limit} />
</div>
),
},
{
id: "updated_at",
accessorKey: "updated_at",
meta: { title: "Updated" },
header: ({ column }) => <DataTableSortHeader column={column} title="Updated" />,
meta: { title: t("teams.table.columns.updated") },
header: ({ column }) => <DataTableSortHeader column={column} title={t("teams.table.columns.updated")} />,
sortDescFirst: false,
size: 130,
enableSorting: true,
cell: (info) => <DateCell value={info.getValue() as string | null} precision="date" fallback="Never" />,
cell: (info) => (
<DateCell value={info.getValue() as string | null} precision="date" fallback={t("teams.table.never")} />
),
},
{
id: "actions",
meta: { className: "text-right", headerClassName: "text-right" },
header: () => <span className="sr-only">Actions</span>,
header: () => <span className="sr-only">{t("teams.table.columns.actions")}</span>,
size: 60,
enableSorting: false,
enableHiding: false,

View file

@ -17,6 +17,7 @@ import { CheckCircleIcon, XCircleIcon, ExclamationIcon } from "@heroicons/react/
import { CopyToClipboard } from "react-copy-to-clipboard";
import { toast } from "@/lib/toast";
import { downloadBlob } from "@/components/EntityUsageExport/utils";
import { Trans, useTranslation } from "react-i18next";
interface BulkCreateUsersProps {
accessToken: string;
@ -57,6 +58,7 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
possibleUIRoles,
onUsersCreated,
}) => {
const { t } = useTranslation();
const [isModalVisible, setIsModalVisible] = useState(false);
const [parsedData, setParsedData] = useState<UserData[]>([]);
const [isProcessing, setIsProcessing] = useState(false);
@ -108,16 +110,14 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
// Check file type
if (file.type !== "text/csv" && !file.name.endsWith(".csv")) {
setFileError(`Invalid file type: ${file.name}. Please upload a CSV file (.csv extension).`);
toast.fromError("Invalid file type. Please upload a CSV file.");
setFileError(t("users.bulkCreate.errors.invalidFileType", { name: file.name }));
toast.fromError(t("users.bulkCreate.errors.invalidFileTypeToast"));
return;
}
// Check file size (limit to 5MB)
if (file.size > 5 * 1024 * 1024) {
setFileError(
`File is too large (${(file.size / (1024 * 1024)).toFixed(1)} MB). Please upload a CSV file smaller than 5MB.`,
);
setFileError(t("users.bulkCreate.errors.fileTooLarge", { size: (file.size / (1024 * 1024)).toFixed(1) }));
return;
}
@ -125,16 +125,14 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
complete: (results) => {
// Check if file is empty
if (!results.data || results.data.length === 0) {
setCsvStructureError("The CSV file appears to be empty. Please upload a file with data.");
setCsvStructureError(t("users.bulkCreate.errors.emptyFile"));
setParsedData([]);
return;
}
// Check if there's only header row
if (results.data.length === 1) {
setCsvStructureError(
"The CSV file only contains headers but no user data. Please add user data to your CSV.",
);
setCsvStructureError(t("users.bulkCreate.errors.headersOnly"));
setParsedData([]);
return;
}
@ -143,9 +141,7 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
// Check if headers exist
if (headers.length === 0 || (headers.length === 1 && headers[0] === "")) {
setCsvStructureError(
"The CSV file doesn't contain any column headers. Please make sure your CSV has headers.",
);
setCsvStructureError(t("users.bulkCreate.errors.noHeaders"));
setParsedData([]);
return;
}
@ -155,9 +151,7 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
// Check if all required columns are present
const missingColumns = requiredColumns.filter((col) => !headers.includes(col));
if (missingColumns.length > 0) {
setCsvStructureError(
`Your CSV is missing these required columns: ${missingColumns.join(", ")}. Please add these columns to your CSV file.`,
);
setCsvStructureError(t("users.bulkCreate.errors.missingColumns", { columns: missingColumns.join(", ") }));
setParsedData([]);
return;
}
@ -176,7 +170,7 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
return {
rowNumber: index + 2,
isValid: false,
error: `Row ${index + 2} has fewer columns than the header row. Please ensure all data is properly formatted.`,
error: t("users.bulkCreate.errors.fewerColumns", { row: index + 2 }),
user_email: "",
user_role: "",
} as UserData;
@ -199,36 +193,39 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
// Email validation
if (!user.user_email) {
errors.push("Email is required");
errors.push(t("users.bulkCreate.validation.emailRequired"));
} else if (!user.user_email.includes("@") || !user.user_email.includes(".")) {
errors.push("Invalid email format (must contain @ and domain)");
errors.push(t("users.bulkCreate.validation.emailInvalid"));
}
// Role validation
if (!user.user_role) {
errors.push("Role is required");
errors.push(t("users.bulkCreate.validation.roleRequired"));
} else {
// Validate user role
const validRoles = ["proxy_admin", "proxy_admin_viewer", "internal_user", "internal_user_viewer"];
if (!validRoles.includes(user.user_role)) {
errors.push(`Invalid role "${user.user_role}". Must be one of: ${validRoles.join(", ")}`);
errors.push(
t("users.bulkCreate.validation.roleInvalid", {
role: user.user_role,
roles: validRoles.join(", "),
}),
);
}
}
// Budget validation
if (user.max_budget && user.max_budget.toString().trim() !== "") {
if (isNaN(parseFloat(user.max_budget.toString()))) {
errors.push(`Max budget "${user.max_budget}" must be a number`);
errors.push(t("users.bulkCreate.validation.budgetNotNumber", { budget: user.max_budget }));
} else if (parseFloat(user.max_budget.toString()) <= 0) {
errors.push("Max budget must be greater than 0");
errors.push(t("users.bulkCreate.validation.budgetNotPositive"));
}
}
// Budget duration validation
if (user.budget_duration && !user.budget_duration.match(/^\d+[dhmwy]$|^\d+mo$/)) {
errors.push(
`Invalid budget duration format "${user.budget_duration}". Use format like "30d", "1mo", "2w", "6h"`,
);
errors.push(t("users.bulkCreate.validation.durationInvalid", { duration: user.budget_duration }));
}
// Teams validation
@ -239,7 +236,7 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
const userTeams = user.teams.split(",").map((t) => t.trim());
const invalidTeams = userTeams.filter((t) => !teamIds.includes(t));
if (invalidTeams.length > 0) {
errors.push(`Unknown team(s): ${invalidTeams.join(", ")}`);
errors.push(t("users.bulkCreate.validation.unknownTeams", { teams: invalidTeams.join(", ") }));
}
}
}
@ -257,24 +254,27 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
setParsedData(userData);
if (userData.length === 0) {
setCsvStructureError("No valid data rows found in the CSV file. Please check your file format.");
setCsvStructureError(t("users.bulkCreate.errors.noDataRows"));
} else if (validData.length === 0) {
setParseError("No valid users found in the CSV. Please check the errors below and fix your CSV file.");
setParseError(t("users.bulkCreate.errors.noValidUsers"));
} else if (validData.length < userData.length) {
setParseError(
`Found ${userData.length - validData.length} row(s) with errors out of ${userData.length} total rows. Please correct them before proceeding.`,
t("users.bulkCreate.errors.rowsWithErrors", {
count: userData.length - validData.length,
total: userData.length,
}),
);
} else {
toast.success(`Successfully parsed ${validData.length} users`);
toast.success(t("users.bulkCreate.toasts.parsed", { count: validData.length }));
}
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : "Unknown error";
setParseError(`Error parsing CSV: ${errorMessage}`);
const errorMessage = error instanceof Error ? error.message : t("users.bulkCreate.errors.unknown");
setParseError(t("users.bulkCreate.errors.parseFailed", { message: errorMessage }));
setParsedData([]);
}
},
error: (error) => {
setParseError(`Failed to parse CSV file: ${error.message}`);
setParseError(t("users.bulkCreate.errors.readFailed", { message: error.message }));
setParsedData([]);
},
header: false,
@ -428,14 +428,14 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
...u,
status: "success",
key: response.key || response.user_id,
error: "User created but failed to generate invitation link",
error: t("users.bulkCreate.errors.inviteLinkFailed"),
}
: u,
),
);
}
} else {
const errorMessage = response?.error || "Failed to create user";
const errorMessage = response?.error || t("users.bulkCreate.errors.createFailed");
setParsedData((current) =>
current.map((u, i) => (i === index ? { ...u, status: "failed", error: errorMessage } : u)),
);
@ -479,34 +479,38 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
window.URL.revokeObjectURL(url);
};
const onLinkCopied = () => toast.success(t("users.bulkCreate.toasts.linkCopied"));
const renderStatusCell = (record: UserData) => {
if (!record.isValid) {
return (
<div>
<div className="flex items-center">
<XCircleIcon className="h-5 w-5 text-destructive mr-2" />
<span className="text-destructive">Invalid</span>
<span className="text-destructive">{t("users.bulkCreate.status.invalid")}</span>
</div>
{record.error && <span className="text-sm text-destructive ml-7">{record.error}</span>}
</div>
);
}
if (!record.status || record.status === "pending") {
return <span className="text-muted-foreground">Pending</span>;
return <span className="text-muted-foreground">{t("users.bulkCreate.status.pending")}</span>;
}
if (record.status === "success") {
return (
<div>
<div className="flex items-center">
<CheckCircleIcon className="h-5 w-5 text-success mr-2" />
<span className="text-success">Success</span>
<span className="text-success">{t("users.bulkCreate.status.success")}</span>
</div>
{record.invitation_link && (
<div className="mt-1">
<div className="flex items-center">
<span className="text-xs text-muted-foreground truncate max-w-[150px]">{record.invitation_link}</span>
<CopyToClipboard text={record.invitation_link} onCopy={() => toast.success("Invitation link copied!")}>
<button className="ml-1 text-info text-xs hover:text-info/80">Copy</button>
<CopyToClipboard text={record.invitation_link} onCopy={onLinkCopied}>
<button className="ml-1 text-info text-xs hover:text-info/80">
{t("users.bulkCreate.status.copy")}
</button>
</CopyToClipboard>
</div>
</div>
@ -518,7 +522,7 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
<div>
<div className="flex items-center">
<XCircleIcon className="h-5 w-5 text-destructive mr-2" />
<span className="text-destructive">Failed</span>
<span className="text-destructive">{t("users.bulkCreate.status.failed")}</span>
</div>
{record.error && <span className="text-sm text-destructive ml-7">{JSON.stringify(record.error)}</span>}
</div>
@ -528,17 +532,23 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
const pageCount = Math.max(1, Math.ceil(parsedData.length / PREVIEW_PAGE_SIZE));
const currentPage = Math.min(pageIndex, pageCount - 1);
const visibleRows = parsedData.slice(currentPage * PREVIEW_PAGE_SIZE, (currentPage + 1) * PREVIEW_PAGE_SIZE);
const validCount = parsedData.filter((d) => d.isValid).length;
const successCount = parsedData.filter((d) => d.status === "success").length;
const failedCount = parsedData.filter((d) => d.status === "failed").length;
const createLabel = isProcessing
? t("users.bulkCreate.preview.creating")
: t("users.bulkCreate.preview.create", { count: validCount });
return (
<>
<Button className="mb-0" onClick={() => setIsModalVisible(true)}>
+ Bulk Invite Users
{t("users.bulkCreate.openButton")}
</Button>
<Dialog open={isModalVisible} onOpenChange={(open) => !open && setIsModalVisible(false)}>
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[800px]">
<DialogHeader>
<DialogTitle>Bulk Invite Users</DialogTitle>
<DialogTitle>{t("users.bulkCreate.title")}</DialogTitle>
</DialogHeader>
<div className="flex flex-col">
{/* Step indicator */}
@ -548,45 +558,40 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
<div className="w-8 h-8 rounded-full bg-info text-info-foreground flex items-center justify-center mr-3">
1
</div>
<h3 className="text-lg font-medium">Download and fill the template</h3>
<h3 className="text-lg font-medium">{t("users.bulkCreate.steps.download")}</h3>
</div>
<div className="ml-11 mb-6">
<p className="mb-4">Add multiple users at once by following these steps:</p>
<p className="mb-4">{t("users.bulkCreate.instructions.intro")}</p>
<ol className="list-decimal list-inside space-y-2 ml-2 mb-4">
<li>Download our CSV template</li>
<li>Add your users&apos; information to the spreadsheet</li>
<li>Save the file and upload it here</li>
<li>After creation, download the results file containing the Virtual Keys for each user</li>
<li>{t("users.bulkCreate.instructions.downloadTemplate")}</li>
<li>{t("users.bulkCreate.instructions.addUsers")}</li>
<li>{t("users.bulkCreate.instructions.saveAndUpload")}</li>
<li>{t("users.bulkCreate.instructions.downloadResults")}</li>
</ol>
<div className="bg-muted p-4 rounded-md border border-border mb-4">
<h4 className="font-medium mb-2">Template Column Names</h4>
<h4 className="font-medium mb-2">{t("users.bulkCreate.templateColumns.title")}</h4>
<div className="grid grid-cols-1 md:grid-cols-2 gap-3">
<div className="flex items-start">
<div className="w-3 h-3 rounded-full bg-destructive mt-1.5 mr-2 shrink-0"></div>
<div>
<p className="font-medium">user_email</p>
<p className="text-sm text-muted-foreground">User&apos;s email address (required)</p>
<p className="text-sm text-muted-foreground">{t("users.bulkCreate.templateColumns.email")}</p>
</div>
</div>
<div className="flex items-start">
<div className="w-3 h-3 rounded-full bg-destructive mt-1.5 mr-2 shrink-0"></div>
<div>
<p className="font-medium">user_role</p>
<p className="text-sm text-muted-foreground">
User&apos;s role (one of: &quot;proxy_admin&quot;, &quot;proxy_admin_viewer&quot;,
&quot;internal_user&quot;, &quot;internal_user_viewer&quot;)
</p>
<p className="text-sm text-muted-foreground">{t("users.bulkCreate.templateColumns.role")}</p>
</div>
</div>
<div className="flex items-start">
<div className="w-3 h-3 rounded-full bg-border mt-1.5 mr-2 shrink-0"></div>
<div>
<p className="font-medium">teams</p>
<p className="text-sm text-muted-foreground">
Comma-separated team IDs (e.g., &quot;team-1,team-2&quot;)
</p>
<p className="text-sm text-muted-foreground">{t("users.bulkCreate.templateColumns.teams")}</p>
</div>
</div>
<div className="flex items-start">
@ -594,7 +599,7 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
<div>
<p className="font-medium">max_budget</p>
<p className="text-sm text-muted-foreground">
Maximum budget as a number (e.g., &quot;100&quot;)
{t("users.bulkCreate.templateColumns.maxBudget")}
</p>
</div>
</div>
@ -603,7 +608,7 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
<div>
<p className="font-medium">budget_duration</p>
<p className="text-sm text-muted-foreground">
Budget reset period (e.g., &quot;30d&quot;, &quot;1mo&quot;)
{t("users.bulkCreate.templateColumns.budgetDuration")}
</p>
</div>
</div>
@ -612,7 +617,7 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
<div>
<p className="font-medium">models</p>
<p className="text-sm text-muted-foreground">
Comma-separated allowed models (e.g., &quot;gpt-3.5-turbo,gpt-4&quot;)
{t("users.bulkCreate.templateColumns.models")}
</p>
</div>
</div>
@ -621,7 +626,7 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
<Button size="lg" className="w-full md:w-auto" onClick={downloadTemplate}>
<Download className="size-4" />
Download CSV Template
{t("users.bulkCreate.downloadTemplate")}
</Button>
</div>
@ -629,7 +634,7 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
<div className="w-8 h-8 rounded-full bg-info text-info-foreground flex items-center justify-center mr-3">
2
</div>
<h3 className="text-lg font-medium">Upload your completed CSV</h3>
<h3 className="text-lg font-medium">{t("users.bulkCreate.steps.upload")}</h3>
</div>
<div className="ml-11">
@ -649,13 +654,16 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
{selectedFile.name}
</strong>
<span className={`block text-xs ${fileError ? "text-destructive" : "text-info"}`}>
{(selectedFile.size / 1024).toFixed(1)} KB • {new Date().toLocaleDateString()}
{t("users.bulkCreate.upload.fileMeta", {
size: (selectedFile.size / 1024).toFixed(1),
date: new Date().toLocaleDateString(),
})}
</span>
</div>
</div>
<Button variant="outline" size="sm" onClick={removeSelectedFile} className="flex items-center">
<Trash2 className="size-4" />
Remove
{t("users.bulkCreate.upload.remove")}
</Button>
</div>
@ -670,7 +678,7 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
<div className="w-full bg-border rounded-full h-1.5">
<div className="bg-info h-1.5 rounded-full w-full animate-pulse"></div>
</div>
<span className="ml-2 text-xs text-info">Processing...</span>
<span className="ml-2 text-xs text-info">{t("users.bulkCreate.upload.processing")}</span>
</div>
)
)}
@ -694,10 +702,12 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
onChange={handleFileInputChange}
/>
<Upload className="size-[30px] text-muted-foreground mb-2" />
<p className="mb-1">Drag and drop your CSV file here</p>
<p className="text-sm text-muted-foreground mb-3">or</p>
<span className={buttonVariants({ variant: "outline", size: "sm" })}>Browse files</span>
<p className="text-xs text-muted-foreground mt-4">Only CSV files (.csv) are supported</p>
<p className="mb-1">{t("users.bulkCreate.upload.dropHere")}</p>
<p className="text-sm text-muted-foreground mb-3">{t("users.bulkCreate.upload.or")}</p>
<span className={buttonVariants({ variant: "outline", size: "sm" })}>
{t("users.bulkCreate.upload.browse")}
</span>
<p className="text-xs text-muted-foreground mt-4">{t("users.bulkCreate.upload.csvOnly")}</p>
</div>
</label>
)}
@ -707,11 +717,9 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
<div className="flex items-start">
<ExclamationIcon className="h-5 w-5 shrink-0 text-warning mr-2 mt-0.5" />
<div className="min-w-0">
<strong className="text-warning">CSV Structure Error</strong>
<strong className="text-warning">{t("users.bulkCreate.upload.structureErrorTitle")}</strong>
<p className="text-warning mt-1 mb-0 break-words">{csvStructureError}</p>
<p className="text-warning mt-2 mb-0">
Please download our template and ensure your CSV follows the required format.
</p>
<p className="text-warning mt-2 mb-0">{t("users.bulkCreate.upload.structureErrorHint")}</p>
</div>
</div>
</div>
@ -726,8 +734,8 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
</div>
<h3 className="text-lg font-medium">
{parsedData.some((user) => user.status === "success" || user.status === "failed")
? "User Creation Results"
: "Review and create users"}
? t("users.bulkCreate.steps.results")
: t("users.bulkCreate.steps.review")}
</h3>
</div>
@ -739,12 +747,9 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
<p className="text-destructive font-medium break-words">{parseError}</p>
{parsedData.some((user) => !user.isValid) && (
<ul className="mt-2 list-disc list-inside text-destructive text-sm">
<li>Check the table below for specific errors in each row</li>
<li>
Common issues include invalid email formats, missing required fields, or incorrect role
values
</li>
<li>Fix these issues in your CSV file and upload again</li>
<li>{t("users.bulkCreate.preview.errorTips.checkTable")}</li>
<li>{t("users.bulkCreate.preview.errorTips.commonIssues")}</li>
<li>{t("users.bulkCreate.preview.errorTips.fixAndReupload")}</li>
</ul>
)}
</div>
@ -757,21 +762,21 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
<div className="flex items-center">
{parsedData.some((user) => user.status === "success" || user.status === "failed") ? (
<div className="flex items-center">
<p className="text-lg font-medium mr-3">Creation Summary</p>
<p className="text-lg font-medium mr-3">{t("users.bulkCreate.preview.creationSummary")}</p>
<p className="text-sm bg-success/15 text-success px-2 py-1 rounded-sm mr-2">
{parsedData.filter((d) => d.status === "success").length} Successful
{t("users.bulkCreate.preview.successful", { count: successCount })}
</p>
{parsedData.some((d) => d.status === "failed") && (
<p className="text-sm bg-destructive/15 text-destructive px-2 py-1 rounded-sm">
{parsedData.filter((d) => d.status === "failed").length} Failed
{t("users.bulkCreate.preview.failed", { count: failedCount })}
</p>
)}
</div>
) : (
<div className="flex items-center">
<p className="text-lg font-medium mr-3">User Preview</p>
<p className="text-lg font-medium mr-3">{t("users.bulkCreate.preview.userPreview")}</p>
<p className="text-sm bg-info/15 text-info px-2 py-1 rounded-sm">
{parsedData.filter((d) => d.isValid).length} of {parsedData.length} users valid
{t("users.bulkCreate.preview.validCount", { valid: validCount, count: parsedData.length })}
</p>
</div>
)}
@ -780,13 +785,13 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
{!parsedData.some((user) => user.status === "success" || user.status === "failed") && (
<div className="flex space-x-3">
<Button variant="outline" onClick={resetParsedData}>
Back
{t("users.bulkCreate.preview.back")}
</Button>
<Button
onClick={handleBulkCreate}
disabled={parsedData.filter((d) => d.isValid).length === 0 || isProcessing}
>
{isProcessing ? "Creating..." : `Create ${parsedData.filter((d) => d.isValid).length} Users`}
{createLabel}
</Button>
</div>
)}
@ -799,11 +804,12 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
<CheckCircleIcon className="h-5 w-5 text-info" />
</div>
<div>
<p className="font-medium text-info">User creation complete</p>
<p className="font-medium text-info">{t("users.bulkCreate.preview.completeTitle")}</p>
<p className="block text-sm text-info mt-1">
<span className="font-medium">Next step:</span> Download the credentials file containing
Virtual Keys and invitation links. Users will need these Virtual Keys to make LLM requests
through LiteLLM.
<Trans
i18nKey="users.bulkCreate.preview.nextStep"
components={{ emphasis: <span className="font-medium" /> }}
/>
</p>
</div>
</div>
@ -814,12 +820,14 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
<Table>
<TableHeader>
<TableRow>
<TableHead className="w-20">Row</TableHead>
<TableHead>Email</TableHead>
<TableHead>Role</TableHead>
<TableHead>Teams</TableHead>
<TableHead className={NUMERIC_CELL_CLASS}>Budget</TableHead>
<TableHead>Status</TableHead>
<TableHead className="w-20">{t("users.bulkCreate.preview.table.row")}</TableHead>
<TableHead>{t("users.bulkCreate.preview.table.email")}</TableHead>
<TableHead>{t("users.bulkCreate.preview.table.role")}</TableHead>
<TableHead>{t("users.bulkCreate.preview.table.teams")}</TableHead>
<TableHead className={NUMERIC_CELL_CLASS}>
{t("users.bulkCreate.preview.table.budget")}
</TableHead>
<TableHead>{t("users.bulkCreate.preview.table.status")}</TableHead>
</TableRow>
</TableHeader>
<TableBody>
@ -840,7 +848,7 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
{pageCount > 1 && (
<div className="flex items-center justify-end gap-3 mt-2">
<span className="text-sm text-muted-foreground">
Page {currentPage + 1} of {pageCount}
{t("users.bulkCreate.preview.page", { page: currentPage + 1, pageCount })}
</span>
<Button
variant="outline"
@ -848,7 +856,7 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
onClick={() => setPageIndex(currentPage - 1)}
disabled={currentPage === 0}
>
Previous
{t("users.bulkCreate.preview.previous")}
</Button>
<Button
variant="outline"
@ -856,7 +864,7 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
onClick={() => setPageIndex(currentPage + 1)}
disabled={currentPage >= pageCount - 1}
>
Next
{t("users.bulkCreate.preview.next")}
</Button>
</div>
)}
@ -864,13 +872,13 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
{!parsedData.some((user) => user.status === "success" || user.status === "failed") && (
<div className="flex justify-end mt-4">
<Button variant="outline" onClick={resetParsedData} className="mr-3">
Back
{t("users.bulkCreate.preview.back")}
</Button>
<Button
onClick={handleBulkCreate}
disabled={parsedData.filter((d) => d.isValid).length === 0 || isProcessing}
>
{isProcessing ? "Creating..." : `Create ${parsedData.filter((d) => d.isValid).length} Users`}
{createLabel}
</Button>
</div>
)}
@ -878,11 +886,11 @@ const BulkCreateUsersButton: React.FC<BulkCreateUsersProps> = ({
{parsedData.some((user) => user.status === "success" || user.status === "failed") && (
<div className="flex justify-end mt-4">
<Button variant="outline" onClick={resetParsedData} className="mr-3">
Start New Bulk Import
{t("users.bulkCreate.preview.startNew")}
</Button>
<Button onClick={downloadResults}>
<Download className="size-4" />
Download User Credentials
{t("users.bulkCreate.preview.downloadCredentials")}
</Button>
</div>
)}

View file

@ -2,6 +2,7 @@ import { Button } from "@/components/ui/button";
import { cn } from "@/lib/cva.config";
import { Filter } from "lucide-react";
import React from "react";
import { useTranslation } from "react-i18next";
interface FiltersButtonProps {
onClick: () => void;
@ -10,17 +11,13 @@ interface FiltersButtonProps {
label?: string;
}
export const FiltersButton: React.FC<FiltersButtonProps> = ({
onClick,
active,
hasActiveFilters,
label = "Filters",
}) => {
export const FiltersButton: React.FC<FiltersButtonProps> = ({ onClick, active, hasActiveFilters, label }) => {
const { t } = useTranslation();
return (
<span className="relative inline-flex">
<Button variant="outline" onClick={onClick} className={cn(active && "bg-muted")}>
<Filter className="size-4" />
{label}
{label ?? t("filters.button.label")}
</Button>
{hasActiveFilters && (
<sup aria-hidden="true" className="absolute -top-0.5 -right-0.5 size-1.5 rounded-full bg-primary" />

View file

@ -1,17 +1,19 @@
import { Button } from "@/components/ui/button";
import { RotateCcw } from "lucide-react";
import React from "react";
import { useTranslation } from "react-i18next";
interface ResetFiltersButtonProps {
onClick: () => void;
label?: string;
}
export const ResetFiltersButton: React.FC<ResetFiltersButtonProps> = ({ onClick, label = "Reset Filters" }) => {
export const ResetFiltersButton: React.FC<ResetFiltersButtonProps> = ({ onClick, label }) => {
const { t } = useTranslation();
return (
<Button variant="outline" onClick={onClick}>
<RotateCcw className="size-4" />
{label}
{label ?? t("filters.reset.label")}
</Button>
);
};

View file

@ -1,6 +1,8 @@
import type { ColumnDef, ColumnFiltersState, OnChangeFn, PaginationState, SortingState } from "@tanstack/react-table";
import type { TFunction } from "i18next";
import { Crown, Info, User, UserPlus } from "lucide-react";
import React, { useState } from "react";
import { useTranslation } from "react-i18next";
import { Member } from "@/components/networking";
import {
@ -70,10 +72,11 @@ function RoleHeaderTitle({ title, tooltip }: { title: string; tooltip?: string }
}
function ResetSpendAction({ disabledReason, onClick }: { disabledReason: string | null; onClick: () => void }) {
const { t } = useTranslation();
return (
<TableIconActionButton
variant="Reset"
tooltipText="Reset spend"
tooltipText={t("memberTable.actions.resetSpend")}
disabled={disabledReason !== null}
disabledTooltipText={disabledReason ?? undefined}
dataTestId="reset-member-spend"
@ -85,6 +88,7 @@ function ResetSpendAction({ disabledReason, onClick }: { disabledReason: string
const ACTIONS_COLUMN_WIDTH = 120;
interface MemberColumnDeps {
t: TFunction;
canEdit: boolean;
onEdit: (member: Member) => void;
onDelete: (member: Member) => void;
@ -121,6 +125,7 @@ const extraColumnDef = (column: MemberTableColumn): ColumnDef<Member> => {
};
const buildColumns = ({
t,
canEdit,
onEdit,
onDelete,
@ -134,32 +139,32 @@ const buildColumns = ({
{
id: "user_alias",
accessorFn: (member) => member.user_alias || undefined,
header: ({ column }) => <DataTableSortHeader column={column} title="Name" />,
header: ({ column }) => <DataTableSortHeader column={column} title={t("memberTable.columns.name")} />,
sortingFn: "text",
sortUndefined: "last",
enableGlobalFilter: true,
meta: { title: "Name" },
meta: { title: t("memberTable.columns.name") },
cell: ({ row }) => row.original.user_alias || <span className="text-muted-foreground">-</span>,
},
{
id: "user_email",
accessorFn: (member) => member.user_email || undefined,
header: ({ column }) => <DataTableSortHeader column={column} title="User Email" />,
header: ({ column }) => <DataTableSortHeader column={column} title={t("memberTable.columns.userEmail")} />,
sortingFn: "text",
sortUndefined: "last",
enableGlobalFilter: true,
meta: { title: "User Email" },
meta: { title: t("memberTable.columns.userEmail") },
cell: ({ row }) => row.original.user_email || "-",
},
{
id: "user_id",
accessorFn: (member) => member.user_id ?? undefined,
header: "User ID",
header: t("memberTable.columns.userId"),
enableSorting: false,
enableGlobalFilter: true,
cell: ({ row }) =>
row.original.user_id === "default_user_id" ? (
<StatusBadge tone="info" label="Default Proxy Admin" />
<StatusBadge tone="info" label={t("memberTable.defaultProxyAdmin")} />
) : (
row.original.user_id || "-"
),
@ -184,7 +189,7 @@ const buildColumns = ({
...extraColumns.map(extraColumnDef),
{
id: "actions",
header: "Actions",
header: t("memberTable.columns.actions"),
size: ACTIONS_COLUMN_WIDTH,
enableSorting: false,
enableGlobalFilter: false,
@ -194,7 +199,7 @@ const buildColumns = ({
<span className="inline-flex items-center gap-2">
<TableIconActionButton
variant="Edit"
tooltipText="Edit member"
tooltipText={t("memberTable.actions.edit")}
dataTestId="edit-member"
onClick={() => onEdit(row.original)}
/>
@ -207,7 +212,7 @@ const buildColumns = ({
{(!showDeleteForMember || showDeleteForMember(row.original)) && (
<TableIconActionButton
variant="Delete"
tooltipText="Delete member"
tooltipText={t("memberTable.actions.delete")}
dataTestId="delete-member"
onClick={() => onDelete(row.original)}
/>
@ -223,7 +228,7 @@ export default function MemberTable({
onEdit,
onDelete,
onAddMember,
roleColumnTitle = "Role",
roleColumnTitle,
roleTooltip,
extraColumns = [],
showDeleteForMember,
@ -231,6 +236,8 @@ export default function MemberTable({
resetSpendDisabledReason,
emptyText,
}: MemberTableProps) {
const { t } = useTranslation();
const roleTitle = roleColumnTitle ?? t("memberTable.columns.role");
const [globalFilter, setGlobalFilter] = useState("");
const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([]);
const [sorting, setSorting] = useState<SortingState>(DEFAULT_SORTING);
@ -249,10 +256,11 @@ export default function MemberTable({
const changeSearch = backToFirstPage(setGlobalFilter);
const columnDeps: MemberColumnDeps = {
t,
canEdit,
onEdit,
onDelete,
roleColumnTitle,
roleColumnTitle: roleTitle,
roleTooltip,
extraColumns,
showDeleteForMember,
@ -261,7 +269,7 @@ export default function MemberTable({
};
const columns = buildColumns(columnDeps);
const roleFilterItems = [
{ value: ALL_ROLES, label: "All Roles" },
{ value: ALL_ROLES, label: t("memberTable.filters.allRoles") },
...memberRoleOptions(members).map((role) => ({ value: role, label: role })),
];
@ -270,7 +278,7 @@ export default function MemberTable({
return (
<div className="flex w-full flex-col gap-2">
<span className="inline-flex text-sm text-foreground">
{members.length} Member{members.length !== 1 ? "s" : ""}
{t("memberTable.memberCount", { count: members.length })}
</span>
<DataTable
data={members}
@ -289,7 +297,7 @@ export default function MemberTable({
onGlobalFilterChange={changeSearch}
noDataMessage={
<span className="text-muted-foreground">
{isNarrowed ? "No members match your search or filters" : emptyText ?? "No data"}
{isNarrowed ? t("memberTable.noMatches") : emptyText ?? t("memberTable.noData")}
</span>
}
toolbar={(table) => (
@ -298,7 +306,7 @@ export default function MemberTable({
table={table}
searchValue={globalFilter}
onSearchChange={changeSearch}
searchPlaceholder="Search by name, email, or user ID"
searchPlaceholder={t("memberTable.searchPlaceholder")}
onOpenFilters={() => setFiltersOpen(true)}
showViewOptions={false}
/>
@ -306,18 +314,18 @@ export default function MemberTable({
table={table}
open={filtersOpen}
onOpenChange={setFiltersOpen}
title="Filters"
description="Narrow down members"
title={t("memberTable.filters.title")}
description={t("memberTable.filters.description")}
>
{({ get, set }) => (
<DataTableFilterField label={roleColumnTitle}>
<DataTableFilterField label={roleTitle}>
<Select
items={roleFilterItems}
value={(get("role") as string | undefined) ?? ALL_ROLES}
onValueChange={(value) => set("role", value === ALL_ROLES ? undefined : value)}
>
<SelectTrigger className="w-full" data-testid="filter-role">
<SelectValue placeholder="All Roles" />
<SelectValue placeholder={t("memberTable.filters.allRoles")} />
</SelectTrigger>
<SelectContent>
{roleFilterItems.map((item) => (
@ -336,7 +344,7 @@ export default function MemberTable({
{onAddMember && canEdit && (
<Button onClick={onAddMember} className="self-start">
<UserPlus className="size-4" />
Add Member
{t("memberTable.addMember")}
</Button>
)}
</div>

View file

@ -1,5 +1,7 @@
import { useRef, useState } from "react";
import type { TFunction } from "i18next";
import { Info, UserPlus } from "lucide-react";
import { useTranslation } from "react-i18next";
import { Alert, AlertTitle } from "@/components/shared/Alert";
import { useForm } from "react-hook-form";
import { userFilterUICall } from "@/components/networking";
@ -47,23 +49,23 @@ interface UserSearchModalProps {
teamId?: string;
}
const defaultRoles = (t: TFunction): Role[] => [
{ label: "admin", value: "admin", description: t("userSearchModal.defaultRoles.adminDescription") },
{ label: "user", value: "user", description: t("userSearchModal.defaultRoles.userDescription") },
];
const UserSearchModal: React.FC<UserSearchModalProps> = ({
isVisible,
onCancel,
onSubmit,
accessToken,
title = "Add Team Member",
roles = [
{
label: "admin",
value: "admin",
description: "Admin role. Can create team keys, add members, and manage settings.",
},
{ label: "user", value: "user", description: "User role. Can view team info, but not manage it." },
],
title,
roles,
defaultRole = "user",
teamId,
}) => {
const { t } = useTranslation();
const roleOptions = roles ?? defaultRoles(t);
const emptyValues: FormValues = { user_email: undefined, user_id: undefined, role: defaultRole };
const form = useForm<FormValues>({ defaultValues: emptyValues });
const selectedUserId = form.watch("user_id");
@ -172,8 +174,8 @@ const UserSearchModal: React.FC<UserSearchModalProps> = ({
autoHighlight="always"
isLoading={loading}
placeholder={placeholder}
emptyText="No results"
loadingText="Loading..."
emptyText={t("userSearchModal.search.noResults")}
loadingText={t("userSearchModal.search.loading")}
inputId={controlProps.id}
/>
</div>
@ -184,39 +186,43 @@ const UserSearchModal: React.FC<UserSearchModalProps> = ({
<Dialog open={isVisible} onOpenChange={(open) => !open && handleClose()} disablePointerDismissal={isSubmitting}>
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[800px]">
<DialogHeader>
<DialogTitle>{title}</DialogTitle>
<DialogTitle>{title ?? t("userSearchModal.defaultTitle")}</DialogTitle>
</DialogHeader>
<TooltipProvider>
<form onSubmit={form.handleSubmit(handleSubmit)} noValidate>
<Alert variant="info" className="mb-4" data-testid="member-existing-users-notice">
<Info />
<AlertTitle>
Search selects from users that already exist. To add someone new, ask a proxy admin to create their
account first.
</AlertTitle>
<AlertTitle>{t("userSearchModal.existingUsersNotice")}</AlertTitle>
</Alert>
<FieldGroup>
<FormField control={form.control} name="user_email" label="Email">
<FormField control={form.control} name="user_email" label={t("userSearchModal.fields.email")}>
{({ id, value, onChange }) =>
renderUserSearch("user_email", "Search by email", { id, value, onChange }, "member-email-search")
renderUserSearch(
"user_email",
t("userSearchModal.fields.emailPlaceholder"),
{ id, value, onChange },
"member-email-search",
)
}
</FormField>
<div className="text-center">OR</div>
<div className="text-center">{t("userSearchModal.fields.or")}</div>
<FormField control={form.control} name="user_id" label="User ID">
{({ id, value, onChange }) => renderUserSearch("user_id", "Search by user ID", { id, value, onChange })}
<FormField control={form.control} name="user_id" label={t("userSearchModal.fields.userId")}>
{({ id, value, onChange }) =>
renderUserSearch("user_id", t("userSearchModal.fields.userIdPlaceholder"), { id, value, onChange })
}
</FormField>
<FormField control={form.control} name="role" label="Member Role">
<FormField control={form.control} name="role" label={t("userSearchModal.fields.role")}>
{({ id, value, onChange }) => (
<Select items={roles} value={value} onValueChange={(next) => onChange(next as string)}>
<Select items={roleOptions} value={value} onValueChange={(next) => onChange(next as string)}>
<SelectTrigger id={id}>
<SelectValue />
</SelectTrigger>
<SelectContent>
{roles.map((role) => (
{roleOptions.map((role) => (
<SelectItem key={role.value} value={role.value}>
<Tooltip>
<TooltipTrigger
@ -240,7 +246,7 @@ const UserSearchModal: React.FC<UserSearchModalProps> = ({
<div className="mt-4 text-right">
<Button type="submit" disabled={isSubmitting || (!selectedUserId && !selectedUserEmail)}>
{isSubmitting ? <UiLoadingSpinner className="size-4" /> : <UserPlus />}
{isSubmitting ? "Adding..." : "Add Member"}
{isSubmitting ? t("userSearchModal.submit.adding") : t("userSearchModal.submit.add")}
</Button>
</div>
</form>

View file

@ -2,6 +2,7 @@
import { useMutation, useQueryClient } from "@tanstack/react-query";
import * as React from "react";
import { useTranslation } from "react-i18next";
import { organizationKeys } from "@/app/(dashboard)/hooks/organizations/useOrganizations";
import { ModelSelect } from "@/components/ModelSelect/ModelSelect";
@ -18,8 +19,8 @@ import VectorStoreSelector from "@/components/vector_store_management/VectorStor
import { useZodForm } from "@/lib/forms/useZodForm";
import { fetchClient } from "@/lib/http/api";
import { BUDGET_DURATION_OPTIONS, NO_RESET } from "../org-settings/OrgSettingsForm";
import { orgSettingsSchema } from "../org-settings/schema";
import { budgetDurationItems, NO_RESET } from "../org-settings/OrgSettingsForm";
import { buildOrgSettingsSchema } from "../org-settings/schema";
import { buildOrgCreateBody, emptyOrgFormValues, type OrgCreateBody } from "./mapper";
const defaultCreateOrganization = async (body: OrgCreateBody): Promise<unknown> => {
@ -40,8 +41,11 @@ export const OrgCreateDialog = ({
accessToken,
createOrganization = defaultCreateOrganization,
}: OrgCreateDialogProps) => {
const { t } = useTranslation();
const queryClient = useQueryClient();
const form = useZodForm(orgSettingsSchema, { defaultValues: emptyOrgFormValues });
const schema = React.useMemo(() => buildOrgSettingsSchema(t), [t]);
const form = useZodForm(schema, { defaultValues: emptyOrgFormValues });
const budgetDurationOptions = budgetDurationItems(t);
const closeAndReset = () => {
form.reset(emptyOrgFormValues);
@ -51,12 +55,12 @@ export const OrgCreateDialog = ({
const mutation = useMutation({
mutationFn: (body: OrgCreateBody) => createOrganization(body),
onSuccess: () => {
toast.success("Organization created successfully");
toast.success(t("organizations.createDialog.toasts.created"));
queryClient.invalidateQueries({ queryKey: organizationKeys.all });
closeAndReset();
},
onError: (error: unknown) =>
toast.fromError(error instanceof Error ? error.message : "Failed to create organization"),
toast.fromError(error instanceof Error ? error.message : t("organizations.createDialog.toasts.createFailed")),
});
const handleOpenChange = (nextOpen: boolean) => {
@ -76,16 +80,20 @@ export const OrgCreateDialog = ({
<Dialog open={open} onOpenChange={handleOpenChange}>
<DialogContent className="sm:max-w-3xl max-h-[90vh] overflow-y-auto">
<DialogHeader>
<DialogTitle>Create Organization</DialogTitle>
<DialogTitle>{t("organizations.createDialog.title")}</DialogTitle>
</DialogHeader>
<form onSubmit={onSubmit} noValidate>
<FieldGroup>
<FormField control={form.control} name="organization_alias" label="Organization Name">
<FormField
control={form.control}
name="organization_alias"
label={t("organizations.createDialog.fields.name")}
>
{({ ref, ...field }) => <Input {...field} ref={ref} />}
</FormField>
<FormField control={form.control} name="models" label="Models">
<FormField control={form.control} name="models" label={t("organizations.createDialog.fields.models")}>
{(field) => (
<ModelSelect
value={field.value}
@ -96,14 +104,22 @@ export const OrgCreateDialog = ({
)}
</FormField>
<FormField control={form.control} name="max_budget" label="Max Budget (USD)">
<FormField
control={form.control}
name="max_budget"
label={t("organizations.createDialog.fields.maxBudget")}
>
{({ ref, ...field }) => <Input {...field} ref={ref} type="number" step="any" min={0} />}
</FormField>
<FormField control={form.control} name="budget_duration" label="Reset Budget">
<FormField
control={form.control}
name="budget_duration"
label={t("organizations.createDialog.fields.budgetDuration")}
>
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
<Select
items={BUDGET_DURATION_OPTIONS}
items={budgetDurationOptions}
value={value === "" ? NO_RESET : value}
onValueChange={(selected) => onChange(selected === NO_RESET ? "" : selected)}
>
@ -111,7 +127,7 @@ export const OrgCreateDialog = ({
<SelectValue />
</SelectTrigger>
<SelectContent>
{BUDGET_DURATION_OPTIONS.map((option) => (
{budgetDurationOptions.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
</SelectItem>
@ -121,26 +137,26 @@ export const OrgCreateDialog = ({
)}
</FormField>
<FormField control={form.control} name="tpm_limit" label="Tokens per minute Limit (TPM)">
<FormField control={form.control} name="tpm_limit" label={t("organizations.createDialog.fields.tpmLimit")}>
{({ ref, ...field }) => <Input {...field} ref={ref} type="number" step={1} min={0} />}
</FormField>
<FormField control={form.control} name="rpm_limit" label="Requests per minute Limit (RPM)">
<FormField control={form.control} name="rpm_limit" label={t("organizations.createDialog.fields.rpmLimit")}>
{({ ref, ...field }) => <Input {...field} ref={ref} type="number" step={1} min={0} />}
</FormField>
<FormField
control={form.control}
name="vector_stores"
label="Allowed Vector Stores"
description="Select vector stores this organization can access. Leave empty for access to all vector stores"
label={t("organizations.createDialog.fields.vectorStores")}
description={t("organizations.createDialog.fields.vectorStoresDescription")}
>
{(field) => (
<VectorStoreSelector
value={field.value}
onChange={field.onChange}
accessToken={accessToken}
placeholder="Select vector stores (optional)"
placeholder={t("organizations.createDialog.fields.vectorStoresPlaceholder")}
/>
)}
</FormField>
@ -148,20 +164,20 @@ export const OrgCreateDialog = ({
<FormField
control={form.control}
name="mcp"
label="Allowed MCP Servers"
description="Select MCP servers, access groups, and toolsets this organization can access. Leave empty for access to all"
label={t("organizations.createDialog.fields.mcp")}
description={t("organizations.createDialog.fields.mcpDescription")}
>
{(field) => (
<MCPServerSelector
value={field.value}
onChange={field.onChange}
accessToken={accessToken}
placeholder="Select MCP servers and access groups (optional)"
placeholder={t("organizations.createDialog.fields.mcpPlaceholder")}
/>
)}
</FormField>
<FormField control={form.control} name="metadata" label="Metadata">
<FormField control={form.control} name="metadata" label={t("organizations.createDialog.fields.metadata")}>
{({ ref, ...field }) => <Textarea {...field} ref={ref} rows={4} />}
</FormField>
</FieldGroup>
@ -173,10 +189,10 @@ export const OrgCreateDialog = ({
onClick={() => handleOpenChange(false)}
disabled={mutation.isPending}
>
Cancel
{t("organizations.createDialog.cancel")}
</Button>
<Button type="submit" disabled={mutation.isPending}>
{mutation.isPending ? "Creating..." : "Create Organization"}
{mutation.isPending ? t("organizations.createDialog.creating") : t("organizations.createDialog.submit")}
</Button>
</DialogFooter>
</form>

View file

@ -1,7 +1,9 @@
"use client";
import { useMutation, useQueryClient } from "@tanstack/react-query";
import type { ParseKeys, TFunction } from "i18next";
import * as React from "react";
import { useTranslation } from "react-i18next";
import { organizationKeys } from "@/app/(dashboard)/hooks/organizations/useOrganizations";
import { ModelSelect } from "@/components/ModelSelect/ModelSelect";
@ -20,16 +22,19 @@ import { useZodForm } from "@/lib/forms/useZodForm";
import { fetchClient } from "@/lib/http/api";
import { buildOrgPatch, orgToForm, type OrgPatchBody } from "./mapper";
import { orgSettingsSchema } from "./schema";
import { buildOrgSettingsSchema } from "./schema";
export const NO_RESET = "never";
export const BUDGET_DURATION_OPTIONS = [
{ value: NO_RESET, label: "No reset" },
{ value: "24h", label: "daily" },
{ value: "7d", label: "weekly" },
{ value: "30d", label: "monthly" },
] as const;
const BUDGET_DURATION_OPTIONS: readonly { value: string; labelKey: ParseKeys }[] = [
{ value: NO_RESET, labelKey: "organizations.settingsForm.budgetDuration.noReset" },
{ value: "24h", labelKey: "organizations.settingsForm.budgetDuration.daily" },
{ value: "7d", labelKey: "organizations.settingsForm.budgetDuration.weekly" },
{ value: "30d", labelKey: "organizations.settingsForm.budgetDuration.monthly" },
];
export const budgetDurationItems = (t: TFunction): { value: string; label: string }[] =>
BUDGET_DURATION_OPTIONS.map((option) => ({ value: option.value, label: t(option.labelKey) }));
const defaultPatchOrganization = async (organizationId: string, body: OrgPatchBody): Promise<unknown> => {
const { data } = await fetchClient.PATCH("/v2/organization/{organization_id}", {
@ -56,19 +61,22 @@ export const OrgSettingsForm = ({
onSaved,
patchOrganization = defaultPatchOrganization,
}: OrgSettingsFormProps) => {
const { t } = useTranslation();
const queryClient = useQueryClient();
const form = useZodForm(orgSettingsSchema, { defaultValues: orgToForm(org) });
const schema = React.useMemo(() => buildOrgSettingsSchema(t), [t]);
const form = useZodForm(schema, { defaultValues: orgToForm(org) });
const budgetDurationOptions = budgetDurationItems(t);
const { isDirty } = form.formState;
const mutation = useMutation({
mutationFn: (body: OrgPatchBody) => patchOrganization(organizationId, body),
onSuccess: () => {
toast.success("Organization settings updated successfully");
toast.success(t("organizations.settingsForm.toasts.updated"));
queryClient.invalidateQueries({ queryKey: organizationKeys.all });
onSaved();
},
onError: (error: unknown) =>
toast.fromError(error instanceof Error ? error.message : "Failed to update organization settings"),
toast.fromError(error instanceof Error ? error.message : t("organizations.settingsForm.toasts.updateFailed")),
});
const onSubmit = form.handleSubmit((values) => {
@ -78,11 +86,11 @@ export const OrgSettingsForm = ({
return (
<form onSubmit={onSubmit} noValidate>
<FieldGroup>
<FormField control={form.control} name="organization_alias" label="Organization Name">
<FormField control={form.control} name="organization_alias" label={t("organizations.settingsForm.fields.name")}>
{({ ref, ...field }) => <Input {...field} ref={ref} />}
</FormField>
<FormField control={form.control} name="models" label="Models">
<FormField control={form.control} name="models" label={t("organizations.settingsForm.fields.models")}>
{(field) => (
<ModelSelect
value={field.value}
@ -93,14 +101,18 @@ export const OrgSettingsForm = ({
)}
</FormField>
<FormField control={form.control} name="max_budget" label="Max Budget (USD)">
<FormField control={form.control} name="max_budget" label={t("organizations.settingsForm.fields.maxBudget")}>
{({ ref, ...field }) => <Input {...field} ref={ref} type="number" step="any" min={0} />}
</FormField>
<FormField control={form.control} name="budget_duration" label="Reset Budget">
<FormField
control={form.control}
name="budget_duration"
label={t("organizations.settingsForm.fields.budgetDuration")}
>
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
<Select
items={BUDGET_DURATION_OPTIONS}
items={budgetDurationOptions}
value={value === "" ? NO_RESET : value}
onValueChange={(selected) => onChange(selected === NO_RESET ? "" : selected)}
>
@ -108,7 +120,7 @@ export const OrgSettingsForm = ({
<SelectValue />
</SelectTrigger>
<SelectContent>
{BUDGET_DURATION_OPTIONS.map((option) => (
{budgetDurationOptions.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
</SelectItem>
@ -118,37 +130,41 @@ export const OrgSettingsForm = ({
)}
</FormField>
<FormField control={form.control} name="tpm_limit" label="Tokens per minute Limit (TPM)">
<FormField control={form.control} name="tpm_limit" label={t("organizations.settingsForm.fields.tpmLimit")}>
{({ ref, ...field }) => <Input {...field} ref={ref} type="number" step={1} min={0} />}
</FormField>
<FormField control={form.control} name="rpm_limit" label="Requests per minute Limit (RPM)">
<FormField control={form.control} name="rpm_limit" label={t("organizations.settingsForm.fields.rpmLimit")}>
{({ ref, ...field }) => <Input {...field} ref={ref} type="number" step={1} min={0} />}
</FormField>
<FormField control={form.control} name="vector_stores" label="Vector Stores">
<FormField
control={form.control}
name="vector_stores"
label={t("organizations.settingsForm.fields.vectorStores")}
>
{(field) => (
<VectorStoreSelector
value={field.value}
onChange={field.onChange}
accessToken={accessToken}
placeholder="Select vector stores"
placeholder={t("organizations.settingsForm.fields.vectorStoresPlaceholder")}
/>
)}
</FormField>
<FormField control={form.control} name="mcp" label="MCP Servers & Access Groups">
<FormField control={form.control} name="mcp" label={t("organizations.settingsForm.fields.mcp")}>
{(field) => (
<MCPServerSelector
value={field.value}
onChange={field.onChange}
accessToken={accessToken}
placeholder="Select MCP servers and access groups"
placeholder={t("organizations.settingsForm.fields.mcpPlaceholder")}
/>
)}
</FormField>
<FormField control={form.control} name="metadata" label="Metadata">
<FormField control={form.control} name="metadata" label={t("organizations.settingsForm.fields.metadata")}>
{({ ref, ...field }) => <Textarea {...field} ref={ref} rows={4} />}
</FormField>
</FieldGroup>
@ -156,10 +172,10 @@ export const OrgSettingsForm = ({
<div className="sticky z-chrome bg-card p-4 border-t border-border -bottom-6 -inset-x-6 mt-6">
<div className="flex justify-end items-center gap-2">
<Button type="button" variant="outline" onClick={onCancel} disabled={mutation.isPending}>
Cancel
{t("organizations.settingsForm.cancel")}
</Button>
<Button type="submit" disabled={!isDirty || mutation.isPending}>
{mutation.isPending ? "Saving..." : "Save Changes"}
{mutation.isPending ? t("organizations.settingsForm.saving") : t("organizations.settingsForm.save")}
</Button>
</div>
</div>

View file

@ -1,3 +1,4 @@
import type { TFunction } from "i18next";
import { z } from "zod";
const isBlank = (value: string): boolean => value.trim() === "";
@ -11,33 +12,40 @@ const isJsonObject = (value: string): boolean => {
}
};
const wholeNumberOrEmpty = z
.string()
.refine((value) => isBlank(value) || /^\d+$/.test(value.trim()), "Must be a non-negative whole number");
const wholeNumberOrEmpty = (t: TFunction) =>
z
.string()
.refine(
(value) => isBlank(value) || /^\d+$/.test(value.trim()),
t("organizations.settingsForm.errors.wholeNumber"),
);
const amountOrEmpty = z
.string()
.refine(
(value) => isBlank(value) || (Number.isFinite(Number(value)) && Number(value) >= 0),
"Must be a non-negative number",
);
const amountOrEmpty = (t: TFunction) =>
z
.string()
.refine(
(value) => isBlank(value) || (Number.isFinite(Number(value)) && Number(value) >= 0),
t("organizations.settingsForm.errors.nonNegativeNumber"),
);
const orgSettingsShape = {
organization_alias: z.string().min(1, "Please input an organization name"),
const buildOrgSettingsShape = (t: TFunction) => ({
organization_alias: z.string().min(1, t("organizations.settingsForm.errors.nameRequired")),
models: z.array(z.string()),
max_budget: amountOrEmpty,
max_budget: amountOrEmpty(t),
budget_duration: z.string(),
tpm_limit: wholeNumberOrEmpty,
rpm_limit: wholeNumberOrEmpty,
tpm_limit: wholeNumberOrEmpty(t),
rpm_limit: wholeNumberOrEmpty(t),
vector_stores: z.array(z.string()),
mcp: z.object({
servers: z.array(z.string()),
accessGroups: z.array(z.string()),
toolsets: z.array(z.string()),
}),
metadata: z.string().refine((value) => isBlank(value) || isJsonObject(value), "Metadata must be a valid JSON object"),
};
metadata: z
.string()
.refine((value) => isBlank(value) || isJsonObject(value), t("organizations.settingsForm.errors.invalidMetadata")),
});
export const orgSettingsSchema = z.object(orgSettingsShape);
export const buildOrgSettingsSchema = (t: TFunction) => z.object(buildOrgSettingsShape(t));
export type OrgSettingsFormValues = z.output<typeof orgSettingsSchema>;
export type OrgSettingsFormValues = z.output<ReturnType<typeof buildOrgSettingsSchema>>;

View file

@ -14,6 +14,7 @@ import { createTeamAliasMap } from "@/utils/teamUtils";
import { BadgeLink } from "@/components/shared/BadgeLink";
import { ArrowLeft } from "lucide-react";
import React, { useMemo, useState } from "react";
import { useTranslation } from "react-i18next";
import MemberTable, { type MemberTableColumn } from "../common_components/MemberTable";
import UserSearchModal from "../common_components/user_search_modal";
import { toast } from "@/lib/toast";
@ -46,6 +47,7 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
is_proxy_admin,
userModels,
}) => {
const { t } = useTranslation();
const queryClient = useQueryClient();
const { data: orgData, isLoading: loading } = useOrganization(organizationId);
const [isEditing, setIsEditing] = useState(false);
@ -77,11 +79,11 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
};
await organizationMemberAddCall(accessToken, organizationId, member);
toast.success("Organization member added successfully");
toast.success(t("organizations.view.toasts.memberAdded"));
setIsAddMemberModalVisible(false);
queryClient.invalidateQueries({ queryKey: organizationKeys.all });
} catch (error) {
toast.fromError("Failed to add organization member");
toast.fromError(t("organizations.view.toasts.memberAddFailed"));
console.error("Error adding organization member:", error);
}
};
@ -97,11 +99,11 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
};
await organizationMemberUpdateCall(accessToken, organizationId, member);
toast.success("Organization member updated successfully");
toast.success(t("organizations.view.toasts.memberUpdated"));
setIsEditMemberModalVisible(false);
queryClient.invalidateQueries({ queryKey: organizationKeys.all });
} catch (error) {
toast.fromError("Failed to update organization member");
toast.fromError(t("organizations.view.toasts.memberUpdateFailed"));
console.error("Error updating organization member:", error);
}
};
@ -111,21 +113,21 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
if (!accessToken) return;
await organizationMemberDeleteCall(accessToken, organizationId, values.user_id);
toast.success("Organization member deleted successfully");
toast.success(t("organizations.view.toasts.memberDeleted"));
setIsEditMemberModalVisible(false);
queryClient.invalidateQueries({ queryKey: organizationKeys.all });
} catch (error) {
toast.fromError("Failed to delete organization member");
toast.fromError(t("organizations.view.toasts.memberDeleteFailed"));
console.error("Error deleting organization member:", error);
}
};
if (loading) {
return <div className="p-4">Loading...</div>;
return <div className="p-4">{t("organizations.view.loading")}</div>;
}
if (!orgData) {
return <div className="p-4">Organization not found</div>;
return <div className="p-4">{t("organizations.view.notFound")}</div>;
}
const orgMemberById = new Map((orgData.members || []).map((m) => [m.user_id, m]));
@ -133,14 +135,14 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
const orgExtraColumns: MemberTableColumn[] = [
{
title: "Spend (USD)",
title: t("organizations.view.members.spendColumn"),
key: "spend",
numeric: true,
sortValue: (record: Member) => orgMemberFor(record)?.spend ?? null,
render: (record: Member) => <MoneyCell value={orgMemberFor(record)?.spend} decimals={4} />,
},
{
title: "Created At",
title: t("organizations.view.members.createdAtColumn"),
key: "created_at",
sortValue: (record: Member) => orgMemberFor(record)?.created_at ?? null,
render: (record: Member) => {
@ -156,12 +158,12 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
<div>
<Button variant="ghost" onClick={onClose} className="mb-4">
<ArrowLeft className="size-4" />
Back to Organizations
{t("organizations.view.backToList")}
</Button>
<h1 className="text-xl font-semibold tracking-tight text-foreground">{orgData.organization_alias}</h1>
<div className="flex items-center gap-1">
<span className="font-mono text-sm text-muted-foreground">{orgData.organization_id}</span>
<CopyButton value={orgData.organization_id} label="Copy organization ID" iconClassName="size-3" />
<CopyButton value={orgData.organization_id} label={t("organizations.view.copyId")} iconClassName="size-3" />
</div>
</div>
</div>
@ -169,13 +171,13 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
<Tabs value={tab} onValueChange={handleTabChange} className="mb-4">
<TabsList variant="line" className="h-auto w-full justify-start rounded-none border-b p-0">
<TabsTrigger value="overview" className="flex-none rounded-none px-4 py-2">
Overview
{t("organizations.view.tabs.overview")}
</TabsTrigger>
<TabsTrigger value="members" className="flex-none rounded-none px-4 py-2">
Members
{t("organizations.view.tabs.members")}
</TabsTrigger>
<TabsTrigger value="settings" className="flex-none rounded-none px-4 py-2">
Settings
{t("organizations.view.tabs.settings")}
</TabsTrigger>
</TabsList>
@ -183,11 +185,20 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
<div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-3">
<Card>
<CardContent>
<p className="text-sm text-muted-foreground">Organization Details</p>
<p className="text-sm text-muted-foreground">{t("organizations.view.overview.detailsTitle")}</p>
<div className="mt-2 text-sm text-foreground">
<p>Created: {new Date(orgData.created_at).toLocaleDateString()}</p>
<p>Updated: {new Date(orgData.updated_at).toLocaleDateString()}</p>
<p>Created By: {orgData.created_by}</p>
<p>
{t("organizations.view.overview.createdLabel")}
{new Date(orgData.created_at).toLocaleDateString()}
</p>
<p>
{t("organizations.view.overview.updatedLabel")}
{new Date(orgData.updated_at).toLocaleDateString()}
</p>
<p>
{t("organizations.view.overview.createdByLabel")}
{orgData.created_by}
</p>
</div>
</CardContent>
</Card>
@ -195,7 +206,7 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
<Card>
<CardContent>
<p className="text-sm text-muted-foreground">
Budget Status
{t("organizations.view.overview.budgetTitle")}
<BudgetPeriodTooltip
spend={orgData.spend}
maxBudget={orgData.litellm_budget_table.max_budget}
@ -206,13 +217,18 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
<div className="mt-2 text-sm text-foreground">
<p className="text-xl font-semibold">${formatNumberWithCommas(orgData.spend, 4)}</p>
<p>
of{" "}
{orgData.litellm_budget_table.max_budget === null
? "Unlimited"
: `$${formatNumberWithCommas(orgData.litellm_budget_table.max_budget, 4)}`}
{t("organizations.view.overview.budgetOf", {
budget:
orgData.litellm_budget_table.max_budget === null
? t("organizations.view.overview.unlimited")
: `$${formatNumberWithCommas(orgData.litellm_budget_table.max_budget, 4)}`,
})}
</p>
{orgData.litellm_budget_table.budget_duration && (
<p className="text-muted-foreground">Reset: {orgData.litellm_budget_table.budget_duration}</p>
<p className="text-muted-foreground">
{t("organizations.view.overview.resetLabel")}
{orgData.litellm_budget_table.budget_duration}
</p>
)}
</div>
</CardContent>
@ -220,12 +236,21 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
<Card>
<CardContent>
<p className="text-sm text-muted-foreground">Rate Limits</p>
<p className="text-sm text-muted-foreground">{t("organizations.view.overview.rateLimitsTitle")}</p>
<div className="mt-2 text-sm text-foreground">
<p>TPM: {orgData.litellm_budget_table.tpm_limit ?? "Unlimited"}</p>
<p>RPM: {orgData.litellm_budget_table.rpm_limit ?? "Unlimited"}</p>
<p>
{t("organizations.view.overview.tpmLabel")}
{orgData.litellm_budget_table.tpm_limit ?? t("organizations.view.overview.unlimited")}
</p>
<p>
{t("organizations.view.overview.rpmLabel")}
{orgData.litellm_budget_table.rpm_limit ?? t("organizations.view.overview.unlimited")}
</p>
{orgData.litellm_budget_table.max_parallel_requests && (
<p>Max Parallel Requests: {orgData.litellm_budget_table.max_parallel_requests}</p>
<p>
{t("organizations.view.overview.maxParallelLabel")}
{orgData.litellm_budget_table.max_parallel_requests}
</p>
)}
</div>
</CardContent>
@ -233,10 +258,10 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
<Card>
<CardContent>
<p className="text-sm text-muted-foreground">Models</p>
<p className="text-sm text-muted-foreground">{t("organizations.view.overview.modelsTitle")}</p>
<div className="mt-2 flex flex-wrap gap-2">
{orgData.models.length === 0 ? (
<BadgeLink>All proxy models</BadgeLink>
<BadgeLink>{t("organizations.view.overview.allProxyModels")}</BadgeLink>
) : (
orgData.models.map((model, index) => <BadgeLink key={index}>{model}</BadgeLink>)
)}
@ -246,7 +271,7 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
<Card>
<CardContent>
<p className="text-sm text-muted-foreground">Teams</p>
<p className="text-sm text-muted-foreground">{t("organizations.view.overview.teamsTitle")}</p>
<div className="mt-2 flex flex-wrap gap-2">
{orgData.teams?.map((team, index) => (
<BadgeLink key={index} href={teamDetailHref(team.team_id)}>
@ -282,9 +307,9 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
}}
onDelete={(member) => handleMemberDelete(member)}
onAddMember={() => setIsAddMemberModalVisible(true)}
roleColumnTitle="Organization Role"
roleColumnTitle={t("organizations.view.members.roleColumn")}
extraColumns={orgExtraColumns}
emptyText="No members found"
emptyText={t("organizations.view.members.empty")}
/>
</div>
</TabsContent>
@ -293,8 +318,10 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
<Card className="max-h-[65vh] overflow-y-auto">
<CardContent>
<div className="mb-4 flex items-center justify-between">
<h2 className="text-lg font-semibold text-foreground">Organization Settings</h2>
{canEditOrg && !isEditing && <Button onClick={() => setIsEditing(true)}>Edit Settings</Button>}
<h2 className="text-lg font-semibold text-foreground">{t("organizations.view.settings.title")}</h2>
{canEditOrg && !isEditing && (
<Button onClick={() => setIsEditing(true)}>{t("organizations.view.settings.editButton")}</Button>
)}
</div>
{isEditing ? (
@ -308,19 +335,19 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
) : (
<div className="space-y-4 text-sm">
<div>
<p className="font-medium text-foreground">Organization Name</p>
<p className="font-medium text-foreground">{t("organizations.view.settings.nameLabel")}</p>
<div>{orgData.organization_alias}</div>
</div>
<div>
<p className="font-medium text-foreground">Organization ID</p>
<p className="font-medium text-foreground">{t("organizations.view.settings.idLabel")}</p>
<div className="font-mono">{orgData.organization_id}</div>
</div>
<div>
<p className="font-medium text-foreground">Created At</p>
<p className="font-medium text-foreground">{t("organizations.view.settings.createdAtLabel")}</p>
<div>{new Date(orgData.created_at).toLocaleString()}</div>
</div>
<div>
<p className="font-medium text-foreground">Models</p>
<p className="font-medium text-foreground">{t("organizations.view.settings.modelsLabel")}</p>
<div className="mt-1 flex flex-wrap gap-2">
{orgData.models.map((model, index) => (
<BadgeLink key={index}>{model}</BadgeLink>
@ -328,19 +355,30 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
</div>
</div>
<div>
<p className="font-medium text-foreground">Rate Limits</p>
<div>TPM: {orgData.litellm_budget_table.tpm_limit ?? "Unlimited"}</div>
<div>RPM: {orgData.litellm_budget_table.rpm_limit ?? "Unlimited"}</div>
<p className="font-medium text-foreground">{t("organizations.view.settings.rateLimitsLabel")}</p>
<div>
{t("organizations.view.settings.tpmLabel")}
{orgData.litellm_budget_table.tpm_limit ?? t("organizations.view.settings.unlimited")}
</div>
<div>
{t("organizations.view.settings.rpmLabel")}
{orgData.litellm_budget_table.rpm_limit ?? t("organizations.view.settings.unlimited")}
</div>
</div>
<div>
<p className="font-medium text-foreground">Budget</p>
<p className="font-medium text-foreground">{t("organizations.view.settings.budgetLabel")}</p>
<div>
Max:{" "}
{orgData.litellm_budget_table.max_budget !== null
? `$${formatNumberWithCommas(orgData.litellm_budget_table.max_budget, 4)}`
: "No Limit"}
{t("organizations.view.settings.maxBudget", {
budget:
orgData.litellm_budget_table.max_budget !== null
? `$${formatNumberWithCommas(orgData.litellm_budget_table.max_budget, 4)}`
: t("organizations.view.settings.noLimit"),
})}
</div>
<div>
{t("organizations.view.settings.resetLabel")}
{orgData.litellm_budget_table.budget_duration || t("organizations.view.settings.never")}
</div>
<div>Reset: {orgData.litellm_budget_table.budget_duration || "Never"}</div>
</div>
<ObjectPermissionsView
@ -361,22 +399,22 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
onCancel={() => setIsAddMemberModalVisible(false)}
onSubmit={handleMemberAdd}
accessToken={accessToken}
title="Add Organization Member"
title={t("organizations.view.addMember.title")}
roles={[
{
label: "org_admin",
value: "org_admin",
description: "Can add and remove members, and change their roles.",
description: t("organizations.view.addMember.roles.orgAdmin"),
},
{
label: "internal_user",
value: "internal_user",
description: "Can view/create keys for themselves within organization.",
description: t("organizations.view.addMember.roles.internalUser"),
},
{
label: "internal_user_viewer",
value: "internal_user_viewer",
description: "Can only view their keys within organization.",
description: t("organizations.view.addMember.roles.internalUserViewer"),
},
]}
defaultRole="internal_user"
@ -388,13 +426,13 @@ const OrganizationInfoView: React.FC<OrganizationInfoProps> = ({
initialData={selectedEditMember}
mode="edit"
config={{
title: "Edit Member",
title: t("organizations.view.editMember.title"),
showEmail: true,
showUserId: true,
roleOptions: [
{ label: "Org Admin", value: "org_admin" },
{ label: "Internal User", value: "internal_user" },
{ label: "Internal User Viewer", value: "internal_user_viewer" },
{ label: t("organizations.view.editMember.roles.orgAdmin"), value: "org_admin" },
{ label: t("organizations.view.editMember.roles.internalUser"), value: "internal_user" },
{ label: t("organizations.view.editMember.roles.internalUserViewer"), value: "internal_user_viewer" },
],
}}
/>

View file

@ -1,4 +1,5 @@
import React, { useState, useEffect } from "react";
import { useTranslation } from "react-i18next";
import { availableTeamListCall, teamMemberAddCall } from "@/components/networking";
import { toast } from "@/lib/toast";
@ -12,6 +13,7 @@ interface AvailableTeamsProps {
}
const AvailableTeamsPanel: React.FC<AvailableTeamsProps> = ({ accessToken, userID }) => {
const { t } = useTranslation();
const [availableTeams, setAvailableTeams] = useState<AvailableTeam[]>([]);
const [isLoading, setIsLoading] = useState(true);
@ -54,11 +56,11 @@ const AvailableTeamsPanel: React.FC<AvailableTeamsProps> = ({ accessToken, userI
role: "user",
});
toast.success("Successfully joined team");
toast.success(t("teams.availableTeams.panel.toasts.joined"));
setAvailableTeams((teams) => teams.filter((team) => team.team_id !== teamId));
} catch (error) {
console.error("Error joining team:", error);
toast.fromError("Failed to join team");
toast.fromError(t("teams.availableTeams.panel.toasts.joinFailed"));
}
};

View file

@ -3,6 +3,7 @@
import { SortingState } from "@tanstack/react-table";
import { Users } from "lucide-react";
import React, { useMemo, useState } from "react";
import { Trans, useTranslation } from "react-i18next";
import { DataTable } from "@/components/shared/DataTable";
@ -17,31 +18,37 @@ interface AvailableTeamsTableProps {
const DEFAULT_SORTING: SortingState = [{ id: "team_alias", desc: false }];
function EmptyState() {
const { t } = useTranslation();
return (
<div className="flex flex-col items-center gap-1 py-6">
<div className="mb-1 flex size-10 items-center justify-center rounded-lg bg-muted">
<Users className="size-5 text-muted-foreground" />
</div>
<div className="text-sm font-medium text-foreground">No available teams to join</div>
<div className="text-sm font-medium text-foreground">{t("teams.availableTeams.table.emptyTitle")}</div>
<div className="text-sm text-muted-foreground">
See how to set available teams{" "}
<a
href="https://docs.litellm.ai/docs/proxy/self_serve#all-settings-for-self-serve--sso-flow"
target="_blank"
rel="noopener noreferrer"
className="text-primary underline-offset-4 hover:underline"
>
here
</a>
<Trans
i18nKey="teams.availableTeams.table.emptyDocs"
components={{
docs: (
<a
href="https://docs.litellm.ai/docs/proxy/self_serve#all-settings-for-self-serve--sso-flow"
target="_blank"
rel="noopener noreferrer"
className="text-primary underline-offset-4 hover:underline"
/>
),
}}
/>
</div>
</div>
);
}
const AvailableTeamsTable: React.FC<AvailableTeamsTableProps> = ({ teams, isLoading, onJoinTeam }) => {
const { t } = useTranslation();
const [sorting, setSorting] = useState<SortingState>(DEFAULT_SORTING);
const columns = useMemo(() => getAvailableTeamsTableColumns({ onJoinTeam }), [onJoinTeam]);
const columns = useMemo(() => getAvailableTeamsTableColumns({ onJoinTeam, t }), [onJoinTeam, t]);
return (
<DataTable
@ -53,7 +60,7 @@ const AvailableTeamsTable: React.FC<AvailableTeamsTableProps> = ({ teams, isLoad
sorting={sorting}
onSortingChange={setSorting}
isLoading={isLoading}
loadingMessage="Loading available teams…"
loadingMessage={t("teams.availableTeams.table.loading")}
noDataMessage={<EmptyState />}
size="compact"
/>

View file

@ -1,7 +1,9 @@
"use client";
import { ColumnDef } from "@tanstack/react-table";
import type { TFunction } from "i18next";
import { MoreHorizontal, UserPlus } from "lucide-react";
import { useTranslation } from "react-i18next";
import { DataTableSortHeader } from "@/components/shared/DataTable";
import { IdentityCell, ModelsCell } from "@/components/shared/table_cells";
@ -23,10 +25,11 @@ export interface AvailableTeam {
}
function AvailableTeamRowActions({ team, onJoinTeam }: { team: AvailableTeam; onJoinTeam: (teamId: string) => void }) {
const { t } = useTranslation();
return (
<DropdownMenu>
<DropdownMenuTrigger
aria-label="Open team actions"
aria-label={t("teams.availableTeams.columns.openActions")}
data-testid={`available-team-actions-${team.team_id}`}
className={cn(buttonVariants({ variant: "ghost", size: "icon-sm" }), "text-muted-foreground")}
>
@ -35,7 +38,7 @@ function AvailableTeamRowActions({ team, onJoinTeam }: { team: AvailableTeam; on
<DropdownMenuContent align="end" className="w-44">
<DropdownMenuItem data-testid="available-team-action-join" onClick={() => onJoinTeam(team.team_id)}>
<UserPlus />
Join team
{t("teams.availableTeams.columns.join")}
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
@ -44,16 +47,18 @@ function AvailableTeamRowActions({ team, onJoinTeam }: { team: AvailableTeam; on
interface AvailableTeamsTableColumnsDeps {
onJoinTeam: (teamId: string) => void;
t: TFunction;
}
export const getAvailableTeamsTableColumns = ({
onJoinTeam,
t,
}: AvailableTeamsTableColumnsDeps): ColumnDef<AvailableTeam>[] => [
{
id: "team_alias",
accessorKey: "team_alias",
meta: { title: "Team Name" },
header: ({ column }) => <DataTableSortHeader column={column} title="Team Name" />,
meta: { title: t("teams.availableTeams.columns.teamName") },
header: ({ column }) => <DataTableSortHeader column={column} title={t("teams.availableTeams.columns.teamName")} />,
size: 220,
enableSorting: true,
cell: ({ row }) => (
@ -63,15 +68,15 @@ export const getAvailableTeamsTableColumns = ({
{
id: "description",
accessorKey: "description",
meta: { title: "Description" },
header: "Description",
meta: { title: t("teams.availableTeams.columns.description") },
header: t("teams.availableTeams.columns.description"),
size: 280,
enableSorting: false,
cell: ({ row }) => {
const description = row.original.description;
return (
<span className="block max-w-72 truncate text-sm text-muted-foreground" title={description || undefined}>
{description || "No description available"}
{description || t("teams.availableTeams.columns.noDescription")}
</span>
);
},
@ -79,18 +84,20 @@ export const getAvailableTeamsTableColumns = ({
{
id: "members",
accessorFn: (team) => team.members_with_roles.length,
meta: { title: "Members" },
header: ({ column }) => <DataTableSortHeader column={column} title="Members" />,
meta: { title: t("teams.availableTeams.columns.members") },
header: ({ column }) => <DataTableSortHeader column={column} title={t("teams.availableTeams.columns.members")} />,
size: 120,
enableSorting: true,
cell: ({ row }) => (
<span className="text-sm text-muted-foreground">{row.original.members_with_roles.length} members</span>
<span className="text-sm text-muted-foreground">
{t("teams.availableTeams.columns.membersCount", { count: row.original.members_with_roles.length })}
</span>
),
},
{
id: "models",
meta: { title: "Models" },
header: "Models",
meta: { title: t("teams.availableTeams.columns.models") },
header: t("teams.availableTeams.columns.models"),
size: 260,
enableSorting: false,
cell: ({ row }) => <ModelsCell models={row.original.models} />,
@ -98,7 +105,7 @@ export const getAvailableTeamsTableColumns = ({
{
id: "actions",
meta: { className: "text-right", headerClassName: "text-right" },
header: () => <span className="sr-only">Actions</span>,
header: () => <span className="sr-only">{t("teams.availableTeams.columns.actions")}</span>,
size: 64,
enableSorting: false,
enableHiding: false,

Some files were not shown because too many files have changed in this diff Show more