mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
Merge branch 'litellm_internal_staging' into litellm_/magical-albattani-a2a51e
This commit is contained in:
yuneng-jiang
GitHub
commit
6866cd1d48
27 changed files with 948 additions and 21 deletions
|
|
@ -17,3 +17,24 @@
|
|||
|
||||
# style: unify ruff format width on 120 (#31518)
|
||||
48b5a5a0cc5a694a11219416ee0b6eb6e620e74e
|
||||
|
||||
# refactor(imports): move collections.abc names out of typing (#35495)
|
||||
397e8e4918777e4e60a7f5e88699e0a9a7dabb3d
|
||||
|
||||
# refactor(lint): apply every safe ruff autofix and zero 28 strict-rule budgets (#35495)
|
||||
b604e2b20c6db2099085a2f0e59b7e99e87eed6f
|
||||
|
||||
# refactor(logging): drop redundant !s conversion flags from f-strings (#35546)
|
||||
7b2d3440cba3160277470f7a0180098ae9b87864
|
||||
|
||||
# perf: build log messages lazily so filtered-out log records cost nothing (#35703)
|
||||
c9887a1f94bc1e7e4bdfe64d640f0509a0bc19dd
|
||||
|
||||
# feat(lint): enforce Final on locals and freeze function parameters (#35807)
|
||||
2708620d6a599cc73c1950a942d26ac26a7ed3d4
|
||||
|
||||
# chore(lint): remove litellm/types from the ruff lint exclusion (#35926)
|
||||
4e32a8bf6a1e1af1e04b67c759841ccef44b2235
|
||||
|
||||
# chore(lint): strip inert type: ignore comments and zero LIT009/LIT010/LIT011 headroom (#35928)
|
||||
338e411103ad5d7003e97f34f04fa36bca542dbe
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.53"
|
||||
version = "0.1.54"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.53"
|
||||
version = "0.1.54"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-enterprise==",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.83"
|
||||
version = "0.4.84"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.83"
|
||||
version = "0.4.84"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -115,8 +115,11 @@ def get_litellm_params(
|
|||
litellm_request_debug: bool | None = None,
|
||||
**kwargs,
|
||||
) -> dict:
|
||||
_litellm_metadata_dict: Final = litellm_metadata if isinstance(litellm_metadata, dict) else None
|
||||
resolved_metadata: Final = _litellm_metadata_dict.copy() if not metadata and _litellm_metadata_dict else metadata
|
||||
|
||||
# Derive litellm_session_id / litellm_trace_id from metadata when not provided (call chaining)
|
||||
_meta: Final = metadata or {}
|
||||
_meta: Final = resolved_metadata or {}
|
||||
if litellm_session_id is None:
|
||||
litellm_session_id = _meta.get("session_id") or _meta.get("trace_id")
|
||||
if litellm_trace_id is None:
|
||||
|
|
@ -139,7 +142,7 @@ def get_litellm_params(
|
|||
"model_alias_map": model_alias_map,
|
||||
"completion_call_id": completion_call_id,
|
||||
"aembedding": aembedding,
|
||||
"metadata": metadata,
|
||||
"metadata": resolved_metadata,
|
||||
"model_info": model_info,
|
||||
"proxy_server_request": proxy_server_request,
|
||||
"preset_cache_key": preset_cache_key,
|
||||
|
|
|
|||
|
|
@ -585,8 +585,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"""
|
||||
base_litellm_params: Final[dict[str, Any]] = {}
|
||||
|
||||
if "metadata" in kwargs:
|
||||
base_litellm_params["metadata"] = kwargs["metadata"]
|
||||
if isinstance(kwargs.get("metadata"), dict):
|
||||
base_litellm_params["metadata"] = kwargs["metadata"].copy()
|
||||
if "litellm_metadata" in kwargs and isinstance(kwargs["litellm_metadata"], dict):
|
||||
base_litellm_params["litellm_metadata"] = kwargs["litellm_metadata"]
|
||||
if "metadata" not in base_litellm_params:
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
|
|||
send_user_api_key_alias=litellm_params.send_user_api_key_alias,
|
||||
send_user_api_key_user_id=litellm_params.send_user_api_key_user_id,
|
||||
send_user_api_key_team_id=litellm_params.send_user_api_key_team_id,
|
||||
timeout=litellm_params.timeout,
|
||||
guardrail_name=guardrail.get("guardrail_name", ""),
|
||||
event_hook=litellm_params.mode,
|
||||
default_on=litellm_params.default_on,
|
||||
|
|
|
|||
|
|
@ -22,9 +22,10 @@ from litellm.types.utils import GenericGuardrailAPIInputs
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.guardrails import LitellmParams
|
||||
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
|
||||
|
||||
GUARDRAIL_TIMEOUT: Final = 5
|
||||
DEFAULT_GUARDRAIL_TIMEOUT: Final = 5.0
|
||||
|
||||
|
||||
class ZscalerAIGuard(CustomGuardrail):
|
||||
|
|
@ -43,6 +44,7 @@ class ZscalerAIGuard(CustomGuardrail):
|
|||
send_user_api_key_alias: bool | None = None,
|
||||
send_user_api_key_user_id: bool | None = None,
|
||||
send_user_api_key_team_id: bool | None = None,
|
||||
timeout: float | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks()))
|
||||
|
|
@ -68,6 +70,7 @@ class ZscalerAIGuard(CustomGuardrail):
|
|||
if send_user_api_key_team_id is not None
|
||||
else os.getenv("SEND_USER_API_KEY_TEAM_ID", "False").lower() in ("true", "1")
|
||||
)
|
||||
self.timeout = self._resolve_timeout(timeout)
|
||||
|
||||
verbose_proxy_logger.debug(
|
||||
"send_user_api_key_alias: %s, \n send_user_api_key_user_id:%s, \n send_user_api_key_team_id:%s",
|
||||
|
|
@ -80,6 +83,29 @@ class ZscalerAIGuard(CustomGuardrail):
|
|||
|
||||
verbose_proxy_logger.debug("ZscalerAIGuard Initializing ...")
|
||||
|
||||
@staticmethod
|
||||
def _resolve_timeout(timeout: float | None) -> float:
|
||||
"""
|
||||
Resolve the effective per-request timeout, falling back to the default
|
||||
when it is unset or non-positive.
|
||||
"""
|
||||
if timeout is None:
|
||||
return DEFAULT_GUARDRAIL_TIMEOUT
|
||||
|
||||
if timeout <= 0:
|
||||
verbose_proxy_logger.warning(
|
||||
"Ignoring non-positive Zscaler AI Guard timeout %s, using %s seconds",
|
||||
timeout,
|
||||
DEFAULT_GUARDRAIL_TIMEOUT,
|
||||
)
|
||||
return DEFAULT_GUARDRAIL_TIMEOUT
|
||||
|
||||
return timeout
|
||||
|
||||
def update_in_memory_litellm_params(self, litellm_params: "LitellmParams") -> None:
|
||||
super().update_in_memory_litellm_params(litellm_params)
|
||||
self.timeout = self._resolve_timeout(litellm_params.timeout)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_metadata_value(request_data: dict | None, key: str) -> str | None:
|
||||
"""
|
||||
|
|
@ -267,7 +293,7 @@ class ZscalerAIGuard(CustomGuardrail):
|
|||
f"{url}",
|
||||
headers=headers,
|
||||
json=data,
|
||||
timeout=GUARDRAIL_TIMEOUT,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -151,6 +151,20 @@ LITELLM_METADATA_ROUTES: Final = (
|
|||
"files",
|
||||
)
|
||||
|
||||
LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset(
|
||||
{
|
||||
"mask_input",
|
||||
"mask_output",
|
||||
"session_id",
|
||||
"trace_id",
|
||||
"trace_metadata",
|
||||
"trace_name",
|
||||
"trace_release",
|
||||
"trace_user_id",
|
||||
"trace_version",
|
||||
}
|
||||
)
|
||||
|
||||
_UNTRUSTED_ROOT_CONTROL_FIELDS: Final = (
|
||||
"proxy_server_request",
|
||||
"standard_logging_object",
|
||||
|
|
@ -458,6 +472,18 @@ def _get_metadata_variable_name(request: Request) -> str:
|
|||
return "metadata"
|
||||
|
||||
|
||||
def _promoted_trace_control_fields(
|
||||
requester_metadata: Mapping[str, Any],
|
||||
litellm_metadata: Mapping[str, Any],
|
||||
) -> tuple[tuple[str, Any], ...]:
|
||||
"""Return the caller's trace-control fields that ``litellm_metadata`` does not already set."""
|
||||
return tuple(
|
||||
(key, value)
|
||||
for key, value in requester_metadata.items()
|
||||
if key in LITELLM_TRACE_CONTROL_METADATA_FIELDS and key not in litellm_metadata
|
||||
)
|
||||
|
||||
|
||||
def _extract_generic_session_id_from_headers(
|
||||
normalized: dict[str, str],
|
||||
) -> str | None:
|
||||
|
|
@ -1670,6 +1696,13 @@ async def add_litellm_data_to_request(
|
|||
# paths may read from it.
|
||||
if "metadata" in data and isinstance(data["metadata"], dict):
|
||||
data[_metadata_variable_name]["requester_metadata"] = copy.deepcopy(data["metadata"])
|
||||
if _metadata_variable_name == "litellm_metadata":
|
||||
data[_metadata_variable_name].update(
|
||||
_promoted_trace_control_fields(
|
||||
requester_metadata=data[_metadata_variable_name]["requester_metadata"],
|
||||
litellm_metadata=data[_metadata_variable_name],
|
||||
)
|
||||
)
|
||||
|
||||
# Merge litellm_metadata into the metadata variable (preserving existing
|
||||
# values). Runs after the user_api_key_* / _pipeline_managed_guardrails
|
||||
|
|
|
|||
|
|
@ -565,6 +565,9 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
# real parent span.
|
||||
_metadata["user_api_key"] = user_api_key_dict.api_key
|
||||
_metadata["litellm_parent_otel_span"] = user_api_key_dict.parent_otel_span
|
||||
_metadata.update(
|
||||
LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=user_api_key_dict)
|
||||
)
|
||||
|
||||
kwargs: Final = {
|
||||
"litellm_params": {
|
||||
|
|
|
|||
|
|
@ -1111,6 +1111,10 @@ async def proxy_startup_event(app: FastAPI):
|
|||
prisma_client=prisma_client,
|
||||
)
|
||||
)
|
||||
ProxyStartupEvent._warn_budget_without_db(
|
||||
max_budget=litellm.max_budget,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
|
||||
### START BATCH WRITING DB + CHECKING NEW MODELS###
|
||||
if prisma_client is not None:
|
||||
|
|
@ -7825,6 +7829,19 @@ def giveup(e):
|
|||
|
||||
|
||||
class ProxyStartupEvent:
|
||||
@staticmethod
|
||||
def _warn_budget_without_db(max_budget: float | None, prisma_client: PrismaClient | None) -> None:
|
||||
if prisma_client is not None or not max_budget or max_budget <= 0:
|
||||
return
|
||||
|
||||
verbose_proxy_logger.warning(
|
||||
"A proxy-wide budget (litellm.max_budget=%s) is configured but no database is connected, "
|
||||
"so the budget will NOT be enforced and requests will never be blocked. Set DATABASE_URL or "
|
||||
"general_settings.database_url and restart. Redis and fail_closed_budget_enforcement do not "
|
||||
"cover the proxy-wide budget because there is no global spend counter; Redis alone is not a substitute.",
|
||||
max_budget,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _initialize_startup_logging(
|
||||
cls,
|
||||
|
|
|
|||
|
|
@ -79,6 +79,15 @@ class ZscalerAIGuardConfigModel(GuardrailConfigModel):
|
|||
json_schema_extra={"ui_type": GuardrailParamUITypes.BOOL},
|
||||
)
|
||||
|
||||
timeout: float | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Timeout for each Zscaler AI Guard API call, in seconds. Must be positive. "
|
||||
"Raise it if scans fail under load with 'Connection timed out'. "
|
||||
"Defaults to 5 seconds."
|
||||
),
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_endpoint_configuration(self) -> "ZscalerAIGuardConfigModel":
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -66,8 +66,8 @@ proxy = [
|
|||
"azure-identity>=1.25.2,<2.0",
|
||||
"azure-storage-blob>=12.28.0,<13.0",
|
||||
"mcp>=1.28.1,<2.0",
|
||||
"litellm-proxy-extras==0.4.83",
|
||||
"litellm-enterprise==0.1.53",
|
||||
"litellm-proxy-extras==0.4.84",
|
||||
"litellm-enterprise==0.1.54",
|
||||
"RestrictedPython>=8.1,<9.0",
|
||||
"rich>=13.9.4,<14.0",
|
||||
"InquirerPy>=0.3.4,<1.0",
|
||||
|
|
|
|||
|
|
@ -7,13 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.4.0] - 2026-08-06
|
||||
|
||||
### Fixed
|
||||
|
||||
- **organization**: Send `PATCH` instead of `POST` to `/organization/update` and `/organization/member_update`, matching the methods the LiteLLM proxy serves; organization and organization member updates previously failed with a 405
|
||||
- **team_member**: Include `role` in the update payload so a role change on an existing `litellm_team_member` is applied instead of being silently dropped
|
||||
|
||||
### Changed
|
||||
|
||||
- The provider source of truth moved to `terraform/provider/` in [BerriAI/litellm](https://github.com/BerriAI/litellm); this repository is now a release mirror. CI in the monorepo statically audits every endpoint the provider calls against the proxy's OpenAPI schema on every change
|
||||
- **mcp_server**, **vector_store**: `env` and `litellm_params` are now marked sensitive, so they are redacted from plan/apply output, and they are no longer read back from the API into state — the configured value is authoritative. If the proxy returns values that differ from the configuration, that drift is no longer surfaced on refresh
|
||||
- Dependency updates: `grpc` and `golang.org/x` modules
|
||||
|
||||
## [0.3.0] - 2026-07-13
|
||||
|
||||
Released from the mirror repository before the source move was complete; this entry backfills it in the monorepo changelog.
|
||||
|
||||
### Added
|
||||
|
||||
- **model**: Add optional `pricing_base_model` attribute that sets `model_info.base_model` (the cost-map lookup key) independently of routing. Deployments whose routing name differs from the pricing key (for example Azure Data Zone, routed as `azure/gpt-4.1` but priced via `us/gpt-4.1-2025-04-14`) can now be billed correctly without breaking routing. When unset, behavior is unchanged and `base_model` continues to drive both routing and pricing (#47)
|
||||
|
||||
## [0.2.2] - 2026-05-13
|
||||
|
||||
|
|
|
|||
|
|
@ -118,6 +118,8 @@ The following arguments are supported:
|
|||
|
||||
* `base_model` - (Required) string. The actual model identifier from the provider (e.g., "gpt-4", "claude-2").
|
||||
|
||||
* `pricing_base_model` - (Optional) string. A pricing key fed to `model_info.base_model` **independently of routing**. When set, `litellm_params.model` still routes via `base_model`, but LiteLLM looks up cost against this key. Useful when the routing/deployment name differs from the cost-map key — e.g. an Azure deployment routed as `azure/gpt-4.1` whose real tier is Data Zone: set `pricing_base_model = "us/gpt-4.1-2025-04-14"` so it is billed at the Data Zone rate. When unset, `base_model` drives pricing as before.
|
||||
|
||||
* `litellm_credential_name` - (Optional) string. Name of a LiteLLM credential to use for this model.
|
||||
|
||||
* `tier` - (Optional) string. The usage tier for this model. Valid values are `"free"` or `"paid"`. Default: `"free"`.
|
||||
|
|
|
|||
|
|
@ -73,6 +73,14 @@ func resourceLiteLLMModel() *schema.Resource {
|
|||
Type: schema.TypeString,
|
||||
Required: true,
|
||||
},
|
||||
"pricing_base_model": {
|
||||
// Optional pricing key fed to model_info.base_model, DECOUPLED
|
||||
// from routing. When set, litellm_params.model still routes via
|
||||
// base_model, but cost is looked up against this key (e.g.
|
||||
// "us/gpt-4.1-2025-04-14" for Azure Data Zone pricing).
|
||||
Type: schema.TypeString,
|
||||
Optional: true,
|
||||
},
|
||||
"tier": {
|
||||
Type: schema.TypeString,
|
||||
Optional: true,
|
||||
|
|
|
|||
|
|
@ -68,6 +68,14 @@ func createOrUpdateModel(d *schema.ResourceData, m interface{}, isUpdate bool) e
|
|||
baseModel := d.Get("base_model").(string)
|
||||
modelName := fmt.Sprintf("%s/%s", customLLMProvider, baseModel)
|
||||
|
||||
// Pricing base_model, decoupled from routing. When pricing_base_model is
|
||||
// set it feeds model_info.base_model (the cost-lookup key) WITHOUT changing
|
||||
// the routing string above; otherwise base_model drives pricing as before.
|
||||
pricingBaseModel := baseModel
|
||||
if v, ok := d.GetOk("pricing_base_model"); ok && v.(string) != "" {
|
||||
pricingBaseModel = v.(string)
|
||||
}
|
||||
|
||||
// Generate a UUID for new models
|
||||
modelID := d.Id()
|
||||
if !isUpdate {
|
||||
|
|
@ -240,7 +248,7 @@ func createOrUpdateModel(d *schema.ResourceData, m interface{}, isUpdate bool) e
|
|||
ModelInfo: ModelInfo{
|
||||
ID: modelID,
|
||||
DBModel: true,
|
||||
BaseModel: baseModel,
|
||||
BaseModel: pricingBaseModel,
|
||||
Tier: d.Get("tier").(string),
|
||||
Mode: d.Get("mode").(string),
|
||||
TeamID: d.Get("team_id").(string),
|
||||
|
|
@ -306,7 +314,16 @@ func resourceLiteLLMModelRead(d *schema.ResourceData, m interface{}) error {
|
|||
d.Set("rpm", GetIntValue(modelResp.LiteLLMParams.RPM, d.Get("rpm").(int)))
|
||||
d.Set("model_api_base", GetStringValue(modelResp.LiteLLMParams.APIBase, d.Get("model_api_base").(string)))
|
||||
d.Set("api_version", GetStringValue(modelResp.LiteLLMParams.APIVersion, d.Get("api_version").(string)))
|
||||
d.Set("base_model", GetStringValue(modelResp.ModelInfo.BaseModel, d.Get("base_model").(string)))
|
||||
// base_model / pricing_base_model read-back. When pricing_base_model is
|
||||
// configured, model_info.base_model holds the PRICING key, so recover the
|
||||
// routing base_model from state (not returned by the API) and read
|
||||
// pricing_base_model from model_info.
|
||||
if pbm, ok := d.GetOk("pricing_base_model"); ok && pbm.(string) != "" {
|
||||
d.Set("base_model", d.Get("base_model").(string))
|
||||
d.Set("pricing_base_model", GetStringValue(modelResp.ModelInfo.BaseModel, pbm.(string)))
|
||||
} else {
|
||||
d.Set("base_model", GetStringValue(modelResp.ModelInfo.BaseModel, d.Get("base_model").(string)))
|
||||
}
|
||||
d.Set("tier", GetStringValue(modelResp.ModelInfo.Tier, d.Get("tier").(string)))
|
||||
d.Set("mode", GetStringValue(modelResp.ModelInfo.Mode, d.Get("mode").(string)))
|
||||
d.Set("team_id", GetStringValue(modelResp.ModelInfo.TeamID, d.Get("team_id").(string)))
|
||||
|
|
|
|||
|
|
@ -396,3 +396,122 @@ async def test_apply_guardrail_block_does_not_log_error(mock_api_call):
|
|||
mock_logger.error.assert_not_called()
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_request_uses_default_timeout_when_unconfigured():
|
||||
"""
|
||||
Regression: unconfigured guardrails must keep the historical 5s timeout.
|
||||
"""
|
||||
guardrail = ZscalerAIGuard(api_key="test_key", policy_id=1)
|
||||
|
||||
assert guardrail.timeout == 5.0
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.guardrails.guardrail_hooks.zscaler_ai_guard.zscaler_ai_guard.get_async_httpx_client"
|
||||
) as mock_get_client:
|
||||
mock_client = Mock()
|
||||
mock_client.post = AsyncMock(return_value=Mock(status_code=200))
|
||||
mock_get_client.return_value = mock_client
|
||||
|
||||
await guardrail._send_request("http://example.com", {}, {})
|
||||
|
||||
assert mock_client.post.call_args.kwargs["timeout"] == 5.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_request_uses_configured_timeout():
|
||||
"""
|
||||
Regression for LIT-5222: a configured timeout must reach the HTTP call.
|
||||
|
||||
Before the fix _send_request passed a module-level constant, so a slow
|
||||
upstream failed at 5s with `Timeout passed=5` no matter what was configured.
|
||||
"""
|
||||
guardrail = ZscalerAIGuard(api_key="test_key", policy_id=1, timeout=30)
|
||||
|
||||
assert guardrail.timeout == 30
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.guardrails.guardrail_hooks.zscaler_ai_guard.zscaler_ai_guard.get_async_httpx_client"
|
||||
) as mock_get_client:
|
||||
mock_client = Mock()
|
||||
mock_client.post = AsyncMock(return_value=Mock(status_code=200))
|
||||
mock_get_client.return_value = mock_client
|
||||
|
||||
await guardrail._send_request("http://example.com", {}, {})
|
||||
|
||||
assert mock_client.post.call_args.kwargs["timeout"] == 30
|
||||
|
||||
|
||||
def test_initialize_guardrail_forwards_configured_timeout():
|
||||
"""
|
||||
Regression for LIT-5222: the `timeout` key from config.yaml must survive
|
||||
initialization. It reaches LitellmParams already, but the initializer used
|
||||
to drop it before it could reach the guardrail instance.
|
||||
"""
|
||||
from litellm.proxy.guardrails.guardrail_hooks.zscaler_ai_guard import (
|
||||
initialize_guardrail,
|
||||
)
|
||||
from litellm.types.guardrails import LitellmParams
|
||||
|
||||
litellm_params = LitellmParams(
|
||||
guardrail="zscaler_ai_guard",
|
||||
mode="pre_call",
|
||||
api_key="test_key",
|
||||
api_base="http://example.com",
|
||||
policy_id=1,
|
||||
timeout="30",
|
||||
)
|
||||
|
||||
guardrail = initialize_guardrail(
|
||||
litellm_params, {"guardrail_name": "zscaler-configured-timeout"}
|
||||
)
|
||||
|
||||
assert guardrail.timeout == 30.0
|
||||
|
||||
|
||||
def test_config_model_exposes_timeout_to_dashboard():
|
||||
"""
|
||||
The dashboard guardrail form is built from get_config_model(), so the field
|
||||
has to be declared there for the setting to be reachable outside config.yaml.
|
||||
"""
|
||||
config_model = ZscalerAIGuard.get_config_model()
|
||||
|
||||
assert config_model is not None
|
||||
assert "timeout" in config_model.model_fields
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad_timeout", [0, -1])
|
||||
def test_non_positive_timeout_falls_back_to_default(bad_timeout):
|
||||
"""
|
||||
Regression: httpx rejects a negative timeout and treats 0 as "fail
|
||||
immediately", so a non-positive value would break every scan instead of
|
||||
relaxing the limit the operator was trying to raise.
|
||||
"""
|
||||
guardrail = ZscalerAIGuard(api_key="test_key", policy_id=1, timeout=bad_timeout)
|
||||
|
||||
assert guardrail.timeout == 5.0
|
||||
|
||||
|
||||
def test_update_in_memory_litellm_params_keeps_timeout_resolved():
|
||||
"""
|
||||
Regression: the base implementation copies every LitellmParams attribute
|
||||
onto the guardrail, so an unset timeout would overwrite the resolved value
|
||||
with None and silently fall back to the shared client's 600s default.
|
||||
"""
|
||||
from litellm.types.guardrails import LitellmParams
|
||||
|
||||
guardrail = ZscalerAIGuard(api_key="test_key", policy_id=1, timeout=30)
|
||||
assert guardrail.timeout == 30
|
||||
|
||||
guardrail.update_in_memory_litellm_params(
|
||||
LitellmParams(guardrail="zscaler_ai_guard", mode="pre_call", api_key="test_key")
|
||||
)
|
||||
assert guardrail.timeout == 5.0
|
||||
|
||||
guardrail.update_in_memory_litellm_params(
|
||||
LitellmParams(
|
||||
guardrail="zscaler_ai_guard", mode="pre_call", api_key="test_key", timeout=45
|
||||
)
|
||||
)
|
||||
assert guardrail.timeout == 45.0
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
|
|||
)
|
||||
from fastapi import Request
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
|
||||
_update_metadata_with_tags_in_header,
|
||||
HttpPassThroughEndpointHelpers,
|
||||
|
|
@ -652,3 +653,119 @@ def test_custom_pricing_used_in_cost_calculation():
|
|||
|
||||
print(f"Cache-aware cost: {cache_cost}")
|
||||
print("✅ Custom pricing parameters are correctly used in cost calculation")
|
||||
|
||||
|
||||
def test_init_kwargs_client_metadata_cannot_spoof_authenticated_identity(
|
||||
mock_request, mock_user_api_key_dict
|
||||
):
|
||||
request = mock_request()
|
||||
passthrough_payload = PassthroughStandardLoggingPayload(
|
||||
url="https://test.com",
|
||||
request_body={},
|
||||
)
|
||||
authenticated_key = UserAPIKeyAuth(
|
||||
api_key="test-key",
|
||||
user_id="test-user",
|
||||
team_id="test-team",
|
||||
end_user_id="test-user",
|
||||
key_alias="real-key",
|
||||
team_alias="Real Team",
|
||||
user_email="real@example.com",
|
||||
org_id="real-org",
|
||||
)
|
||||
|
||||
result = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint(
|
||||
request=request,
|
||||
user_api_key_dict=authenticated_key,
|
||||
passthrough_logging_payload=passthrough_payload,
|
||||
litellm_call_id="test-call-id",
|
||||
logging_obj=LiteLLMLoggingObj(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="test-call-type",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="test-call-id",
|
||||
function_id="test-function-id",
|
||||
),
|
||||
_parsed_body={
|
||||
"litellm_metadata": {
|
||||
"user_api_key_org_id": "victim-org",
|
||||
"user_api_key_end_user_id": "victim-end-user",
|
||||
"user_api_key_user_id": "victim-user",
|
||||
"user_api_key_team_id": "victim-team",
|
||||
"user_api_key_team_alias": "Victim Team",
|
||||
"user_api_key_alias": "victim-key",
|
||||
"user_api_key_user_email": "victim@example.com",
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
metadata = result["litellm_params"]["metadata"]
|
||||
assert metadata["user_api_key_user_id"] == "test-user"
|
||||
assert metadata["user_api_key_team_id"] == "test-team"
|
||||
assert metadata["user_api_key_team_alias"] == "Real Team"
|
||||
assert metadata["user_api_key_alias"] == "real-key"
|
||||
assert metadata["user_api_key_user_email"] == "real@example.com"
|
||||
assert metadata["user_api_key_org_id"] == "real-org"
|
||||
assert metadata["user_api_key_end_user_id"] == "test-user"
|
||||
|
||||
|
||||
def test_init_kwargs_no_authenticated_identity_field_is_client_settable(
|
||||
mock_request, mock_user_api_key_dict
|
||||
):
|
||||
authenticated_key = UserAPIKeyAuth(
|
||||
api_key="test-key",
|
||||
user_id="test-user",
|
||||
team_id="test-team",
|
||||
end_user_id="test-end-user",
|
||||
key_alias="real-key",
|
||||
team_alias="Real Team",
|
||||
user_email="real@example.com",
|
||||
org_id="real-org",
|
||||
organization_alias="Real Org",
|
||||
project_id="real-project",
|
||||
project_alias="Real Project",
|
||||
spend=1.5,
|
||||
max_budget=10.0,
|
||||
user_spend=2.5,
|
||||
user_max_budget=20.0,
|
||||
team_spend=3.5,
|
||||
team_max_budget=30.0,
|
||||
metadata={"real": "auth-metadata"},
|
||||
)
|
||||
expected = dict(
|
||||
LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(
|
||||
user_api_key_dict=authenticated_key
|
||||
)
|
||||
)
|
||||
assert len(expected) >= 20
|
||||
|
||||
spoofed = {key: f"SPOOFED-{key}" for key in expected}
|
||||
|
||||
result = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint(
|
||||
request=mock_request(),
|
||||
user_api_key_dict=authenticated_key,
|
||||
passthrough_logging_payload=PassthroughStandardLoggingPayload(
|
||||
url="https://test.com", request_body={}
|
||||
),
|
||||
litellm_call_id="test-call-id",
|
||||
logging_obj=LiteLLMLoggingObj(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="test-call-type",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="test-call-id",
|
||||
function_id="test-function-id",
|
||||
),
|
||||
_parsed_body={"litellm_metadata": dict(spoofed), "metadata": dict(spoofed)},
|
||||
)
|
||||
|
||||
metadata = result["litellm_params"]["metadata"]
|
||||
survived = {
|
||||
key: metadata.get(key)
|
||||
for key in expected
|
||||
if metadata.get(key) != expected[key]
|
||||
}
|
||||
assert survived == {}, f"client-supplied values survived for: {sorted(survived)}"
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from unittest.mock import patch, MagicMock, AsyncMock
|
|||
from create_mock_standard_logging_payload import create_standard_logging_payload
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -1816,6 +1817,7 @@ def test_init_auto_router_deployment_success(mock_auto_router, model_list):
|
|||
default_model="gpt-5-mini",
|
||||
embedding_model="text-embedding-3-small",
|
||||
litellm_router_instance=router,
|
||||
max_input_chars=DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS,
|
||||
)
|
||||
|
||||
# Verify the auto-router was added to the router's auto_routers dict
|
||||
|
|
|
|||
|
|
@ -162,3 +162,56 @@ class TestGetLitellmParamsDataResidency:
|
|||
api_base="https://eu.api.openai.com/v1",
|
||||
)
|
||||
assert result["data_residency"] is None
|
||||
|
||||
|
||||
class TestMetadataFallsBackToLitellmMetadata:
|
||||
def test_metadata_falls_back_to_litellm_metadata_when_absent(self):
|
||||
result = get_litellm_params(litellm_metadata={"trace_id": "trace-1"})
|
||||
assert result["metadata"] == {"trace_id": "trace-1"}
|
||||
assert result["litellm_metadata"] == {"trace_id": "trace-1"}
|
||||
|
||||
def test_empty_metadata_falls_back_to_litellm_metadata(self):
|
||||
result = get_litellm_params(metadata={}, litellm_metadata={"trace_id": "trace-1"})
|
||||
assert result["metadata"] == {"trace_id": "trace-1"}
|
||||
|
||||
def test_metadata_wins_when_both_present(self):
|
||||
result = get_litellm_params(
|
||||
metadata={"trace_id": "from-metadata"},
|
||||
litellm_metadata={"trace_id": "from-litellm-metadata"},
|
||||
)
|
||||
assert result["metadata"] == {"trace_id": "from-metadata"}
|
||||
|
||||
@pytest.mark.parametrize("bad_value", ["not-json-a-string", 12345, ["a"], True])
|
||||
def test_non_dict_litellm_metadata_is_ignored(self, bad_value):
|
||||
result = get_litellm_params(litellm_metadata=bad_value)
|
||||
assert result["metadata"] is None
|
||||
|
||||
def test_metadata_stays_none_without_litellm_metadata(self):
|
||||
result = get_litellm_params(api_key="test-key")
|
||||
assert result["metadata"] is None
|
||||
|
||||
def test_session_and_trace_id_derived_from_litellm_metadata(self):
|
||||
result = get_litellm_params(
|
||||
litellm_metadata={"trace_id": "trace-1", "session_id": "session-1"},
|
||||
)
|
||||
assert result["litellm_session_id"] == "session-1"
|
||||
assert result["litellm_trace_id"] == "trace-1"
|
||||
|
||||
def test_explicit_session_and_trace_id_are_not_overridden(self):
|
||||
result = get_litellm_params(
|
||||
litellm_session_id="explicit-session",
|
||||
litellm_trace_id="explicit-trace",
|
||||
litellm_metadata={"trace_id": "trace-1", "session_id": "session-1"},
|
||||
)
|
||||
assert result["litellm_session_id"] == "explicit-session"
|
||||
assert result["litellm_trace_id"] == "explicit-trace"
|
||||
|
||||
def test_litellm_metadata_fallback_is_copied_not_aliased(self):
|
||||
litellm_metadata = {"trace_id": "trace-1"}
|
||||
|
||||
result = get_litellm_params(litellm_metadata=litellm_metadata)
|
||||
|
||||
assert result["metadata"] == litellm_metadata
|
||||
assert result["metadata"] is not litellm_metadata
|
||||
result["metadata"].pop("trace_id")
|
||||
assert litellm_metadata == {"trace_id": "trace-1"}
|
||||
|
|
|
|||
|
|
@ -526,6 +526,26 @@ class TestUpdateFromKwargs:
|
|||
)
|
||||
assert logging_obj.litellm_params["litellm_call_id"] == "call-empty"
|
||||
|
||||
@pytest.mark.parametrize("caller_metadata", [None, "not-a-dict", 42])
|
||||
def test_non_dict_caller_metadata_does_not_break_the_merge(self, logging_obj, caller_metadata):
|
||||
logging_obj.update_from_kwargs(
|
||||
kwargs={"metadata": caller_metadata, "litellm_metadata": {"user_api_key_hash": "hashed"}},
|
||||
litellm_params={"metadata": {"user_api_key_hash": "hashed", "litellm_api_version": "1.0"}},
|
||||
)
|
||||
|
||||
assert logging_obj.litellm_params["metadata"]["user_api_key_hash"] == "hashed"
|
||||
|
||||
def test_does_not_mutate_caller_metadata_dict(self, logging_obj):
|
||||
caller_metadata: dict = {}
|
||||
|
||||
logging_obj.update_from_kwargs(
|
||||
kwargs={"metadata": caller_metadata, "litellm_metadata": {"user_api_key_hash": "hashed"}},
|
||||
litellm_params={"metadata": {"user_api_key_hash": "hashed", "litellm_api_version": "1.0"}},
|
||||
)
|
||||
|
||||
assert caller_metadata == {}
|
||||
assert logging_obj.litellm_params["metadata"]["user_api_key_hash"] == "hashed"
|
||||
|
||||
|
||||
def test_logging_prevent_double_logging(logging_obj):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import List, Optional, Union
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
|
@ -31,6 +32,7 @@ from typing_extensions import TypedDict
|
|||
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy.proxy_server import (
|
||||
ProxyStartupEvent,
|
||||
_initialize_shared_aiohttp_session,
|
||||
_resolve_pydantic_type,
|
||||
_resolve_typed_dict_type,
|
||||
|
|
@ -728,3 +730,50 @@ def test_otel_global_provider_published_after_callback_init():
|
|||
"preset logger will not exist yet and a second generic logger will own "
|
||||
"the global provider, orphaning gen-ai spans"
|
||||
)
|
||||
|
||||
|
||||
def test_startup_warns_for_global_budget_without_database(caplog):
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
|
||||
ProxyStartupEvent._warn_budget_without_db(max_budget=100.0, prisma_client=None)
|
||||
|
||||
assert "litellm.max_budget=100.0" in caplog.text
|
||||
assert "will NOT be enforced" in caplog.text
|
||||
assert "requests will never be blocked" in caplog.text
|
||||
|
||||
|
||||
def test_startup_does_not_warn_for_global_budget_with_database(caplog):
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
|
||||
ProxyStartupEvent._warn_budget_without_db(max_budget=100.0, prisma_client=MagicMock())
|
||||
|
||||
assert "litellm.max_budget" not in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_budget", [0, None])
|
||||
def test_startup_does_not_warn_without_global_budget(caplog, max_budget):
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
|
||||
ProxyStartupEvent._warn_budget_without_db(max_budget=max_budget, prisma_client=None)
|
||||
|
||||
assert "litellm.max_budget" not in caplog.text
|
||||
|
||||
|
||||
def test_proxy_startup_event_warns_for_global_budget_without_database():
|
||||
"""Pin the lifespan call that prevents silent DB-less budgets.
|
||||
|
||||
The call must follow Prisma setup so DB-backed deployments do not false-positive.
|
||||
Direct ``_warn_budget_without_db`` tests cover the warning behavior itself.
|
||||
"""
|
||||
wrapped = getattr(proxy_startup_event, "__wrapped__", proxy_startup_event)
|
||||
source = inspect.getsource(wrapped)
|
||||
budget_check_pos = source.find("if prisma_client is not None and litellm.max_budget > 0:")
|
||||
warn_pos = source.find("_warn_budget_without_db(")
|
||||
next_startup_section_pos = source.find(
|
||||
"await ProxyStartupEvent.initialize_scheduled_background_jobs(",
|
||||
budget_check_pos,
|
||||
)
|
||||
|
||||
assert budget_check_pos != -1, "global budget startup block not found"
|
||||
assert warn_pos != -1, "DB-less budget warning call not found"
|
||||
assert next_startup_section_pos != -1, "startup section after budget block not found"
|
||||
assert budget_check_pos < warn_pos < next_startup_section_pos, (
|
||||
"DB-less budget warning must run after Prisma setup and the DB-backed budget block"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from litellm.proxy.litellm_pre_call_utils import (
|
|||
_get_dynamic_logging_metadata,
|
||||
_get_enforced_params,
|
||||
_get_metadata_variable_name,
|
||||
_promoted_trace_control_fields,
|
||||
_resolve_credential_from_model_config,
|
||||
_resolve_provider_from_deployment,
|
||||
_update_model_if_key_alias_exists,
|
||||
|
|
@ -5869,4 +5870,183 @@ async def test_key_level_callback_vars_survive_the_strip():
|
|||
)
|
||||
|
||||
assert updated[TRUSTED_CALLBACK_VARS_FIELD] == {"dd_api_key": "key-dd-key", "dd_site": "us5.datadoghq.com"}
|
||||
assert updated["dd_site"] == "us5.datadoghq.com"
|
||||
assert updated["dd_site"] == "us5.datadoghq.com"
|
||||
|
||||
|
||||
class TestPromotedTraceControlFields:
|
||||
"""LIT-5137: caller metadata trace fields must reach litellm_metadata."""
|
||||
|
||||
def _make_request(self, path: str) -> MagicMock:
|
||||
request = MagicMock(spec=Request)
|
||||
request.url = MagicMock()
|
||||
request.url.path = path
|
||||
request.url.__str__.return_value = f"http://localhost{path}"
|
||||
request.method = "POST"
|
||||
request.query_params = {}
|
||||
request.headers = {"Content-Type": "application/json"}
|
||||
request.client = MagicMock()
|
||||
request.client.host = "127.0.0.1"
|
||||
return request
|
||||
|
||||
async def _run(self, path: str, data: dict, headers: dict | None = None) -> dict:
|
||||
request = self._make_request(path)
|
||||
if headers is not None:
|
||||
request.headers = {"Content-Type": "application/json", **headers}
|
||||
return await add_litellm_data_to_request(
|
||||
data=data,
|
||||
request=request,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
|
||||
proxy_config=MagicMock(),
|
||||
general_settings={},
|
||||
version="test-version",
|
||||
)
|
||||
|
||||
def test_returns_litellm_metadata_for_responses_route(self):
|
||||
assert _get_metadata_variable_name(self._make_request("/v1/responses")) == "litellm_metadata"
|
||||
|
||||
def test_promotes_trace_prefixed_and_allow_listed_fields(self):
|
||||
requester_metadata = {
|
||||
"trace_id": "trace-1",
|
||||
"trace_name": "name-1",
|
||||
"trace_user_id": "user-1",
|
||||
"trace_metadata": {"tenant_id": "tenant-1"},
|
||||
"trace_version": "v1",
|
||||
"trace_release": "r1",
|
||||
"session_id": "session-1",
|
||||
"mask_input": True,
|
||||
"mask_output": True,
|
||||
}
|
||||
|
||||
promoted = _promoted_trace_control_fields(
|
||||
requester_metadata=requester_metadata,
|
||||
litellm_metadata={},
|
||||
)
|
||||
|
||||
assert dict(promoted) == requester_metadata
|
||||
|
||||
def test_does_not_promote_unlisted_trace_prefixed_fields(self):
|
||||
"""trace_public flips a trace to publicly readable, so the allow-list is explicit."""
|
||||
promoted = _promoted_trace_control_fields(
|
||||
requester_metadata={"trace_id": "trace-1", "trace_public": True, "trace_tags": ["a"]},
|
||||
litellm_metadata={},
|
||||
)
|
||||
|
||||
assert dict(promoted) == {"trace_id": "trace-1"}
|
||||
|
||||
def test_does_not_promote_non_trace_fields(self):
|
||||
promoted = _promoted_trace_control_fields(
|
||||
requester_metadata={
|
||||
"trace_id": "trace-1",
|
||||
"tags": ["free-tier"],
|
||||
"user_api_key": "forged",
|
||||
"user_api_key_user_id": "forged-user",
|
||||
"spend_logs_metadata": {"forged": True},
|
||||
"guardrails": ["disabled"],
|
||||
"debug_langfuse": True,
|
||||
"session": "not-session-id",
|
||||
"existing_trace_id": "victim-trace",
|
||||
"update_trace_keys": ["input", "output"],
|
||||
},
|
||||
litellm_metadata={},
|
||||
)
|
||||
|
||||
assert dict(promoted) == {"trace_id": "trace-1"}
|
||||
|
||||
def test_does_not_promote_trace_mutation_controls(self):
|
||||
"""existing_trace_id + update_trace_keys let a caller overwrite any trace in the project."""
|
||||
promoted = _promoted_trace_control_fields(
|
||||
requester_metadata={
|
||||
"trace_id": "trace-1",
|
||||
"existing_trace_id": "someone-elses-trace",
|
||||
"update_trace_keys": ["input", "output"],
|
||||
},
|
||||
litellm_metadata={},
|
||||
)
|
||||
|
||||
assert dict(promoted) == {"trace_id": "trace-1"}
|
||||
|
||||
def test_existing_litellm_metadata_value_wins(self):
|
||||
promoted = _promoted_trace_control_fields(
|
||||
requester_metadata={"trace_id": "from-body", "session_id": "from-body", "trace_name": "from-body"},
|
||||
litellm_metadata={"trace_id": "from-header", "session_id": "from-header"},
|
||||
)
|
||||
|
||||
assert dict(promoted) == {"trace_name": "from-body"}
|
||||
|
||||
def test_empty_requester_metadata_promotes_nothing(self):
|
||||
assert _promoted_trace_control_fields(requester_metadata={}, litellm_metadata={}) == ()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_route_end_to_end(self):
|
||||
caller_metadata = {
|
||||
"trace_id": "22662678-30c1-41a1-a24b-216d6e5fb83d",
|
||||
"session_id": "218af06c-28a2-4705-8a0a-5f9970d39326",
|
||||
"trace_user_id": "user-123",
|
||||
"trace_metadata": {"tenant_id": "tenant-1"},
|
||||
"mask_input": True,
|
||||
}
|
||||
|
||||
updated = await self._run(
|
||||
"/v1/responses",
|
||||
{"model": "gpt-4.1-mini", "input": "say resp", "metadata": copy.deepcopy(caller_metadata)},
|
||||
)
|
||||
|
||||
litellm_metadata = updated["litellm_metadata"]
|
||||
for key, value in caller_metadata.items():
|
||||
assert litellm_metadata[key] == value
|
||||
assert updated["metadata"] == caller_metadata
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_messages_route_end_to_end(self):
|
||||
updated = await self._run(
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": "claude-sonnet-4-5",
|
||||
"max_tokens": 32,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"metadata": {"trace_id": "msg-trace-1", "session_id": "msg-session-1"},
|
||||
},
|
||||
)
|
||||
|
||||
assert updated["litellm_metadata"]["trace_id"] == "msg-trace-1"
|
||||
assert updated["litellm_metadata"]["session_id"] == "msg-session-1"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_id_header_beats_body_metadata(self):
|
||||
updated = await self._run(
|
||||
"/v1/responses",
|
||||
{"model": "gpt-4.1-mini", "input": "say resp", "metadata": {"session_id": "from-body"}},
|
||||
headers={"x-litellm-session-id": "from-header-12345678"},
|
||||
)
|
||||
|
||||
assert updated["litellm_metadata"]["session_id"] == "from-header-12345678"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_forged_user_api_key_fields_are_not_promoted(self):
|
||||
updated = await self._run(
|
||||
"/v1/responses",
|
||||
{
|
||||
"model": "gpt-4.1-mini",
|
||||
"input": "say resp",
|
||||
"metadata": {"trace_id": "trace-1", "user_api_key_user_id": "forged", "spend_logs_metadata": {"a": 1}},
|
||||
},
|
||||
)
|
||||
|
||||
litellm_metadata = updated["litellm_metadata"]
|
||||
assert litellm_metadata["trace_id"] == "trace-1"
|
||||
assert litellm_metadata.get("user_api_key_user_id") != "forged"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completions_route_is_untouched(self):
|
||||
updated = await self._run(
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": "gpt-4.1-mini",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"metadata": {"trace_id": "trace-1", "session_id": "session-1"},
|
||||
},
|
||||
)
|
||||
|
||||
assert "litellm_metadata" not in updated
|
||||
assert updated["metadata"]["trace_id"] == "trace-1"
|
||||
assert updated["metadata"]["session_id"] == "session-1"
|
||||
|
|
|
|||
|
|
@ -636,8 +636,11 @@ describe("AddAutoRouterTab", () => {
|
|||
expect(labels).toEqual(["Anthropic Family", "OpenAI Family", "Custom Configuration"]);
|
||||
});
|
||||
|
||||
it("never lets a wildcard deployment satisfy a preset", async () => {
|
||||
const wildcard = [{ model_name: "openai-wild", litellm_params: { model: "openai/*" } }];
|
||||
it.each([
|
||||
["a wildcard group", "openai/*"],
|
||||
["a plain group over a wildcard underlying model", "openai-wild"],
|
||||
])("never lets %s satisfy a preset when the hub lists no expansions", async (_label, modelName) => {
|
||||
const wildcard = [{ model_name: modelName, litellm_params: { model: "openai/*" } }];
|
||||
mockFetchAvailableModels.mockResolvedValue(groupsFor(wildcard));
|
||||
mockFetchAllModelDeployments.mockResolvedValue(wildcard);
|
||||
|
||||
|
|
@ -650,4 +653,59 @@ describe("AddAutoRouterTab", () => {
|
|||
expect(isOptionDisabled(optionByLabel("OpenAI Family")!)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("wildcard-matched presets", () => {
|
||||
const WILDCARD_DEPLOYMENTS = [{ model_name: "someprovider/*", litellm_params: { model: "someprovider/*" } }];
|
||||
|
||||
const expandedGroupFor = (model: string): string => `someprovider/${model}`;
|
||||
|
||||
const EXPANDED_HUB_GROUPS: ModelGroup[] = [
|
||||
{ model_group: "someprovider/*", mode: "chat" },
|
||||
...[...new Set(getAllPresets().flatMap((preset) => [...getRequiredModelsInPreset(preset)]))].map((model) => ({
|
||||
model_group: expandedGroupFor(model),
|
||||
mode: "chat",
|
||||
})),
|
||||
];
|
||||
|
||||
it("enables a preset whose models exist only as wildcard-expanded groups, labeling the match", async () => {
|
||||
mockFetchAvailableModels.mockResolvedValue(EXPANDED_HUB_GROUPS);
|
||||
mockFetchAllModelDeployments.mockResolvedValue(WILDCARD_DEPLOYMENTS);
|
||||
|
||||
renderWithProviders(<Harness />);
|
||||
openTemplateDropdown();
|
||||
|
||||
await waitFor(() => {
|
||||
expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false);
|
||||
});
|
||||
expect(optionByLabel("Anthropic Family")!.textContent).toContain("Matches your deployments");
|
||||
});
|
||||
|
||||
it("prefills the expanded group names and submits them", async () => {
|
||||
const user = userEvent.setup();
|
||||
mockFetchAvailableModels.mockResolvedValue(EXPANDED_HUB_GROUPS);
|
||||
mockFetchAllModelDeployments.mockResolvedValue(WILDCARD_DEPLOYMENTS);
|
||||
|
||||
renderWithProviders(<Harness />);
|
||||
openTemplateDropdown();
|
||||
await waitFor(() => {
|
||||
expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false);
|
||||
});
|
||||
fireEvent.click(optionByLabel("Anthropic Family")!);
|
||||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "wildcard-router");
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0]).toMatchObject({
|
||||
complexity_router_config: {
|
||||
tiers: {
|
||||
SIMPLE: ANTHROPIC_TIERS.SIMPLE.map(expandedGroupFor),
|
||||
MEDIUM: ANTHROPIC_TIERS.MEDIUM.map(expandedGroupFor),
|
||||
COMPLEX: ANTHROPIC_TIERS.COMPLEX.map(expandedGroupFor),
|
||||
REASONING: ANTHROPIC_TIERS.REASONING.map(expandedGroupFor),
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -191,6 +191,148 @@ describe("autorouter_presets", () => {
|
|||
);
|
||||
});
|
||||
|
||||
describe("wildcard deployment matching (expanded model groups)", () => {
|
||||
const wildcardDeployment = (pattern: string) => ({ modelGroup: pattern, underlyingModels: [pattern] });
|
||||
|
||||
const simpleTierConfig = (presetModel: string) => ({
|
||||
tiers: { SIMPLE: [presetModel], MEDIUM: [], COMPLEX: [], REASONING: [] },
|
||||
classifier_type: "heuristic" as const,
|
||||
session_affinity: false,
|
||||
});
|
||||
|
||||
it("resolves a preset model to a group expanded from a wildcard deployment", () => {
|
||||
const availability = buildModelAvailability(
|
||||
["anthropic/*", "anthropic/claude-opus-5", "bedrock/anthropic.claude-opus-5"],
|
||||
[wildcardDeployment("anthropic/*")],
|
||||
);
|
||||
const config = simpleTierConfig("claude-opus-5");
|
||||
expect(getMissingModels(config, availability)).toEqual([]);
|
||||
expect(buildPresetPrefill(config, availability).complexityRouterConfig.tiers.SIMPLE).toEqual([
|
||||
"anthropic/claude-opus-5",
|
||||
]);
|
||||
});
|
||||
|
||||
it("normalizes an expanded group's namespaced own name the same way as a deployment's", () => {
|
||||
const availability = buildModelAvailability(
|
||||
["bedrock/*", "bedrock/us.anthropic.claude-sonnet-5"],
|
||||
[wildcardDeployment("bedrock/*")],
|
||||
);
|
||||
expect(getMissingModels(simpleTierConfig("claude-sonnet-5"), availability)).toEqual([]);
|
||||
});
|
||||
|
||||
it("anchors a partial wildcard pattern and treats its dots literally", () => {
|
||||
const availability = buildModelAvailability(
|
||||
["bedrock/us.anthropic.claude-opus-5", "bedrock/usXanthropic.claude-fable-5"],
|
||||
[wildcardDeployment("bedrock/us.*")],
|
||||
);
|
||||
expect(getMissingModels(simpleTierConfig("claude-opus-5"), availability)).toEqual([]);
|
||||
expect(getMissingModels(simpleTierConfig("claude-fable-5"), availability)).toEqual(["claude-fable-5"]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["gpt-5.4", "openai/gpt-5.4-mini"],
|
||||
["gpt-5.4-mini", "openai/gpt-5.4"],
|
||||
["o3", "openai/o3-mini"],
|
||||
])("never lets %s be satisfied by the expanded group %s", (presetModel, expandedGroup) => {
|
||||
const availability = buildModelAvailability(["openai/*", expandedGroup], [wildcardDeployment("openai/*")]);
|
||||
expect(getMissingModels(simpleTierConfig(presetModel), availability)).toEqual([presetModel]);
|
||||
});
|
||||
|
||||
it("anchors the pattern's suffix and keeps middle segments in order", () => {
|
||||
const availability = buildModelAvailability(
|
||||
["bedrock/us.anthropic.claude-opus-5", "bedrock/anthropic.us.claude-sonnet-5"],
|
||||
[wildcardDeployment("bedrock/*.anthropic.*")],
|
||||
);
|
||||
expect(getMissingModels(simpleTierConfig("claude-opus-5"), availability)).toEqual([]);
|
||||
expect(getMissingModels(simpleTierConfig("claude-sonnet-5"), availability)).toEqual(["claude-sonnet-5"]);
|
||||
});
|
||||
|
||||
it("matches a pathological many-star pattern in linear time instead of backtracking", () => {
|
||||
const hostile = `prov/a*${"a*".repeat(30)}b`;
|
||||
const nonMatching = `prov/${"a".repeat(120)}`;
|
||||
const availability = buildModelAvailability([nonMatching], [wildcardDeployment(hostile)]);
|
||||
expect(availability.underlyingIndex.size).toBe(0);
|
||||
});
|
||||
|
||||
it("expands a bare-star model_name through its underlying wildcard, not as match-all", () => {
|
||||
const availability = buildModelAvailability(
|
||||
["openai/gpt-5.4", "team-a/claude-opus-5"],
|
||||
[{ modelGroup: "*", underlyingModels: ["openai/*"] }],
|
||||
);
|
||||
expect(getMissingModels(simpleTierConfig("gpt-5.4"), availability)).toEqual([]);
|
||||
expect(getMissingModels(simpleTierConfig("claude-opus-5"), availability)).toEqual(["claude-opus-5"]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["a bare-star underlying", "*"],
|
||||
["a non-wildcard underlying", "openai/gpt-4o"],
|
||||
["a slashless wildcard underlying", "gpt*"],
|
||||
])("derives no pattern from a bare-star model_name with %s", (_label, underlying) => {
|
||||
const availability = buildModelAvailability(
|
||||
["openai/gpt-5.4"],
|
||||
[{ modelGroup: "*", underlyingModels: [underlying] }],
|
||||
);
|
||||
expect(availability.underlyingIndex.size).toBe(0);
|
||||
});
|
||||
|
||||
it("derives no pattern from a slashless wildcard model_name", () => {
|
||||
const availability = buildModelAvailability(["gpt-5.4"], [wildcardDeployment("gpt*")]);
|
||||
expect(availability.underlyingIndex.size).toBe(0);
|
||||
});
|
||||
|
||||
it("does not trust a group's name when no wildcard deployment covers it", () => {
|
||||
const availability = buildModelAvailability(
|
||||
["team-a/claude-opus-5", "openai/*"],
|
||||
[wildcardDeployment("openai/*")],
|
||||
);
|
||||
expect(getMissingModels(simpleTierConfig("claude-opus-5"), availability)).toEqual(["claude-opus-5"]);
|
||||
});
|
||||
|
||||
it("never resolves to the wildcard group itself when the hub lists no expansions", () => {
|
||||
const availability = buildModelAvailability(["openai/*"], [wildcardDeployment("openai/*")]);
|
||||
expect(getMissingModels(simpleTierConfig("gpt-5.4"), availability)).toEqual(["gpt-5.4"]);
|
||||
expect(availability.underlyingIndex.size).toBe(0);
|
||||
});
|
||||
|
||||
it("applies a wildcard deployment's pattern even when the wildcard group is not itself listed", () => {
|
||||
const availability = buildModelAvailability(["anthropic/claude-opus-5"], [wildcardDeployment("anthropic/*")]);
|
||||
expect(getMissingModels(simpleTierConfig("claude-opus-5"), availability)).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps the groups-only availability strict even when expanded groups are listed", () => {
|
||||
const availability = groupsOnly(["anthropic/*", "anthropic/claude-opus-5"]);
|
||||
expect(getMissingModels(simpleTierConfig("claude-opus-5"), availability)).toEqual(["claude-opus-5"]);
|
||||
});
|
||||
|
||||
it("prefers the alphabetically first covered group when several expansions serve the model", () => {
|
||||
const availability = buildModelAvailability(
|
||||
["bedrock/us.anthropic.claude-opus-5", "anthropic/claude-opus-5", "bedrock/anthropic.claude-opus-5"],
|
||||
[wildcardDeployment("anthropic/*"), wildcardDeployment("bedrock/*")],
|
||||
);
|
||||
const config = simpleTierConfig("claude-opus-5");
|
||||
expect(buildPresetPrefill(config, availability).complexityRouterConfig.tiers.SIMPLE).toEqual([
|
||||
"anthropic/claude-opus-5",
|
||||
]);
|
||||
});
|
||||
|
||||
it.each(getAllPresets().map((preset) => [preset.key, preset] as const))(
|
||||
"fully resolves the %s preset through wildcard-expanded groups only",
|
||||
(_key, preset) => {
|
||||
const required = [...getRequiredModelsInPreset(preset)];
|
||||
const expandedGroups = required.map((model) => `someprovider/${model}`);
|
||||
const availability = buildModelAvailability(
|
||||
["someprovider/*", ...expandedGroups],
|
||||
[wildcardDeployment("someprovider/*")],
|
||||
);
|
||||
expect(getMissingModelsInPreset(preset, availability)).toEqual([]);
|
||||
const prefilled = buildPresetPrefill(preset.complexity_router_config, availability);
|
||||
const prefilledModels = Object.values(prefilled.complexityRouterConfig.tiers).flat();
|
||||
expect(prefilledModels.length).toBeGreaterThan(0);
|
||||
for (const model of prefilledModels) expect(expandedGroups).toContain(model);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("deploymentRefsFromModelInfo", () => {
|
||||
it("keeps litellm_params.model and model_info.base_model, drops rows with neither or no name", () => {
|
||||
const refs = deploymentRefsFromModelInfo([
|
||||
|
|
|
|||
|
|
@ -77,12 +77,30 @@ const normalizeUnderlyingModel = (model: string): string | null => {
|
|||
return stripped.toLowerCase() || null;
|
||||
};
|
||||
|
||||
// A linear glob scan rather than a RegExp: patterns are admin-controlled model_name values, and a
|
||||
// backtracking regex built from one ("a*a*a*...") can freeze another admin's dashboard.
|
||||
const matchesWildcard = (pattern: string, name: string): boolean => {
|
||||
const parts = pattern.split("*");
|
||||
if (parts.length === 1) return pattern === name;
|
||||
const head = parts[0];
|
||||
const tail = parts[parts.length - 1];
|
||||
if (!name.startsWith(head) || !name.endsWith(tail)) return false;
|
||||
if (name.length < head.length + tail.length) return false;
|
||||
const scanEnd = name.length - tail.length;
|
||||
const scanResult = parts.slice(1, -1).reduce((searchFrom: number, part: string) => {
|
||||
if (searchFrom < 0) return -1;
|
||||
const found = name.indexOf(part, searchFrom);
|
||||
return found === -1 || found + part.length > scanEnd ? -1 : found + part.length;
|
||||
}, head.length);
|
||||
return scanResult >= 0;
|
||||
};
|
||||
|
||||
export const buildModelAvailability = (
|
||||
modelGroups: Iterable<string>,
|
||||
deployments: readonly DeploymentModelRef[],
|
||||
): ModelAvailability => {
|
||||
const groups = new Set(modelGroups);
|
||||
const entries = deployments
|
||||
const literalEntries = deployments
|
||||
.filter((deployment) => groups.has(deployment.modelGroup))
|
||||
.flatMap((deployment) =>
|
||||
deployment.underlyingModels
|
||||
|
|
@ -90,6 +108,22 @@ export const buildModelAvailability = (
|
|||
.filter((key): key is string => key !== null)
|
||||
.map((key) => ({ key, modelGroup: deployment.modelGroup })),
|
||||
);
|
||||
// Mirrors get_known_models_from_wildcard: a bare "*" model_name expands via its underlying
|
||||
// wildcard (or not at all), and a wildcard without a "/" expands to nothing.
|
||||
const wildcardPatterns = Array.from(
|
||||
new Set(
|
||||
deployments
|
||||
.flatMap((deployment) =>
|
||||
deployment.modelGroup === "*" ? deployment.underlyingModels : [deployment.modelGroup],
|
||||
)
|
||||
.filter((pattern) => pattern !== "*" && pattern.includes("*") && pattern.includes("/")),
|
||||
),
|
||||
);
|
||||
const wildcardEntries = Array.from(groups)
|
||||
.filter((group) => !group.includes("*") && wildcardPatterns.some((pattern) => matchesWildcard(pattern, group)))
|
||||
.map((group) => ({ key: normalizeUnderlyingModel(group), modelGroup: group }))
|
||||
.filter((entry): entry is { key: string; modelGroup: string } => entry.key !== null);
|
||||
const entries = [...literalEntries, ...wildcardEntries];
|
||||
const grouped = new Map<string, Set<string>>();
|
||||
for (const entry of entries) {
|
||||
const groupsForKey = grouped.get(entry.key) ?? new Set<string>();
|
||||
|
|
|
|||
6
uv.lock
generated
6
uv.lock
generated
|
|
@ -10,7 +10,7 @@ resolution-markers = [
|
|||
]
|
||||
|
||||
[options]
|
||||
exclude-newer = "2026-08-02T02:14:05.876141Z"
|
||||
exclude-newer = "2026-08-04T00:00:57.623181Z"
|
||||
exclude-newer-span = "P3D"
|
||||
|
||||
[manifest]
|
||||
|
|
@ -4583,12 +4583,12 @@ proxy-dev = [
|
|||
|
||||
[[package]]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.53"
|
||||
version = "0.1.54"
|
||||
source = { editable = "enterprise" }
|
||||
|
||||
[[package]]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.83"
|
||||
version = "0.4.84"
|
||||
source = { editable = "litellm-proxy-extras" }
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue