diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 2527239b904..a0943cff53d 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -17,3 +17,24 @@ # style: unify ruff format width on 120 (#31518) 48b5a5a0cc5a694a11219416ee0b6eb6e620e74e + +# refactor(imports): move collections.abc names out of typing (#35495) +397e8e4918777e4e60a7f5e88699e0a9a7dabb3d + +# refactor(lint): apply every safe ruff autofix and zero 28 strict-rule budgets (#35495) +b604e2b20c6db2099085a2f0e59b7e99e87eed6f + +# refactor(logging): drop redundant !s conversion flags from f-strings (#35546) +7b2d3440cba3160277470f7a0180098ae9b87864 + +# perf: build log messages lazily so filtered-out log records cost nothing (#35703) +c9887a1f94bc1e7e4bdfe64d640f0509a0bc19dd + +# feat(lint): enforce Final on locals and freeze function parameters (#35807) +2708620d6a599cc73c1950a942d26ac26a7ed3d4 + +# chore(lint): remove litellm/types from the ruff lint exclusion (#35926) +4e32a8bf6a1e1af1e04b67c759841ccef44b2235 + +# chore(lint): strip inert type: ignore comments and zero LIT009/LIT010/LIT011 headroom (#35928) +338e411103ad5d7003e97f34f04fa36bca542dbe diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index 5489eba1494..a069bd81eca 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.53" +version = "0.1.54" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.53" +version = "0.1.54" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index beddd899472..fc58ff68b4d 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.83" +version = "0.4.84" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.83" +version = "0.4.84" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index d6433ad3332..f251ab4d74a 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -115,8 +115,11 @@ def get_litellm_params( litellm_request_debug: bool | None = None, **kwargs, ) -> dict: + _litellm_metadata_dict: Final = litellm_metadata if isinstance(litellm_metadata, dict) else None + resolved_metadata: Final = _litellm_metadata_dict.copy() if not metadata and _litellm_metadata_dict else metadata + # Derive litellm_session_id / litellm_trace_id from metadata when not provided (call chaining) - _meta: Final = metadata or {} + _meta: Final = resolved_metadata or {} if litellm_session_id is None: litellm_session_id = _meta.get("session_id") or _meta.get("trace_id") if litellm_trace_id is None: @@ -139,7 +142,7 @@ def get_litellm_params( "model_alias_map": model_alias_map, "completion_call_id": completion_call_id, "aembedding": aembedding, - "metadata": metadata, + "metadata": resolved_metadata, "model_info": model_info, "proxy_server_request": proxy_server_request, "preset_cache_key": preset_cache_key, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9475441e214..99721c3ffa2 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -585,8 +585,8 @@ class Logging(LiteLLMLoggingBaseClass): """ base_litellm_params: Final[dict[str, Any]] = {} - if "metadata" in kwargs: - base_litellm_params["metadata"] = kwargs["metadata"] + if isinstance(kwargs.get("metadata"), dict): + base_litellm_params["metadata"] = kwargs["metadata"].copy() if "litellm_metadata" in kwargs and isinstance(kwargs["litellm_metadata"], dict): base_litellm_params["litellm_metadata"] = kwargs["litellm_metadata"] if "metadata" not in base_litellm_params: diff --git a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/__init__.py index 408260d8483..270c28b094e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/__init__.py @@ -18,6 +18,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" send_user_api_key_alias=litellm_params.send_user_api_key_alias, send_user_api_key_user_id=litellm_params.send_user_api_key_user_id, send_user_api_key_team_id=litellm_params.send_user_api_key_team_id, + timeout=litellm_params.timeout, guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, default_on=litellm_params.default_on, diff --git a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py index c5c66988cb4..1aefa38ecf8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py @@ -22,9 +22,10 @@ from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.guardrails import LitellmParams from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel -GUARDRAIL_TIMEOUT: Final = 5 +DEFAULT_GUARDRAIL_TIMEOUT: Final = 5.0 class ZscalerAIGuard(CustomGuardrail): @@ -43,6 +44,7 @@ class ZscalerAIGuard(CustomGuardrail): send_user_api_key_alias: bool | None = None, send_user_api_key_user_id: bool | None = None, send_user_api_key_team_id: bool | None = None, + timeout: float | None = None, **kwargs, ): kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) @@ -68,6 +70,7 @@ class ZscalerAIGuard(CustomGuardrail): if send_user_api_key_team_id is not None else os.getenv("SEND_USER_API_KEY_TEAM_ID", "False").lower() in ("true", "1") ) + self.timeout = self._resolve_timeout(timeout) verbose_proxy_logger.debug( "send_user_api_key_alias: %s, \n send_user_api_key_user_id:%s, \n send_user_api_key_team_id:%s", @@ -80,6 +83,29 @@ class ZscalerAIGuard(CustomGuardrail): verbose_proxy_logger.debug("ZscalerAIGuard Initializing ...") + @staticmethod + def _resolve_timeout(timeout: float | None) -> float: + """ + Resolve the effective per-request timeout, falling back to the default + when it is unset or non-positive. + """ + if timeout is None: + return DEFAULT_GUARDRAIL_TIMEOUT + + if timeout <= 0: + verbose_proxy_logger.warning( + "Ignoring non-positive Zscaler AI Guard timeout %s, using %s seconds", + timeout, + DEFAULT_GUARDRAIL_TIMEOUT, + ) + return DEFAULT_GUARDRAIL_TIMEOUT + + return timeout + + def update_in_memory_litellm_params(self, litellm_params: "LitellmParams") -> None: + super().update_in_memory_litellm_params(litellm_params) + self.timeout = self._resolve_timeout(litellm_params.timeout) + @staticmethod def _resolve_metadata_value(request_data: dict | None, key: str) -> str | None: """ @@ -267,7 +293,7 @@ class ZscalerAIGuard(CustomGuardrail): f"{url}", headers=headers, json=data, - timeout=GUARDRAIL_TIMEOUT, + timeout=self.timeout, ) response.raise_for_status() return response diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index b00ba35b14e..83ae59ef050 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -151,6 +151,20 @@ LITELLM_METADATA_ROUTES: Final = ( "files", ) +LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset( + { + "mask_input", + "mask_output", + "session_id", + "trace_id", + "trace_metadata", + "trace_name", + "trace_release", + "trace_user_id", + "trace_version", + } +) + _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = ( "proxy_server_request", "standard_logging_object", @@ -458,6 +472,18 @@ def _get_metadata_variable_name(request: Request) -> str: return "metadata" +def _promoted_trace_control_fields( + requester_metadata: Mapping[str, Any], + litellm_metadata: Mapping[str, Any], +) -> tuple[tuple[str, Any], ...]: + """Return the caller's trace-control fields that ``litellm_metadata`` does not already set.""" + return tuple( + (key, value) + for key, value in requester_metadata.items() + if key in LITELLM_TRACE_CONTROL_METADATA_FIELDS and key not in litellm_metadata + ) + + def _extract_generic_session_id_from_headers( normalized: dict[str, str], ) -> str | None: @@ -1670,6 +1696,13 @@ async def add_litellm_data_to_request( # paths may read from it. if "metadata" in data and isinstance(data["metadata"], dict): data[_metadata_variable_name]["requester_metadata"] = copy.deepcopy(data["metadata"]) + if _metadata_variable_name == "litellm_metadata": + data[_metadata_variable_name].update( + _promoted_trace_control_fields( + requester_metadata=data[_metadata_variable_name]["requester_metadata"], + litellm_metadata=data[_metadata_variable_name], + ) + ) # Merge litellm_metadata into the metadata variable (preserving existing # values). Runs after the user_api_key_* / _pipeline_managed_guardrails diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 64e52d252ca..8a526fcd6cb 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -565,6 +565,9 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): # real parent span. _metadata["user_api_key"] = user_api_key_dict.api_key _metadata["litellm_parent_otel_span"] = user_api_key_dict.parent_otel_span + _metadata.update( + LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=user_api_key_dict) + ) kwargs: Final = { "litellm_params": { diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 07eaed9fe45..2e24a2d4f3c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1111,6 +1111,10 @@ async def proxy_startup_event(app: FastAPI): prisma_client=prisma_client, ) ) + ProxyStartupEvent._warn_budget_without_db( + max_budget=litellm.max_budget, + prisma_client=prisma_client, + ) ### START BATCH WRITING DB + CHECKING NEW MODELS### if prisma_client is not None: @@ -7825,6 +7829,19 @@ def giveup(e): class ProxyStartupEvent: + @staticmethod + def _warn_budget_without_db(max_budget: float | None, prisma_client: PrismaClient | None) -> None: + if prisma_client is not None or not max_budget or max_budget <= 0: + return + + verbose_proxy_logger.warning( + "A proxy-wide budget (litellm.max_budget=%s) is configured but no database is connected, " + "so the budget will NOT be enforced and requests will never be blocked. Set DATABASE_URL or " + "general_settings.database_url and restart. Redis and fail_closed_budget_enforcement do not " + "cover the proxy-wide budget because there is no global spend counter; Redis alone is not a substitute.", + max_budget, + ) + @classmethod def _initialize_startup_logging( cls, diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/zscaler_ai_guard.py b/litellm/types/proxy/guardrails/guardrail_hooks/zscaler_ai_guard.py index 3991cee8548..37125c4d583 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/zscaler_ai_guard.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/zscaler_ai_guard.py @@ -79,6 +79,15 @@ class ZscalerAIGuardConfigModel(GuardrailConfigModel): json_schema_extra={"ui_type": GuardrailParamUITypes.BOOL}, ) + timeout: float | None = Field( + default=None, + description=( + "Timeout for each Zscaler AI Guard API call, in seconds. Must be positive. " + "Raise it if scans fail under load with 'Connection timed out'. " + "Defaults to 5 seconds." + ), + ) + @model_validator(mode="after") def validate_endpoint_configuration(self) -> "ZscalerAIGuardConfigModel": """ diff --git a/pyproject.toml b/pyproject.toml index 414b09eb3b4..35fd949c2e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,8 +66,8 @@ proxy = [ "azure-identity>=1.25.2,<2.0", "azure-storage-blob>=12.28.0,<13.0", "mcp>=1.28.1,<2.0", - "litellm-proxy-extras==0.4.83", - "litellm-enterprise==0.1.53", + "litellm-proxy-extras==0.4.84", + "litellm-enterprise==0.1.54", "RestrictedPython>=8.1,<9.0", "rich>=13.9.4,<14.0", "InquirerPy>=0.3.4,<1.0", diff --git a/terraform/provider/CHANGELOG.md b/terraform/provider/CHANGELOG.md index 101519c0b08..7c744f04064 100644 --- a/terraform/provider/CHANGELOG.md +++ b/terraform/provider/CHANGELOG.md @@ -7,13 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.4.0] - 2026-08-06 + ### Fixed - **organization**: Send `PATCH` instead of `POST` to `/organization/update` and `/organization/member_update`, matching the methods the LiteLLM proxy serves; organization and organization member updates previously failed with a 405 +- **team_member**: Include `role` in the update payload so a role change on an existing `litellm_team_member` is applied instead of being silently dropped ### Changed - The provider source of truth moved to `terraform/provider/` in [BerriAI/litellm](https://github.com/BerriAI/litellm); this repository is now a release mirror. CI in the monorepo statically audits every endpoint the provider calls against the proxy's OpenAPI schema on every change +- **mcp_server**, **vector_store**: `env` and `litellm_params` are now marked sensitive, so they are redacted from plan/apply output, and they are no longer read back from the API into state — the configured value is authoritative. If the proxy returns values that differ from the configuration, that drift is no longer surfaced on refresh +- Dependency updates: `grpc` and `golang.org/x` modules + +## [0.3.0] - 2026-07-13 + +Released from the mirror repository before the source move was complete; this entry backfills it in the monorepo changelog. + +### Added + +- **model**: Add optional `pricing_base_model` attribute that sets `model_info.base_model` (the cost-map lookup key) independently of routing. Deployments whose routing name differs from the pricing key (for example Azure Data Zone, routed as `azure/gpt-4.1` but priced via `us/gpt-4.1-2025-04-14`) can now be billed correctly without breaking routing. When unset, behavior is unchanged and `base_model` continues to drive both routing and pricing (#47) ## [0.2.2] - 2026-05-13 diff --git a/terraform/provider/docs/resources/model.md b/terraform/provider/docs/resources/model.md index 5a46fe2f073..0409b48b391 100644 --- a/terraform/provider/docs/resources/model.md +++ b/terraform/provider/docs/resources/model.md @@ -118,6 +118,8 @@ The following arguments are supported: * `base_model` - (Required) string. The actual model identifier from the provider (e.g., "gpt-4", "claude-2"). +* `pricing_base_model` - (Optional) string. A pricing key fed to `model_info.base_model` **independently of routing**. When set, `litellm_params.model` still routes via `base_model`, but LiteLLM looks up cost against this key. Useful when the routing/deployment name differs from the cost-map key — e.g. an Azure deployment routed as `azure/gpt-4.1` whose real tier is Data Zone: set `pricing_base_model = "us/gpt-4.1-2025-04-14"` so it is billed at the Data Zone rate. When unset, `base_model` drives pricing as before. + * `litellm_credential_name` - (Optional) string. Name of a LiteLLM credential to use for this model. * `tier` - (Optional) string. The usage tier for this model. Valid values are `"free"` or `"paid"`. Default: `"free"`. diff --git a/terraform/provider/litellm/resource_model.go b/terraform/provider/litellm/resource_model.go index 2858b6e763d..4bad057871d 100644 --- a/terraform/provider/litellm/resource_model.go +++ b/terraform/provider/litellm/resource_model.go @@ -73,6 +73,14 @@ func resourceLiteLLMModel() *schema.Resource { Type: schema.TypeString, Required: true, }, + "pricing_base_model": { + // Optional pricing key fed to model_info.base_model, DECOUPLED + // from routing. When set, litellm_params.model still routes via + // base_model, but cost is looked up against this key (e.g. + // "us/gpt-4.1-2025-04-14" for Azure Data Zone pricing). + Type: schema.TypeString, + Optional: true, + }, "tier": { Type: schema.TypeString, Optional: true, diff --git a/terraform/provider/litellm/resource_model_crud.go b/terraform/provider/litellm/resource_model_crud.go index 40766c8e312..fc5d5b09dd5 100644 --- a/terraform/provider/litellm/resource_model_crud.go +++ b/terraform/provider/litellm/resource_model_crud.go @@ -68,6 +68,14 @@ func createOrUpdateModel(d *schema.ResourceData, m interface{}, isUpdate bool) e baseModel := d.Get("base_model").(string) modelName := fmt.Sprintf("%s/%s", customLLMProvider, baseModel) + // Pricing base_model, decoupled from routing. When pricing_base_model is + // set it feeds model_info.base_model (the cost-lookup key) WITHOUT changing + // the routing string above; otherwise base_model drives pricing as before. + pricingBaseModel := baseModel + if v, ok := d.GetOk("pricing_base_model"); ok && v.(string) != "" { + pricingBaseModel = v.(string) + } + // Generate a UUID for new models modelID := d.Id() if !isUpdate { @@ -240,7 +248,7 @@ func createOrUpdateModel(d *schema.ResourceData, m interface{}, isUpdate bool) e ModelInfo: ModelInfo{ ID: modelID, DBModel: true, - BaseModel: baseModel, + BaseModel: pricingBaseModel, Tier: d.Get("tier").(string), Mode: d.Get("mode").(string), TeamID: d.Get("team_id").(string), @@ -306,7 +314,16 @@ func resourceLiteLLMModelRead(d *schema.ResourceData, m interface{}) error { d.Set("rpm", GetIntValue(modelResp.LiteLLMParams.RPM, d.Get("rpm").(int))) d.Set("model_api_base", GetStringValue(modelResp.LiteLLMParams.APIBase, d.Get("model_api_base").(string))) d.Set("api_version", GetStringValue(modelResp.LiteLLMParams.APIVersion, d.Get("api_version").(string))) - d.Set("base_model", GetStringValue(modelResp.ModelInfo.BaseModel, d.Get("base_model").(string))) + // base_model / pricing_base_model read-back. When pricing_base_model is + // configured, model_info.base_model holds the PRICING key, so recover the + // routing base_model from state (not returned by the API) and read + // pricing_base_model from model_info. + if pbm, ok := d.GetOk("pricing_base_model"); ok && pbm.(string) != "" { + d.Set("base_model", d.Get("base_model").(string)) + d.Set("pricing_base_model", GetStringValue(modelResp.ModelInfo.BaseModel, pbm.(string))) + } else { + d.Set("base_model", GetStringValue(modelResp.ModelInfo.BaseModel, d.Get("base_model").(string))) + } d.Set("tier", GetStringValue(modelResp.ModelInfo.Tier, d.Get("tier").(string))) d.Set("mode", GetStringValue(modelResp.ModelInfo.Mode, d.Get("mode").(string))) d.Set("team_id", GetStringValue(modelResp.ModelInfo.TeamID, d.Get("team_id").(string))) diff --git a/tests/guardrails_tests/test_zscaler_ai_guard.py b/tests/guardrails_tests/test_zscaler_ai_guard.py index 51c86c15dcb..76e498673b0 100644 --- a/tests/guardrails_tests/test_zscaler_ai_guard.py +++ b/tests/guardrails_tests/test_zscaler_ai_guard.py @@ -396,3 +396,122 @@ async def test_apply_guardrail_block_does_not_log_error(mock_api_call): mock_logger.error.assert_not_called() assert exc_info.value.status_code == 400 + + +@pytest.mark.asyncio +async def test_send_request_uses_default_timeout_when_unconfigured(): + """ + Regression: unconfigured guardrails must keep the historical 5s timeout. + """ + guardrail = ZscalerAIGuard(api_key="test_key", policy_id=1) + + assert guardrail.timeout == 5.0 + + with patch( + "litellm.proxy.guardrails.guardrail_hooks.zscaler_ai_guard.zscaler_ai_guard.get_async_httpx_client" + ) as mock_get_client: + mock_client = Mock() + mock_client.post = AsyncMock(return_value=Mock(status_code=200)) + mock_get_client.return_value = mock_client + + await guardrail._send_request("http://example.com", {}, {}) + + assert mock_client.post.call_args.kwargs["timeout"] == 5.0 + + +@pytest.mark.asyncio +async def test_send_request_uses_configured_timeout(): + """ + Regression for LIT-5222: a configured timeout must reach the HTTP call. + + Before the fix _send_request passed a module-level constant, so a slow + upstream failed at 5s with `Timeout passed=5` no matter what was configured. + """ + guardrail = ZscalerAIGuard(api_key="test_key", policy_id=1, timeout=30) + + assert guardrail.timeout == 30 + + with patch( + "litellm.proxy.guardrails.guardrail_hooks.zscaler_ai_guard.zscaler_ai_guard.get_async_httpx_client" + ) as mock_get_client: + mock_client = Mock() + mock_client.post = AsyncMock(return_value=Mock(status_code=200)) + mock_get_client.return_value = mock_client + + await guardrail._send_request("http://example.com", {}, {}) + + assert mock_client.post.call_args.kwargs["timeout"] == 30 + + +def test_initialize_guardrail_forwards_configured_timeout(): + """ + Regression for LIT-5222: the `timeout` key from config.yaml must survive + initialization. It reaches LitellmParams already, but the initializer used + to drop it before it could reach the guardrail instance. + """ + from litellm.proxy.guardrails.guardrail_hooks.zscaler_ai_guard import ( + initialize_guardrail, + ) + from litellm.types.guardrails import LitellmParams + + litellm_params = LitellmParams( + guardrail="zscaler_ai_guard", + mode="pre_call", + api_key="test_key", + api_base="http://example.com", + policy_id=1, + timeout="30", + ) + + guardrail = initialize_guardrail( + litellm_params, {"guardrail_name": "zscaler-configured-timeout"} + ) + + assert guardrail.timeout == 30.0 + + +def test_config_model_exposes_timeout_to_dashboard(): + """ + The dashboard guardrail form is built from get_config_model(), so the field + has to be declared there for the setting to be reachable outside config.yaml. + """ + config_model = ZscalerAIGuard.get_config_model() + + assert config_model is not None + assert "timeout" in config_model.model_fields + + +@pytest.mark.parametrize("bad_timeout", [0, -1]) +def test_non_positive_timeout_falls_back_to_default(bad_timeout): + """ + Regression: httpx rejects a negative timeout and treats 0 as "fail + immediately", so a non-positive value would break every scan instead of + relaxing the limit the operator was trying to raise. + """ + guardrail = ZscalerAIGuard(api_key="test_key", policy_id=1, timeout=bad_timeout) + + assert guardrail.timeout == 5.0 + + +def test_update_in_memory_litellm_params_keeps_timeout_resolved(): + """ + Regression: the base implementation copies every LitellmParams attribute + onto the guardrail, so an unset timeout would overwrite the resolved value + with None and silently fall back to the shared client's 600s default. + """ + from litellm.types.guardrails import LitellmParams + + guardrail = ZscalerAIGuard(api_key="test_key", policy_id=1, timeout=30) + assert guardrail.timeout == 30 + + guardrail.update_in_memory_litellm_params( + LitellmParams(guardrail="zscaler_ai_guard", mode="pre_call", api_key="test_key") + ) + assert guardrail.timeout == 5.0 + + guardrail.update_in_memory_litellm_params( + LitellmParams( + guardrail="zscaler_ai_guard", mode="pre_call", api_key="test_key", timeout=45 + ) + ) + assert guardrail.timeout == 45.0 diff --git a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py index 65448c6281e..c263b8ce381 100644 --- a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py +++ b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py @@ -30,6 +30,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( ) from fastapi import Request from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( _update_metadata_with_tags_in_header, HttpPassThroughEndpointHelpers, @@ -652,3 +653,119 @@ def test_custom_pricing_used_in_cost_calculation(): print(f"Cache-aware cost: {cache_cost}") print("✅ Custom pricing parameters are correctly used in cost calculation") + + +def test_init_kwargs_client_metadata_cannot_spoof_authenticated_identity( + mock_request, mock_user_api_key_dict +): + request = mock_request() + passthrough_payload = PassthroughStandardLoggingPayload( + url="https://test.com", + request_body={}, + ) + authenticated_key = UserAPIKeyAuth( + api_key="test-key", + user_id="test-user", + team_id="test-team", + end_user_id="test-user", + key_alias="real-key", + team_alias="Real Team", + user_email="real@example.com", + org_id="real-org", + ) + + result = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( + request=request, + user_api_key_dict=authenticated_key, + passthrough_logging_payload=passthrough_payload, + litellm_call_id="test-call-id", + logging_obj=LiteLLMLoggingObj( + model="test-model", + messages=[], + stream=False, + call_type="test-call-type", + start_time=datetime.now(), + litellm_call_id="test-call-id", + function_id="test-function-id", + ), + _parsed_body={ + "litellm_metadata": { + "user_api_key_org_id": "victim-org", + "user_api_key_end_user_id": "victim-end-user", + "user_api_key_user_id": "victim-user", + "user_api_key_team_id": "victim-team", + "user_api_key_team_alias": "Victim Team", + "user_api_key_alias": "victim-key", + "user_api_key_user_email": "victim@example.com", + } + }, + ) + + metadata = result["litellm_params"]["metadata"] + assert metadata["user_api_key_user_id"] == "test-user" + assert metadata["user_api_key_team_id"] == "test-team" + assert metadata["user_api_key_team_alias"] == "Real Team" + assert metadata["user_api_key_alias"] == "real-key" + assert metadata["user_api_key_user_email"] == "real@example.com" + assert metadata["user_api_key_org_id"] == "real-org" + assert metadata["user_api_key_end_user_id"] == "test-user" + + +def test_init_kwargs_no_authenticated_identity_field_is_client_settable( + mock_request, mock_user_api_key_dict +): + authenticated_key = UserAPIKeyAuth( + api_key="test-key", + user_id="test-user", + team_id="test-team", + end_user_id="test-end-user", + key_alias="real-key", + team_alias="Real Team", + user_email="real@example.com", + org_id="real-org", + organization_alias="Real Org", + project_id="real-project", + project_alias="Real Project", + spend=1.5, + max_budget=10.0, + user_spend=2.5, + user_max_budget=20.0, + team_spend=3.5, + team_max_budget=30.0, + metadata={"real": "auth-metadata"}, + ) + expected = dict( + LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( + user_api_key_dict=authenticated_key + ) + ) + assert len(expected) >= 20 + + spoofed = {key: f"SPOOFED-{key}" for key in expected} + + result = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( + request=mock_request(), + user_api_key_dict=authenticated_key, + passthrough_logging_payload=PassthroughStandardLoggingPayload( + url="https://test.com", request_body={} + ), + litellm_call_id="test-call-id", + logging_obj=LiteLLMLoggingObj( + model="test-model", + messages=[], + stream=False, + call_type="test-call-type", + start_time=datetime.now(), + litellm_call_id="test-call-id", + function_id="test-function-id", + ), + _parsed_body={"litellm_metadata": dict(spoofed), "metadata": dict(spoofed)}, + ) + + metadata = result["litellm_params"]["metadata"] + survived = { + key: metadata.get(key) + for key in expected + if metadata.get(key) != expected[key] + } + assert survived == {}, f"client-supplied values survived for: {sorted(survived)}" diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index bcc70fae67c..0655763d41b 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -15,6 +15,7 @@ from unittest.mock import patch, MagicMock, AsyncMock from create_mock_standard_logging_payload import create_standard_logging_payload from litellm.types.utils import StandardLoggingPayload from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo +from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS @pytest.fixture @@ -1816,6 +1817,7 @@ def test_init_auto_router_deployment_success(mock_auto_router, model_list): default_model="gpt-5-mini", embedding_model="text-embedding-3-small", litellm_router_instance=router, + max_input_chars=DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ) # Verify the auto-router was added to the router's auto_routers dict diff --git a/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py b/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py index 55db31efd2c..fb4cb494bee 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py +++ b/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py @@ -162,3 +162,56 @@ class TestGetLitellmParamsDataResidency: api_base="https://eu.api.openai.com/v1", ) assert result["data_residency"] is None + + +class TestMetadataFallsBackToLitellmMetadata: + def test_metadata_falls_back_to_litellm_metadata_when_absent(self): + result = get_litellm_params(litellm_metadata={"trace_id": "trace-1"}) + assert result["metadata"] == {"trace_id": "trace-1"} + assert result["litellm_metadata"] == {"trace_id": "trace-1"} + + def test_empty_metadata_falls_back_to_litellm_metadata(self): + result = get_litellm_params(metadata={}, litellm_metadata={"trace_id": "trace-1"}) + assert result["metadata"] == {"trace_id": "trace-1"} + + def test_metadata_wins_when_both_present(self): + result = get_litellm_params( + metadata={"trace_id": "from-metadata"}, + litellm_metadata={"trace_id": "from-litellm-metadata"}, + ) + assert result["metadata"] == {"trace_id": "from-metadata"} + + @pytest.mark.parametrize("bad_value", ["not-json-a-string", 12345, ["a"], True]) + def test_non_dict_litellm_metadata_is_ignored(self, bad_value): + result = get_litellm_params(litellm_metadata=bad_value) + assert result["metadata"] is None + + def test_metadata_stays_none_without_litellm_metadata(self): + result = get_litellm_params(api_key="test-key") + assert result["metadata"] is None + + def test_session_and_trace_id_derived_from_litellm_metadata(self): + result = get_litellm_params( + litellm_metadata={"trace_id": "trace-1", "session_id": "session-1"}, + ) + assert result["litellm_session_id"] == "session-1" + assert result["litellm_trace_id"] == "trace-1" + + def test_explicit_session_and_trace_id_are_not_overridden(self): + result = get_litellm_params( + litellm_session_id="explicit-session", + litellm_trace_id="explicit-trace", + litellm_metadata={"trace_id": "trace-1", "session_id": "session-1"}, + ) + assert result["litellm_session_id"] == "explicit-session" + assert result["litellm_trace_id"] == "explicit-trace" + + def test_litellm_metadata_fallback_is_copied_not_aliased(self): + litellm_metadata = {"trace_id": "trace-1"} + + result = get_litellm_params(litellm_metadata=litellm_metadata) + + assert result["metadata"] == litellm_metadata + assert result["metadata"] is not litellm_metadata + result["metadata"].pop("trace_id") + assert litellm_metadata == {"trace_id": "trace-1"} diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index a09c45cb141..23e0975cd08 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -526,6 +526,26 @@ class TestUpdateFromKwargs: ) assert logging_obj.litellm_params["litellm_call_id"] == "call-empty" + @pytest.mark.parametrize("caller_metadata", [None, "not-a-dict", 42]) + def test_non_dict_caller_metadata_does_not_break_the_merge(self, logging_obj, caller_metadata): + logging_obj.update_from_kwargs( + kwargs={"metadata": caller_metadata, "litellm_metadata": {"user_api_key_hash": "hashed"}}, + litellm_params={"metadata": {"user_api_key_hash": "hashed", "litellm_api_version": "1.0"}}, + ) + + assert logging_obj.litellm_params["metadata"]["user_api_key_hash"] == "hashed" + + def test_does_not_mutate_caller_metadata_dict(self, logging_obj): + caller_metadata: dict = {} + + logging_obj.update_from_kwargs( + kwargs={"metadata": caller_metadata, "litellm_metadata": {"user_api_key_hash": "hashed"}}, + litellm_params={"metadata": {"user_api_key_hash": "hashed", "litellm_api_version": "1.0"}}, + ) + + assert caller_metadata == {} + assert logging_obj.litellm_params["metadata"]["user_api_key_hash"] == "hashed" + def test_logging_prevent_double_logging(logging_obj): """ diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index cf83300ab3b..6ac1e15e7b5 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -20,6 +20,7 @@ from __future__ import annotations import asyncio import inspect import json +import logging import os from typing import List, Optional, Union from unittest.mock import AsyncMock, MagicMock, patch @@ -31,6 +32,7 @@ from typing_extensions import TypedDict import litellm.proxy.proxy_server as ps from litellm.proxy.proxy_server import ( + ProxyStartupEvent, _initialize_shared_aiohttp_session, _resolve_pydantic_type, _resolve_typed_dict_type, @@ -728,3 +730,50 @@ def test_otel_global_provider_published_after_callback_init(): "preset logger will not exist yet and a second generic logger will own " "the global provider, orphaning gen-ai spans" ) + + +def test_startup_warns_for_global_budget_without_database(caplog): + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + ProxyStartupEvent._warn_budget_without_db(max_budget=100.0, prisma_client=None) + + assert "litellm.max_budget=100.0" in caplog.text + assert "will NOT be enforced" in caplog.text + assert "requests will never be blocked" in caplog.text + + +def test_startup_does_not_warn_for_global_budget_with_database(caplog): + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + ProxyStartupEvent._warn_budget_without_db(max_budget=100.0, prisma_client=MagicMock()) + + assert "litellm.max_budget" not in caplog.text + + +@pytest.mark.parametrize("max_budget", [0, None]) +def test_startup_does_not_warn_without_global_budget(caplog, max_budget): + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + ProxyStartupEvent._warn_budget_without_db(max_budget=max_budget, prisma_client=None) + + assert "litellm.max_budget" not in caplog.text + + +def test_proxy_startup_event_warns_for_global_budget_without_database(): + """Pin the lifespan call that prevents silent DB-less budgets. + + The call must follow Prisma setup so DB-backed deployments do not false-positive. + Direct ``_warn_budget_without_db`` tests cover the warning behavior itself. + """ + wrapped = getattr(proxy_startup_event, "__wrapped__", proxy_startup_event) + source = inspect.getsource(wrapped) + budget_check_pos = source.find("if prisma_client is not None and litellm.max_budget > 0:") + warn_pos = source.find("_warn_budget_without_db(") + next_startup_section_pos = source.find( + "await ProxyStartupEvent.initialize_scheduled_background_jobs(", + budget_check_pos, + ) + + assert budget_check_pos != -1, "global budget startup block not found" + assert warn_pos != -1, "DB-less budget warning call not found" + assert next_startup_section_pos != -1, "startup section after budget block not found" + assert budget_check_pos < warn_pos < next_startup_section_pos, ( + "DB-less budget warning must run after Prisma setup and the DB-backed budget block" + ) diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 0e9aac7bf85..6d6fd2e5507 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -20,6 +20,7 @@ from litellm.proxy.litellm_pre_call_utils import ( _get_dynamic_logging_metadata, _get_enforced_params, _get_metadata_variable_name, + _promoted_trace_control_fields, _resolve_credential_from_model_config, _resolve_provider_from_deployment, _update_model_if_key_alias_exists, @@ -5869,4 +5870,183 @@ async def test_key_level_callback_vars_survive_the_strip(): ) assert updated[TRUSTED_CALLBACK_VARS_FIELD] == {"dd_api_key": "key-dd-key", "dd_site": "us5.datadoghq.com"} - assert updated["dd_site"] == "us5.datadoghq.com" \ No newline at end of file + assert updated["dd_site"] == "us5.datadoghq.com" + + +class TestPromotedTraceControlFields: + """LIT-5137: caller metadata trace fields must reach litellm_metadata.""" + + def _make_request(self, path: str) -> MagicMock: + request = MagicMock(spec=Request) + request.url = MagicMock() + request.url.path = path + request.url.__str__.return_value = f"http://localhost{path}" + request.method = "POST" + request.query_params = {} + request.headers = {"Content-Type": "application/json"} + request.client = MagicMock() + request.client.host = "127.0.0.1" + return request + + async def _run(self, path: str, data: dict, headers: dict | None = None) -> dict: + request = self._make_request(path) + if headers is not None: + request.headers = {"Content-Type": "application/json", **headers} + return await add_litellm_data_to_request( + data=data, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + def test_returns_litellm_metadata_for_responses_route(self): + assert _get_metadata_variable_name(self._make_request("/v1/responses")) == "litellm_metadata" + + def test_promotes_trace_prefixed_and_allow_listed_fields(self): + requester_metadata = { + "trace_id": "trace-1", + "trace_name": "name-1", + "trace_user_id": "user-1", + "trace_metadata": {"tenant_id": "tenant-1"}, + "trace_version": "v1", + "trace_release": "r1", + "session_id": "session-1", + "mask_input": True, + "mask_output": True, + } + + promoted = _promoted_trace_control_fields( + requester_metadata=requester_metadata, + litellm_metadata={}, + ) + + assert dict(promoted) == requester_metadata + + def test_does_not_promote_unlisted_trace_prefixed_fields(self): + """trace_public flips a trace to publicly readable, so the allow-list is explicit.""" + promoted = _promoted_trace_control_fields( + requester_metadata={"trace_id": "trace-1", "trace_public": True, "trace_tags": ["a"]}, + litellm_metadata={}, + ) + + assert dict(promoted) == {"trace_id": "trace-1"} + + def test_does_not_promote_non_trace_fields(self): + promoted = _promoted_trace_control_fields( + requester_metadata={ + "trace_id": "trace-1", + "tags": ["free-tier"], + "user_api_key": "forged", + "user_api_key_user_id": "forged-user", + "spend_logs_metadata": {"forged": True}, + "guardrails": ["disabled"], + "debug_langfuse": True, + "session": "not-session-id", + "existing_trace_id": "victim-trace", + "update_trace_keys": ["input", "output"], + }, + litellm_metadata={}, + ) + + assert dict(promoted) == {"trace_id": "trace-1"} + + def test_does_not_promote_trace_mutation_controls(self): + """existing_trace_id + update_trace_keys let a caller overwrite any trace in the project.""" + promoted = _promoted_trace_control_fields( + requester_metadata={ + "trace_id": "trace-1", + "existing_trace_id": "someone-elses-trace", + "update_trace_keys": ["input", "output"], + }, + litellm_metadata={}, + ) + + assert dict(promoted) == {"trace_id": "trace-1"} + + def test_existing_litellm_metadata_value_wins(self): + promoted = _promoted_trace_control_fields( + requester_metadata={"trace_id": "from-body", "session_id": "from-body", "trace_name": "from-body"}, + litellm_metadata={"trace_id": "from-header", "session_id": "from-header"}, + ) + + assert dict(promoted) == {"trace_name": "from-body"} + + def test_empty_requester_metadata_promotes_nothing(self): + assert _promoted_trace_control_fields(requester_metadata={}, litellm_metadata={}) == () + + @pytest.mark.asyncio + async def test_responses_route_end_to_end(self): + caller_metadata = { + "trace_id": "22662678-30c1-41a1-a24b-216d6e5fb83d", + "session_id": "218af06c-28a2-4705-8a0a-5f9970d39326", + "trace_user_id": "user-123", + "trace_metadata": {"tenant_id": "tenant-1"}, + "mask_input": True, + } + + updated = await self._run( + "/v1/responses", + {"model": "gpt-4.1-mini", "input": "say resp", "metadata": copy.deepcopy(caller_metadata)}, + ) + + litellm_metadata = updated["litellm_metadata"] + for key, value in caller_metadata.items(): + assert litellm_metadata[key] == value + assert updated["metadata"] == caller_metadata + + @pytest.mark.asyncio + async def test_messages_route_end_to_end(self): + updated = await self._run( + "/v1/messages", + { + "model": "claude-sonnet-4-5", + "max_tokens": 32, + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"trace_id": "msg-trace-1", "session_id": "msg-session-1"}, + }, + ) + + assert updated["litellm_metadata"]["trace_id"] == "msg-trace-1" + assert updated["litellm_metadata"]["session_id"] == "msg-session-1" + + @pytest.mark.asyncio + async def test_session_id_header_beats_body_metadata(self): + updated = await self._run( + "/v1/responses", + {"model": "gpt-4.1-mini", "input": "say resp", "metadata": {"session_id": "from-body"}}, + headers={"x-litellm-session-id": "from-header-12345678"}, + ) + + assert updated["litellm_metadata"]["session_id"] == "from-header-12345678" + + @pytest.mark.asyncio + async def test_forged_user_api_key_fields_are_not_promoted(self): + updated = await self._run( + "/v1/responses", + { + "model": "gpt-4.1-mini", + "input": "say resp", + "metadata": {"trace_id": "trace-1", "user_api_key_user_id": "forged", "spend_logs_metadata": {"a": 1}}, + }, + ) + + litellm_metadata = updated["litellm_metadata"] + assert litellm_metadata["trace_id"] == "trace-1" + assert litellm_metadata.get("user_api_key_user_id") != "forged" + + @pytest.mark.asyncio + async def test_chat_completions_route_is_untouched(self): + updated = await self._run( + "/v1/chat/completions", + { + "model": "gpt-4.1-mini", + "messages": [{"role": "user", "content": "hello"}], + "metadata": {"trace_id": "trace-1", "session_id": "session-1"}, + }, + ) + + assert "litellm_metadata" not in updated + assert updated["metadata"]["trace_id"] == "trace-1" + assert updated["metadata"]["session_id"] == "session-1" diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index 57afdbbd28b..6a5a1e0f159 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -636,8 +636,11 @@ describe("AddAutoRouterTab", () => { expect(labels).toEqual(["Anthropic Family", "OpenAI Family", "Custom Configuration"]); }); - it("never lets a wildcard deployment satisfy a preset", async () => { - const wildcard = [{ model_name: "openai-wild", litellm_params: { model: "openai/*" } }]; + it.each([ + ["a wildcard group", "openai/*"], + ["a plain group over a wildcard underlying model", "openai-wild"], + ])("never lets %s satisfy a preset when the hub lists no expansions", async (_label, modelName) => { + const wildcard = [{ model_name: modelName, litellm_params: { model: "openai/*" } }]; mockFetchAvailableModels.mockResolvedValue(groupsFor(wildcard)); mockFetchAllModelDeployments.mockResolvedValue(wildcard); @@ -650,4 +653,59 @@ describe("AddAutoRouterTab", () => { expect(isOptionDisabled(optionByLabel("OpenAI Family")!)).toBe(true); }); }); + + describe("wildcard-matched presets", () => { + const WILDCARD_DEPLOYMENTS = [{ model_name: "someprovider/*", litellm_params: { model: "someprovider/*" } }]; + + const expandedGroupFor = (model: string): string => `someprovider/${model}`; + + const EXPANDED_HUB_GROUPS: ModelGroup[] = [ + { model_group: "someprovider/*", mode: "chat" }, + ...[...new Set(getAllPresets().flatMap((preset) => [...getRequiredModelsInPreset(preset)]))].map((model) => ({ + model_group: expandedGroupFor(model), + mode: "chat", + })), + ]; + + it("enables a preset whose models exist only as wildcard-expanded groups, labeling the match", async () => { + mockFetchAvailableModels.mockResolvedValue(EXPANDED_HUB_GROUPS); + mockFetchAllModelDeployments.mockResolvedValue(WILDCARD_DEPLOYMENTS); + + renderWithProviders(); + openTemplateDropdown(); + + await waitFor(() => { + expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); + }); + expect(optionByLabel("Anthropic Family")!.textContent).toContain("Matches your deployments"); + }); + + it("prefills the expanded group names and submits them", async () => { + const user = userEvent.setup(); + mockFetchAvailableModels.mockResolvedValue(EXPANDED_HUB_GROUPS); + mockFetchAllModelDeployments.mockResolvedValue(WILDCARD_DEPLOYMENTS); + + renderWithProviders(); + openTemplateDropdown(); + await waitFor(() => { + expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); + }); + fireEvent.click(optionByLabel("Anthropic Family")!); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "wildcard-router"); + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0]).toMatchObject({ + complexity_router_config: { + tiers: { + SIMPLE: ANTHROPIC_TIERS.SIMPLE.map(expandedGroupFor), + MEDIUM: ANTHROPIC_TIERS.MEDIUM.map(expandedGroupFor), + COMPLEX: ANTHROPIC_TIERS.COMPLEX.map(expandedGroupFor), + REASONING: ANTHROPIC_TIERS.REASONING.map(expandedGroupFor), + }, + }, + }); + }); + }); }); diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index 0d965be054b..fca8420966f 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -191,6 +191,148 @@ describe("autorouter_presets", () => { ); }); + describe("wildcard deployment matching (expanded model groups)", () => { + const wildcardDeployment = (pattern: string) => ({ modelGroup: pattern, underlyingModels: [pattern] }); + + const simpleTierConfig = (presetModel: string) => ({ + tiers: { SIMPLE: [presetModel], MEDIUM: [], COMPLEX: [], REASONING: [] }, + classifier_type: "heuristic" as const, + session_affinity: false, + }); + + it("resolves a preset model to a group expanded from a wildcard deployment", () => { + const availability = buildModelAvailability( + ["anthropic/*", "anthropic/claude-opus-5", "bedrock/anthropic.claude-opus-5"], + [wildcardDeployment("anthropic/*")], + ); + const config = simpleTierConfig("claude-opus-5"); + expect(getMissingModels(config, availability)).toEqual([]); + expect(buildPresetPrefill(config, availability).complexityRouterConfig.tiers.SIMPLE).toEqual([ + "anthropic/claude-opus-5", + ]); + }); + + it("normalizes an expanded group's namespaced own name the same way as a deployment's", () => { + const availability = buildModelAvailability( + ["bedrock/*", "bedrock/us.anthropic.claude-sonnet-5"], + [wildcardDeployment("bedrock/*")], + ); + expect(getMissingModels(simpleTierConfig("claude-sonnet-5"), availability)).toEqual([]); + }); + + it("anchors a partial wildcard pattern and treats its dots literally", () => { + const availability = buildModelAvailability( + ["bedrock/us.anthropic.claude-opus-5", "bedrock/usXanthropic.claude-fable-5"], + [wildcardDeployment("bedrock/us.*")], + ); + expect(getMissingModels(simpleTierConfig("claude-opus-5"), availability)).toEqual([]); + expect(getMissingModels(simpleTierConfig("claude-fable-5"), availability)).toEqual(["claude-fable-5"]); + }); + + it.each([ + ["gpt-5.4", "openai/gpt-5.4-mini"], + ["gpt-5.4-mini", "openai/gpt-5.4"], + ["o3", "openai/o3-mini"], + ])("never lets %s be satisfied by the expanded group %s", (presetModel, expandedGroup) => { + const availability = buildModelAvailability(["openai/*", expandedGroup], [wildcardDeployment("openai/*")]); + expect(getMissingModels(simpleTierConfig(presetModel), availability)).toEqual([presetModel]); + }); + + it("anchors the pattern's suffix and keeps middle segments in order", () => { + const availability = buildModelAvailability( + ["bedrock/us.anthropic.claude-opus-5", "bedrock/anthropic.us.claude-sonnet-5"], + [wildcardDeployment("bedrock/*.anthropic.*")], + ); + expect(getMissingModels(simpleTierConfig("claude-opus-5"), availability)).toEqual([]); + expect(getMissingModels(simpleTierConfig("claude-sonnet-5"), availability)).toEqual(["claude-sonnet-5"]); + }); + + it("matches a pathological many-star pattern in linear time instead of backtracking", () => { + const hostile = `prov/a*${"a*".repeat(30)}b`; + const nonMatching = `prov/${"a".repeat(120)}`; + const availability = buildModelAvailability([nonMatching], [wildcardDeployment(hostile)]); + expect(availability.underlyingIndex.size).toBe(0); + }); + + it("expands a bare-star model_name through its underlying wildcard, not as match-all", () => { + const availability = buildModelAvailability( + ["openai/gpt-5.4", "team-a/claude-opus-5"], + [{ modelGroup: "*", underlyingModels: ["openai/*"] }], + ); + expect(getMissingModels(simpleTierConfig("gpt-5.4"), availability)).toEqual([]); + expect(getMissingModels(simpleTierConfig("claude-opus-5"), availability)).toEqual(["claude-opus-5"]); + }); + + it.each([ + ["a bare-star underlying", "*"], + ["a non-wildcard underlying", "openai/gpt-4o"], + ["a slashless wildcard underlying", "gpt*"], + ])("derives no pattern from a bare-star model_name with %s", (_label, underlying) => { + const availability = buildModelAvailability( + ["openai/gpt-5.4"], + [{ modelGroup: "*", underlyingModels: [underlying] }], + ); + expect(availability.underlyingIndex.size).toBe(0); + }); + + it("derives no pattern from a slashless wildcard model_name", () => { + const availability = buildModelAvailability(["gpt-5.4"], [wildcardDeployment("gpt*")]); + expect(availability.underlyingIndex.size).toBe(0); + }); + + it("does not trust a group's name when no wildcard deployment covers it", () => { + const availability = buildModelAvailability( + ["team-a/claude-opus-5", "openai/*"], + [wildcardDeployment("openai/*")], + ); + expect(getMissingModels(simpleTierConfig("claude-opus-5"), availability)).toEqual(["claude-opus-5"]); + }); + + it("never resolves to the wildcard group itself when the hub lists no expansions", () => { + const availability = buildModelAvailability(["openai/*"], [wildcardDeployment("openai/*")]); + expect(getMissingModels(simpleTierConfig("gpt-5.4"), availability)).toEqual(["gpt-5.4"]); + expect(availability.underlyingIndex.size).toBe(0); + }); + + it("applies a wildcard deployment's pattern even when the wildcard group is not itself listed", () => { + const availability = buildModelAvailability(["anthropic/claude-opus-5"], [wildcardDeployment("anthropic/*")]); + expect(getMissingModels(simpleTierConfig("claude-opus-5"), availability)).toEqual([]); + }); + + it("keeps the groups-only availability strict even when expanded groups are listed", () => { + const availability = groupsOnly(["anthropic/*", "anthropic/claude-opus-5"]); + expect(getMissingModels(simpleTierConfig("claude-opus-5"), availability)).toEqual(["claude-opus-5"]); + }); + + it("prefers the alphabetically first covered group when several expansions serve the model", () => { + const availability = buildModelAvailability( + ["bedrock/us.anthropic.claude-opus-5", "anthropic/claude-opus-5", "bedrock/anthropic.claude-opus-5"], + [wildcardDeployment("anthropic/*"), wildcardDeployment("bedrock/*")], + ); + const config = simpleTierConfig("claude-opus-5"); + expect(buildPresetPrefill(config, availability).complexityRouterConfig.tiers.SIMPLE).toEqual([ + "anthropic/claude-opus-5", + ]); + }); + + it.each(getAllPresets().map((preset) => [preset.key, preset] as const))( + "fully resolves the %s preset through wildcard-expanded groups only", + (_key, preset) => { + const required = [...getRequiredModelsInPreset(preset)]; + const expandedGroups = required.map((model) => `someprovider/${model}`); + const availability = buildModelAvailability( + ["someprovider/*", ...expandedGroups], + [wildcardDeployment("someprovider/*")], + ); + expect(getMissingModelsInPreset(preset, availability)).toEqual([]); + const prefilled = buildPresetPrefill(preset.complexity_router_config, availability); + const prefilledModels = Object.values(prefilled.complexityRouterConfig.tiers).flat(); + expect(prefilledModels.length).toBeGreaterThan(0); + for (const model of prefilledModels) expect(expandedGroups).toContain(model); + }, + ); + }); + describe("deploymentRefsFromModelInfo", () => { it("keeps litellm_params.model and model_info.base_model, drops rows with neither or no name", () => { const refs = deploymentRefsFromModelInfo([ diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index 5b6e3dc6f20..ae3f30c90f5 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -77,12 +77,30 @@ const normalizeUnderlyingModel = (model: string): string | null => { return stripped.toLowerCase() || null; }; +// A linear glob scan rather than a RegExp: patterns are admin-controlled model_name values, and a +// backtracking regex built from one ("a*a*a*...") can freeze another admin's dashboard. +const matchesWildcard = (pattern: string, name: string): boolean => { + const parts = pattern.split("*"); + if (parts.length === 1) return pattern === name; + const head = parts[0]; + const tail = parts[parts.length - 1]; + if (!name.startsWith(head) || !name.endsWith(tail)) return false; + if (name.length < head.length + tail.length) return false; + const scanEnd = name.length - tail.length; + const scanResult = parts.slice(1, -1).reduce((searchFrom: number, part: string) => { + if (searchFrom < 0) return -1; + const found = name.indexOf(part, searchFrom); + return found === -1 || found + part.length > scanEnd ? -1 : found + part.length; + }, head.length); + return scanResult >= 0; +}; + export const buildModelAvailability = ( modelGroups: Iterable, deployments: readonly DeploymentModelRef[], ): ModelAvailability => { const groups = new Set(modelGroups); - const entries = deployments + const literalEntries = deployments .filter((deployment) => groups.has(deployment.modelGroup)) .flatMap((deployment) => deployment.underlyingModels @@ -90,6 +108,22 @@ export const buildModelAvailability = ( .filter((key): key is string => key !== null) .map((key) => ({ key, modelGroup: deployment.modelGroup })), ); + // Mirrors get_known_models_from_wildcard: a bare "*" model_name expands via its underlying + // wildcard (or not at all), and a wildcard without a "/" expands to nothing. + const wildcardPatterns = Array.from( + new Set( + deployments + .flatMap((deployment) => + deployment.modelGroup === "*" ? deployment.underlyingModels : [deployment.modelGroup], + ) + .filter((pattern) => pattern !== "*" && pattern.includes("*") && pattern.includes("/")), + ), + ); + const wildcardEntries = Array.from(groups) + .filter((group) => !group.includes("*") && wildcardPatterns.some((pattern) => matchesWildcard(pattern, group))) + .map((group) => ({ key: normalizeUnderlyingModel(group), modelGroup: group })) + .filter((entry): entry is { key: string; modelGroup: string } => entry.key !== null); + const entries = [...literalEntries, ...wildcardEntries]; const grouped = new Map>(); for (const entry of entries) { const groupsForKey = grouped.get(entry.key) ?? new Set(); diff --git a/uv.lock b/uv.lock index 9c2897b5e4f..a42a164e5f0 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-02T02:14:05.876141Z" +exclude-newer = "2026-08-04T00:00:57.623181Z" exclude-newer-span = "P3D" [manifest] @@ -4583,12 +4583,12 @@ proxy-dev = [ [[package]] name = "litellm-enterprise" -version = "0.1.53" +version = "0.1.54" source = { editable = "enterprise" } [[package]] name = "litellm-proxy-extras" -version = "0.4.83" +version = "0.4.84" source = { editable = "litellm-proxy-extras" } [[package]]