diff --git a/litellm/proxy/hooks/batch_rate_limiter.py b/litellm/proxy/hooks/batch_rate_limiter.py index 6b8dca3e8c4..38d4209499e 100644 --- a/litellm/proxy/hooks/batch_rate_limiter.py +++ b/litellm/proxy/hooks/batch_rate_limiter.py @@ -145,23 +145,25 @@ class _PROXY_BatchRateLimiter(CustomLogger): """ Skip downloading batch input files when configured or when there is nothing to enforce (no applicable rate limits and no model allowlist). + + When the caller's key has a model allowlist to enforce, no skip path + is honored: the JSONL must still be downloaded so + ``_enforce_batch_file_model_access`` can validate every ``body.model`` + entry. Otherwise a restricted key could smuggle unauthorized models + into the file via an admin-configured skip (global disable, + per-model, per-provider) or via the user-controlled + ``custom_llm_provider`` field. """ + if self._key_requires_batch_model_access_check(user_api_key_dict): + return False + from litellm.proxy.proxy_server import general_settings if general_settings.get("disable_batch_input_file_rate_limiting") is True: return True - # Only honor the metadata-based skip when the key has no model - # allowlist to enforce. Otherwise a caller could set this flag in - # the request body (it lands in ``litellm_metadata`` for batch - # routes) and skip ``_enforce_batch_file_model_access``, smuggling - # restricted models into the JSONL. litellm_metadata = data.get("litellm_metadata") or {} - if litellm_metadata.get( - "skip_batch_input_file_rate_limiting" - ) is True and not self._key_requires_batch_model_access_check( - user_api_key_dict - ): + if litellm_metadata.get("skip_batch_input_file_rate_limiting") is True: return True batch_model = self._get_batch_routing_model(data) @@ -189,19 +191,18 @@ class _PROXY_BatchRateLimiter(CustomLogger): ) return True - if not self._key_requires_batch_model_access_check(user_api_key_dict): - descriptors = self.parallel_request_limiter._create_rate_limit_descriptors( - user_api_key_dict=user_api_key_dict, - data=data, - rpm_limit_type=None, - tpm_limit_type=None, - model_has_failures=False, + descriptors = self.parallel_request_limiter._create_rate_limit_descriptors( + user_api_key_dict=user_api_key_dict, + data=data, + rpm_limit_type=None, + tpm_limit_type=None, + model_has_failures=False, + ) + if not self._has_applicable_batch_rate_limits(descriptors): + verbose_proxy_logger.debug( + "Skipping batch input file processing: no rate limits configured" ) - if not self._has_applicable_batch_rate_limits(descriptors): - verbose_proxy_logger.debug( - "Skipping batch input file processing: no rate limits configured" - ) - return True + return True return False diff --git a/tests/test_litellm/proxy/hooks/test_batch_file_validation.py b/tests/test_litellm/proxy/hooks/test_batch_file_validation.py index 4a1fe1a37dd..fdaaee381f7 100644 --- a/tests/test_litellm/proxy/hooks/test_batch_file_validation.py +++ b/tests/test_litellm/proxy/hooks/test_batch_file_validation.py @@ -14,7 +14,6 @@ from fastapi import HTTPException from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - # --------------------------------------------------------------------------- # Token counter — covers all three batch payload shapes # --------------------------------------------------------------------------- @@ -268,7 +267,7 @@ async def test_pre_call_skips_file_fetch_when_disabled_in_general_settings(): internal_usage_cache=MagicMock(), parallel_request_limiter=MagicMock(), ) - user = UserAPIKeyAuth(api_key="sk-ok", user_id="alice", models=["gpt-4o"]) + user = UserAPIKeyAuth(api_key="sk-ok", user_id="alice", models=["*"]) with patch( "litellm.proxy.proxy_server.general_settings", @@ -293,7 +292,7 @@ async def test_pre_call_skips_file_fetch_for_configured_provider(): internal_usage_cache=MagicMock(), parallel_request_limiter=MagicMock(), ) - user = UserAPIKeyAuth(api_key="sk-ok", user_id="alice", models=["gpt-4o"]) + user = UserAPIKeyAuth(api_key="sk-ok", user_id="alice", models=["*"]) with patch( "litellm.proxy.proxy_server.general_settings",