From 27f693348c560fdbf3574e2ce8a8ad2c54444ff5 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 22 Sep 2026 06:06:16 +0000 Subject: [PATCH] feat(headroom): read min_tokens from HEADROOM_MIN_TOKENS when the config omits it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_lazy_openapi_snapshot.json | 16 ++++--- .../guardrail_hooks/headroom/headroom.py | 25 +++++++++-- .../guardrails/guardrail_hooks/headroom.py | 6 +-- .../guardrail_hooks/test_headroom.py | 42 ++++++++++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 +-- 5 files changed, 78 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index b51946d0709..ee38d8167fd 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -12119,11 +12119,17 @@ "title": "Metadata" }, "min_tokens": { - "default": 0, - "description": "Skip the compression round trip when the compressible messages total fewer than this many tokens. The default of 0 always calls the compression service.", - "minimum": 0.0, - "title": "Min Tokens", - "type": "integer" + "anyOf": [ + { + "minimum": 0.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Skip the compression round trip when the compressible messages total fewer than this many tokens. Falls back to the HEADROOM_MIN_TOKENS env var; when neither is set, every request is compressed.", + "title": "Min Tokens" }, "mock_redacted_text": { "anyOf": [ diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index 33828e6a445..5f01fe7543e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -70,6 +70,23 @@ _HASH_CACHE_TTL_SECONDS: Final = 15 * 60 # untranslated messages can be read with concrete types (values pass through by # reference, so this is a shallow top-level reconstruction). _REQUEST_DATA_ADAPTER: Final = TypeAdapter(dict[str, object]) +MIN_TOKENS_ENV_VAR: Final = "HEADROOM_MIN_TOKENS" + + +def _resolve_min_tokens(configured: int | None) -> int | None: + """Config wins over the env var; neither set means every request is compressed.""" + if configured is not None: + return configured + raw: Final = get_secret_str(MIN_TOKENS_ENV_VAR) + if raw is None or not raw.strip(): + return None + try: + value: Final = int(raw) + except ValueError as exc: + raise ValueError(f"{MIN_TOKENS_ENV_VAR} must be a non-negative integer, got {raw!r}") from exc + if value < 0: + raise ValueError(f"{MIN_TOKENS_ENV_VAR} must be a non-negative integer, got {raw!r}") + return value def _is_str_object_dict(value: object) -> TypeGuard[dict[str, object]]: # guard-ok: isinstance narrows correctly; predicate is trivially correct # fmt: skip @@ -515,7 +532,7 @@ class HeadroomGuardrail(CustomGuardrail): unreachable_fallback: str | None = None, timeout: float | None = None, ccr_retrieval: bool = True, - min_tokens: int = 0, + min_tokens: int | None = None, ): self.headroom_api_base = (api_base or get_secret_str("HEADROOM_API_BASE") or "").rstrip("/") if not self.headroom_api_base: @@ -530,7 +547,7 @@ class HeadroomGuardrail(CustomGuardrail): ) self.timeout: httpx.Timeout = self._resolve_timeout(timeout) self.ccr_retrieval = ccr_retrieval - self.min_tokens = min_tokens + self.min_tokens: int | None = _resolve_min_tokens(min_tokens) self.async_handler = get_async_httpx_client( llm_provider=httpxSpecialProvider.GuardrailCallback, ) @@ -833,13 +850,13 @@ class HeadroomGuardrail(CustomGuardrail): model: Final = self.headroom_model or request_data.get("model") flattened: Final = _flatten_messages_for_compression(compressible) compressible_tokens: Final = ( - await _estimate_compressible_tokens(model=model, messages=flattened) if self.min_tokens > 0 else None + await _estimate_compressible_tokens(model=model, messages=flattened) if self.min_tokens else None ) from litellm.proxy.common_utils.callback_utils import ( add_guardrail_to_applied_guardrails_header, ) - if compressible_tokens is not None and compressible_tokens < self.min_tokens: + if self.min_tokens and compressible_tokens is not None and compressible_tokens < self.min_tokens: verbose_proxy_logger.debug( "Headroom: %s compressible tokens below min_tokens=%s; skipping compression", compressible_tokens, diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/headroom.py b/litellm/types/proxy/guardrails/guardrail_hooks/headroom.py index 19c78adf39e..2bf95ae7c6d 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/headroom.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/headroom.py @@ -30,12 +30,12 @@ class HeadroomGuardrailConfigModel(GuardrailConfigModel[BaseModel]): default=True, description="Inject the Headroom retrieval tool for hashes declared by the compression service.", ) - min_tokens: int = Field( - default=0, + min_tokens: int | None = Field( + default=None, ge=0, description=( "Skip the compression round trip when the compressible messages total fewer than this many tokens. " - "The default of 0 always calls the compression service." + "Falls back to the HEADROOM_MIN_TOKENS env var; when neither is set, every request is compressed." ), ) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py index a87c13e80e2..763161ed1c6 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -1137,7 +1137,47 @@ def test_initialize_guardrail_passes_min_tokens(monkeypatch: pytest.MonkeyPatch) ) assert configured_guardrail.min_tokens == 42 - assert default_guardrail.min_tokens == 0 + assert default_guardrail.min_tokens is None + + +def test_min_tokens_falls_back_to_env_var(monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("HEADROOM_MIN_TOKENS", raising=False) + assert _make_guardrail(min_tokens=None).min_tokens is None + + monkeypatch.setenv("HEADROOM_MIN_TOKENS", "1500") + assert _make_guardrail(min_tokens=None).min_tokens == 1500 + assert _make_guardrail(min_tokens=7).min_tokens == 7 + + monkeypatch.setenv("HEADROOM_MIN_TOKENS", "") + assert _make_guardrail(min_tokens=None).min_tokens is None + + monkeypatch.setenv("HEADROOM_MIN_TOKENS", "-5") + with pytest.raises(ValueError, match="HEADROOM_MIN_TOKENS"): + _make_guardrail(min_tokens=None) + + monkeypatch.setenv("HEADROOM_MIN_TOKENS", "lots") + with pytest.raises(ValueError, match="HEADROOM_MIN_TOKENS"): + _make_guardrail(min_tokens=None) + + +@pytest.mark.asyncio +async def test_env_var_threshold_skips_compression(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("HEADROOM_MIN_TOKENS", "1000000") + guardrail = _make_guardrail(min_tokens=None) + inputs = GenericGuardrailAPIInputs( + texts=[str(m["content"]) for m in ORIGINAL_MESSAGES], + structured_messages=ORIGINAL_MESSAGES, + ) + request_data = {"model": "gpt-4o"} + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as post: + result = await guardrail.apply_guardrail(inputs=inputs, request_data=request_data, input_type="request") + + assert result is inputs + post.assert_not_awaited() + response = _recorded_guardrail_response(request_data) + assert response["skipped"] == "below_min_tokens" + assert response["min_tokens"] == 1000000 @pytest.mark.asyncio diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a4ebd3d4de1..2f8fa5cc100 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -32642,10 +32642,9 @@ export interface components { } | null; /** * Min Tokens - * @description Skip the compression round trip when the compressible messages total fewer than this many tokens. The default of 0 always calls the compression service. - * @default 0 + * @description Skip the compression round trip when the compressible messages total fewer than this many tokens. Falls back to the HEADROOM_MIN_TOKENS env var; when neither is set, every request is compressed. */ - min_tokens: number; + min_tokens?: number | null; /** * Mock Redacted Text * @description Mock redacted text for testing