feat(headroom): read min_tokens from HEADROOM_MIN_TOKENS when the config omits it
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-09-22 06:06:16 +00:00
parent f622f69842
commit 27f693348c
5 changed files with 78 additions and 16 deletions

View file

@ -12119,11 +12119,17 @@
"title": "Metadata"
},
"min_tokens": {
"default": 0,
"description": "Skip the compression round trip when the compressible messages total fewer than this many tokens. The default of 0 always calls the compression service.",
"minimum": 0.0,
"title": "Min Tokens",
"type": "integer"
"anyOf": [
{
"minimum": 0.0,
"type": "integer"
},
{
"type": "null"
}
],
"description": "Skip the compression round trip when the compressible messages total fewer than this many tokens. Falls back to the HEADROOM_MIN_TOKENS env var; when neither is set, every request is compressed.",
"title": "Min Tokens"
},
"mock_redacted_text": {
"anyOf": [

View file

@ -70,6 +70,23 @@ _HASH_CACHE_TTL_SECONDS: Final = 15 * 60
# untranslated messages can be read with concrete types (values pass through by
# reference, so this is a shallow top-level reconstruction).
_REQUEST_DATA_ADAPTER: Final = TypeAdapter(dict[str, object])
MIN_TOKENS_ENV_VAR: Final = "HEADROOM_MIN_TOKENS"
def _resolve_min_tokens(configured: int | None) -> int | None:
"""Config wins over the env var; neither set means every request is compressed."""
if configured is not None:
return configured
raw: Final = get_secret_str(MIN_TOKENS_ENV_VAR)
if raw is None or not raw.strip():
return None
try:
value: Final = int(raw)
except ValueError as exc:
raise ValueError(f"{MIN_TOKENS_ENV_VAR} must be a non-negative integer, got {raw!r}") from exc
if value < 0:
raise ValueError(f"{MIN_TOKENS_ENV_VAR} must be a non-negative integer, got {raw!r}")
return value
def _is_str_object_dict(value: object) -> TypeGuard[dict[str, object]]: # guard-ok: isinstance narrows correctly; predicate is trivially correct # fmt: skip
@ -515,7 +532,7 @@ class HeadroomGuardrail(CustomGuardrail):
unreachable_fallback: str | None = None,
timeout: float | None = None,
ccr_retrieval: bool = True,
min_tokens: int = 0,
min_tokens: int | None = None,
):
self.headroom_api_base = (api_base or get_secret_str("HEADROOM_API_BASE") or "").rstrip("/")
if not self.headroom_api_base:
@ -530,7 +547,7 @@ class HeadroomGuardrail(CustomGuardrail):
)
self.timeout: httpx.Timeout = self._resolve_timeout(timeout)
self.ccr_retrieval = ccr_retrieval
self.min_tokens = min_tokens
self.min_tokens: int | None = _resolve_min_tokens(min_tokens)
self.async_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.GuardrailCallback,
)
@ -833,13 +850,13 @@ class HeadroomGuardrail(CustomGuardrail):
model: Final = self.headroom_model or request_data.get("model")
flattened: Final = _flatten_messages_for_compression(compressible)
compressible_tokens: Final = (
await _estimate_compressible_tokens(model=model, messages=flattened) if self.min_tokens > 0 else None
await _estimate_compressible_tokens(model=model, messages=flattened) if self.min_tokens else None
)
from litellm.proxy.common_utils.callback_utils import (
add_guardrail_to_applied_guardrails_header,
)
if compressible_tokens is not None and compressible_tokens < self.min_tokens:
if self.min_tokens and compressible_tokens is not None and compressible_tokens < self.min_tokens:
verbose_proxy_logger.debug(
"Headroom: %s compressible tokens below min_tokens=%s; skipping compression",
compressible_tokens,

View file

@ -30,12 +30,12 @@ class HeadroomGuardrailConfigModel(GuardrailConfigModel[BaseModel]):
default=True,
description="Inject the Headroom retrieval tool for hashes declared by the compression service.",
)
min_tokens: int = Field(
default=0,
min_tokens: int | None = Field(
default=None,
ge=0,
description=(
"Skip the compression round trip when the compressible messages total fewer than this many tokens. "
"The default of 0 always calls the compression service."
"Falls back to the HEADROOM_MIN_TOKENS env var; when neither is set, every request is compressed."
),
)

View file

@ -1137,7 +1137,47 @@ def test_initialize_guardrail_passes_min_tokens(monkeypatch: pytest.MonkeyPatch)
)
assert configured_guardrail.min_tokens == 42
assert default_guardrail.min_tokens == 0
assert default_guardrail.min_tokens is None
def test_min_tokens_falls_back_to_env_var(monkeypatch: pytest.MonkeyPatch):
monkeypatch.delenv("HEADROOM_MIN_TOKENS", raising=False)
assert _make_guardrail(min_tokens=None).min_tokens is None
monkeypatch.setenv("HEADROOM_MIN_TOKENS", "1500")
assert _make_guardrail(min_tokens=None).min_tokens == 1500
assert _make_guardrail(min_tokens=7).min_tokens == 7
monkeypatch.setenv("HEADROOM_MIN_TOKENS", "")
assert _make_guardrail(min_tokens=None).min_tokens is None
monkeypatch.setenv("HEADROOM_MIN_TOKENS", "-5")
with pytest.raises(ValueError, match="HEADROOM_MIN_TOKENS"):
_make_guardrail(min_tokens=None)
monkeypatch.setenv("HEADROOM_MIN_TOKENS", "lots")
with pytest.raises(ValueError, match="HEADROOM_MIN_TOKENS"):
_make_guardrail(min_tokens=None)
@pytest.mark.asyncio
async def test_env_var_threshold_skips_compression(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("HEADROOM_MIN_TOKENS", "1000000")
guardrail = _make_guardrail(min_tokens=None)
inputs = GenericGuardrailAPIInputs(
texts=[str(m["content"]) for m in ORIGINAL_MESSAGES],
structured_messages=ORIGINAL_MESSAGES,
)
request_data = {"model": "gpt-4o"}
with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as post:
result = await guardrail.apply_guardrail(inputs=inputs, request_data=request_data, input_type="request")
assert result is inputs
post.assert_not_awaited()
response = _recorded_guardrail_response(request_data)
assert response["skipped"] == "below_min_tokens"
assert response["min_tokens"] == 1000000
@pytest.mark.asyncio

View file

@ -32642,10 +32642,9 @@ export interface components {
} | null;
/**
* Min Tokens
* @description Skip the compression round trip when the compressible messages total fewer than this many tokens. The default of 0 always calls the compression service.
* @default 0
* @description Skip the compression round trip when the compressible messages total fewer than this many tokens. Falls back to the HEADROOM_MIN_TOKENS env var; when neither is set, every request is compressed.
*/
min_tokens: number;
min_tokens?: number | null;
/**
* Mock Redacted Text
* @description Mock redacted text for testing