mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
feat(headroom): read min_tokens from HEADROOM_MIN_TOKENS when the config omits it
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
f622f69842
commit
27f693348c
5 changed files with 78 additions and 16 deletions
|
|
@ -12119,11 +12119,17 @@
|
|||
"title": "Metadata"
|
||||
},
|
||||
"min_tokens": {
|
||||
"default": 0,
|
||||
"description": "Skip the compression round trip when the compressible messages total fewer than this many tokens. The default of 0 always calls the compression service.",
|
||||
"minimum": 0.0,
|
||||
"title": "Min Tokens",
|
||||
"type": "integer"
|
||||
"anyOf": [
|
||||
{
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Skip the compression round trip when the compressible messages total fewer than this many tokens. Falls back to the HEADROOM_MIN_TOKENS env var; when neither is set, every request is compressed.",
|
||||
"title": "Min Tokens"
|
||||
},
|
||||
"mock_redacted_text": {
|
||||
"anyOf": [
|
||||
|
|
|
|||
|
|
@ -70,6 +70,23 @@ _HASH_CACHE_TTL_SECONDS: Final = 15 * 60
|
|||
# untranslated messages can be read with concrete types (values pass through by
|
||||
# reference, so this is a shallow top-level reconstruction).
|
||||
_REQUEST_DATA_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
MIN_TOKENS_ENV_VAR: Final = "HEADROOM_MIN_TOKENS"
|
||||
|
||||
|
||||
def _resolve_min_tokens(configured: int | None) -> int | None:
|
||||
"""Config wins over the env var; neither set means every request is compressed."""
|
||||
if configured is not None:
|
||||
return configured
|
||||
raw: Final = get_secret_str(MIN_TOKENS_ENV_VAR)
|
||||
if raw is None or not raw.strip():
|
||||
return None
|
||||
try:
|
||||
value: Final = int(raw)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"{MIN_TOKENS_ENV_VAR} must be a non-negative integer, got {raw!r}") from exc
|
||||
if value < 0:
|
||||
raise ValueError(f"{MIN_TOKENS_ENV_VAR} must be a non-negative integer, got {raw!r}")
|
||||
return value
|
||||
|
||||
|
||||
def _is_str_object_dict(value: object) -> TypeGuard[dict[str, object]]: # guard-ok: isinstance narrows correctly; predicate is trivially correct # fmt: skip
|
||||
|
|
@ -515,7 +532,7 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
unreachable_fallback: str | None = None,
|
||||
timeout: float | None = None,
|
||||
ccr_retrieval: bool = True,
|
||||
min_tokens: int = 0,
|
||||
min_tokens: int | None = None,
|
||||
):
|
||||
self.headroom_api_base = (api_base or get_secret_str("HEADROOM_API_BASE") or "").rstrip("/")
|
||||
if not self.headroom_api_base:
|
||||
|
|
@ -530,7 +547,7 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
)
|
||||
self.timeout: httpx.Timeout = self._resolve_timeout(timeout)
|
||||
self.ccr_retrieval = ccr_retrieval
|
||||
self.min_tokens = min_tokens
|
||||
self.min_tokens: int | None = _resolve_min_tokens(min_tokens)
|
||||
self.async_handler = get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.GuardrailCallback,
|
||||
)
|
||||
|
|
@ -833,13 +850,13 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
model: Final = self.headroom_model or request_data.get("model")
|
||||
flattened: Final = _flatten_messages_for_compression(compressible)
|
||||
compressible_tokens: Final = (
|
||||
await _estimate_compressible_tokens(model=model, messages=flattened) if self.min_tokens > 0 else None
|
||||
await _estimate_compressible_tokens(model=model, messages=flattened) if self.min_tokens else None
|
||||
)
|
||||
from litellm.proxy.common_utils.callback_utils import (
|
||||
add_guardrail_to_applied_guardrails_header,
|
||||
)
|
||||
|
||||
if compressible_tokens is not None and compressible_tokens < self.min_tokens:
|
||||
if self.min_tokens and compressible_tokens is not None and compressible_tokens < self.min_tokens:
|
||||
verbose_proxy_logger.debug(
|
||||
"Headroom: %s compressible tokens below min_tokens=%s; skipping compression",
|
||||
compressible_tokens,
|
||||
|
|
|
|||
|
|
@ -30,12 +30,12 @@ class HeadroomGuardrailConfigModel(GuardrailConfigModel[BaseModel]):
|
|||
default=True,
|
||||
description="Inject the Headroom retrieval tool for hashes declared by the compression service.",
|
||||
)
|
||||
min_tokens: int = Field(
|
||||
default=0,
|
||||
min_tokens: int | None = Field(
|
||||
default=None,
|
||||
ge=0,
|
||||
description=(
|
||||
"Skip the compression round trip when the compressible messages total fewer than this many tokens. "
|
||||
"The default of 0 always calls the compression service."
|
||||
"Falls back to the HEADROOM_MIN_TOKENS env var; when neither is set, every request is compressed."
|
||||
),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1137,7 +1137,47 @@ def test_initialize_guardrail_passes_min_tokens(monkeypatch: pytest.MonkeyPatch)
|
|||
)
|
||||
|
||||
assert configured_guardrail.min_tokens == 42
|
||||
assert default_guardrail.min_tokens == 0
|
||||
assert default_guardrail.min_tokens is None
|
||||
|
||||
|
||||
def test_min_tokens_falls_back_to_env_var(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.delenv("HEADROOM_MIN_TOKENS", raising=False)
|
||||
assert _make_guardrail(min_tokens=None).min_tokens is None
|
||||
|
||||
monkeypatch.setenv("HEADROOM_MIN_TOKENS", "1500")
|
||||
assert _make_guardrail(min_tokens=None).min_tokens == 1500
|
||||
assert _make_guardrail(min_tokens=7).min_tokens == 7
|
||||
|
||||
monkeypatch.setenv("HEADROOM_MIN_TOKENS", "")
|
||||
assert _make_guardrail(min_tokens=None).min_tokens is None
|
||||
|
||||
monkeypatch.setenv("HEADROOM_MIN_TOKENS", "-5")
|
||||
with pytest.raises(ValueError, match="HEADROOM_MIN_TOKENS"):
|
||||
_make_guardrail(min_tokens=None)
|
||||
|
||||
monkeypatch.setenv("HEADROOM_MIN_TOKENS", "lots")
|
||||
with pytest.raises(ValueError, match="HEADROOM_MIN_TOKENS"):
|
||||
_make_guardrail(min_tokens=None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_var_threshold_skips_compression(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("HEADROOM_MIN_TOKENS", "1000000")
|
||||
guardrail = _make_guardrail(min_tokens=None)
|
||||
inputs = GenericGuardrailAPIInputs(
|
||||
texts=[str(m["content"]) for m in ORIGINAL_MESSAGES],
|
||||
structured_messages=ORIGINAL_MESSAGES,
|
||||
)
|
||||
request_data = {"model": "gpt-4o"}
|
||||
|
||||
with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as post:
|
||||
result = await guardrail.apply_guardrail(inputs=inputs, request_data=request_data, input_type="request")
|
||||
|
||||
assert result is inputs
|
||||
post.assert_not_awaited()
|
||||
response = _recorded_guardrail_response(request_data)
|
||||
assert response["skipped"] == "below_min_tokens"
|
||||
assert response["min_tokens"] == 1000000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -32642,10 +32642,9 @@ export interface components {
|
|||
} | null;
|
||||
/**
|
||||
* Min Tokens
|
||||
* @description Skip the compression round trip when the compressible messages total fewer than this many tokens. The default of 0 always calls the compression service.
|
||||
* @default 0
|
||||
* @description Skip the compression round trip when the compressible messages total fewer than this many tokens. Falls back to the HEADROOM_MIN_TOKENS env var; when neither is set, every request is compressed.
|
||||
*/
|
||||
min_tokens: number;
|
||||
min_tokens?: number | null;
|
||||
/**
|
||||
* Mock Redacted Text
|
||||
* @description Mock redacted text for testing
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue