mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
feat(headroom): make min_tokens opt-in with a default of 0 instead of a hardcoded 1000
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
a7ea76f9d3
commit
f9b2a8540e
5 changed files with 9 additions and 11 deletions
|
|
@ -12077,8 +12077,8 @@
|
|||
"title": "Metadata"
|
||||
},
|
||||
"min_tokens": {
|
||||
"default": 1000,
|
||||
"description": "Skip the compression round trip when the compressible messages total fewer than this many tokens. 0 always calls the compression service.",
|
||||
"default": 0,
|
||||
"description": "Skip the compression round trip when the compressible messages total fewer than this many tokens. The default of 0 always calls the compression service.",
|
||||
"minimum": 0.0,
|
||||
"title": "Min Tokens",
|
||||
"type": "integer"
|
||||
|
|
|
|||
|
|
@ -56,7 +56,6 @@ if TYPE_CHECKING:
|
|||
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
|
||||
|
||||
BYPASS_HEADER: Final = "x-headroom-bypass"
|
||||
DEFAULT_MIN_TOKENS: Final = 1000
|
||||
_STREAM_CONVERTIBLE_CALL_TYPES: Final = frozenset(
|
||||
(CallTypes.completion, CallTypes.acompletion, CallTypes.responses, CallTypes.aresponses)
|
||||
)
|
||||
|
|
@ -515,7 +514,7 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
unreachable_fallback: str | None = None,
|
||||
timeout: float | None = None,
|
||||
ccr_retrieval: bool = True,
|
||||
min_tokens: int | None = None,
|
||||
min_tokens: int = 0,
|
||||
):
|
||||
self.headroom_api_base = (api_base or get_secret_str("HEADROOM_API_BASE") or "").rstrip("/")
|
||||
if not self.headroom_api_base:
|
||||
|
|
@ -530,7 +529,7 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
)
|
||||
self.timeout: httpx.Timeout = self._resolve_timeout(timeout)
|
||||
self.ccr_retrieval = ccr_retrieval
|
||||
self.min_tokens: int = DEFAULT_MIN_TOKENS if min_tokens is None else min_tokens
|
||||
self.min_tokens = min_tokens
|
||||
self.async_handler = get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.GuardrailCallback,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -31,11 +31,11 @@ class HeadroomGuardrailConfigModel(GuardrailConfigModel[BaseModel]):
|
|||
description="Inject the Headroom retrieval tool for hashes declared by the compression service.",
|
||||
)
|
||||
min_tokens: int = Field(
|
||||
default=1000,
|
||||
default=0,
|
||||
ge=0,
|
||||
description=(
|
||||
"Skip the compression round trip when the compressible messages total fewer than this many tokens. "
|
||||
"0 always calls the compression service."
|
||||
"The default of 0 always calls the compression service."
|
||||
),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ from fastapi import HTTPException
|
|||
|
||||
import litellm
|
||||
from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import (
|
||||
DEFAULT_MIN_TOKENS,
|
||||
HEADROOM_RETRIEVE_TOOL_NAME,
|
||||
HeadroomGuardrail,
|
||||
has_headroom_retrieve_tool,
|
||||
|
|
@ -1138,7 +1137,7 @@ def test_initialize_guardrail_passes_min_tokens(monkeypatch: pytest.MonkeyPatch)
|
|||
)
|
||||
|
||||
assert configured_guardrail.min_tokens == 42
|
||||
assert default_guardrail.min_tokens == DEFAULT_MIN_TOKENS
|
||||
assert default_guardrail.min_tokens == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -32341,8 +32341,8 @@ export interface components {
|
|||
} | null;
|
||||
/**
|
||||
* Min Tokens
|
||||
* @description Skip the compression round trip when the compressible messages total fewer than this many tokens. 0 always calls the compression service.
|
||||
* @default 1000
|
||||
* @description Skip the compression round trip when the compressible messages total fewer than this many tokens. The default of 0 always calls the compression service.
|
||||
* @default 0
|
||||
*/
|
||||
min_tokens: number;
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue