diff --git a/litellm/constants.py b/litellm/constants.py index 3b860af6a9a..49514fc4d0e 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -494,12 +494,6 @@ DEFAULT_MAX_LRU_CACHE_SIZE: Final = int(os.getenv("DEFAULT_MAX_LRU_CACHE_SIZE", _REALTIME_BODY_CACHE_SIZE = 1000 # Keep realtime helper caches bounded; workloads rarely exceed 1k models/intents INITIAL_RETRY_DELAY: Final = float(os.getenv("INITIAL_RETRY_DELAY", 0.5)) MAX_RETRY_DELAY: Final = float(os.getenv("MAX_RETRY_DELAY", 8.0)) -LENS_UPDATE_ATTEMPTS: Final = get_env_int("LENS_UPDATE_ATTEMPTS", 40) -LENS_UPDATE_BACKOFF_SECONDS: Final = float(os.getenv("LENS_UPDATE_BACKOFF_SECONDS", 0.02)) -LENS_MODEL_RETRIES: Final = get_env_int("LENS_MODEL_RETRIES", 4) -LENS_MODEL_RETRY_MAX_SECONDS: Final = float(os.getenv("LENS_MODEL_RETRY_MAX_SECONDS", 60)) -LENS_WORKER_SLOTS: Final = get_env_int("LENS_WORKER_SLOTS", 3) -LENS_WORKER_POLL_SECONDS: Final = float(os.getenv("LENS_WORKER_POLL_SECONDS", 2)) JITTER: Final = float(os.getenv("JITTER", 0.75)) DEFAULT_IN_MEMORY_TTL = int(os.getenv("DEFAULT_IN_MEMORY_TTL", 5)) # default time to live for the in-memory cache DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE: Final = int( diff --git a/litellm/proxy/lens/worker.py b/litellm/proxy/lens/worker.py index 92b6c63284c..baeac6c096e 100644 --- a/litellm/proxy/lens/worker.py +++ b/litellm/proxy/lens/worker.py @@ -9,13 +9,6 @@ from typing import Final import httpx from pydantic import BaseModel, ConfigDict, ValidationError -from litellm.constants import ( - LENS_MODEL_RETRIES, - LENS_MODEL_RETRY_MAX_SECONDS, - LENS_WORKER_POLL_SECONDS, - LENS_WORKER_SLOTS, -) - from .analysis import AnalysisResponseError, analyze_sample, validation_details from .models import ( Claim, @@ -32,6 +25,10 @@ from .models import ( from .release import PROTOCOL_VERSION, release_tag logger: Final = logging.getLogger("litellm.lens.worker") +MODEL_RETRIES: Final = 4 +MODEL_RETRY_MAX_SECONDS: Final = 60.0 +SLOTS: Final = 3 +POLL_SECONDS: Final = 2.0 class ClaimedJobIdentity(BaseModel): @@ -56,12 +53,12 @@ class ModelErrorEnvelope(BaseModel): def retry_delay(error: httpx.TransportError | httpx.HTTPStatusError, attempt: int) -> float: - backoff: Final = float(min(2**attempt, LENS_MODEL_RETRY_MAX_SECONDS)) + backoff: Final = float(min(2**attempt, MODEL_RETRY_MAX_SECONDS)) if not isinstance(error, httpx.HTTPStatusError): return backoff requested: Final = error.response.headers.get("retry-after", "") try: - return min(max(float(requested), backoff), LENS_MODEL_RETRY_MAX_SECONDS) + return min(max(float(requested), backoff), MODEL_RETRY_MAX_SECONDS) except ValueError: return backoff @@ -145,7 +142,7 @@ class LensWorker: 503, 504, ) - if not retryable or attempt >= LENS_MODEL_RETRIES: + if not retryable or attempt >= MODEL_RETRIES: raise await self.sleep(retry_delay(exc, attempt)) return await self.model_request(path, body, attempt + 1) @@ -270,7 +267,7 @@ async def main() -> None: async with httpx.AsyncClient( base_url=url, headers=MappingProxyType({"Authorization": f"Bearer {token}"}), timeout=180 ) as client: - await LensWorker(client).serve(LENS_WORKER_SLOTS, LENS_WORKER_POLL_SECONDS) + await LensWorker(client).serve(SLOTS, POLL_SECONDS) if __name__ == "__main__": diff --git a/tests/unit/proxy/lens/test_worker.py b/tests/unit/proxy/lens/test_worker.py index acf435264cc..4d656da1956 100644 --- a/tests/unit/proxy/lens/test_worker.py +++ b/tests/unit/proxy/lens/test_worker.py @@ -6,7 +6,6 @@ import httpx import pytest from pydantic import ValidationError -from litellm.constants import LENS_MODEL_RETRIES, LENS_MODEL_RETRY_MAX_SECONDS from litellm.proxy.lens.models import ( Claim, Execution, @@ -19,7 +18,13 @@ from litellm.proxy.lens.models import ( TracePart, ) from litellm.proxy.lens.state import queue_job -from litellm.proxy.lens.worker import LensWorker, failure_message, retry_delay +from litellm.proxy.lens.worker import ( + MODEL_RETRIES, + MODEL_RETRY_MAX_SECONDS, + LensWorker, + failure_message, + retry_delay, +) from tests.unit.proxy.lens.test_state import NOW, lens @@ -71,9 +76,9 @@ async def test_transient_retries_are_bounded() -> None: await LensWorker(client, sleep=sleep).model_request( "/model", ModelRequest(purpose="extract", prompt="review") ) - assert attempts.qsize() == LENS_MODEL_RETRIES + 1 + assert attempts.qsize() == MODEL_RETRIES + 1 assert tuple(delays.get_nowait() for _ in range(delays.qsize())) == tuple( - float(min(2**n, LENS_MODEL_RETRY_MAX_SECONDS)) for n in range(LENS_MODEL_RETRIES) + float(min(2**n, MODEL_RETRY_MAX_SECONDS)) for n in range(MODEL_RETRIES) ) @@ -102,7 +107,7 @@ async def test_rate_limited_model_waits_as_long_as_the_provider_asks_then_comple @pytest.mark.parametrize( ("retry_after", "attempt", "expected"), - (("", 1, 2), ("5", 0, 5), ("1", 3, 8), ("9999", 0, LENS_MODEL_RETRY_MAX_SECONDS), ("soon", 2, 4)), + (("", 1, 2), ("5", 0, 5), ("1", 3, 8), ("9999", 0, MODEL_RETRY_MAX_SECONDS), ("soon", 2, 4)), ) def test_retry_delay_prefers_the_providers_wait_within_bounds(retry_after: str, attempt: int, expected: float) -> None: request: Final = httpx.Request("POST", "https://proxy.test/model")