fix(lens): keep worker settings inside the standalone worker package

litellm/constants.py

@@ -494,12 +494,6 @@ DEFAULT_MAX_LRU_CACHE_SIZE: Final = int(os.getenv("DEFAULT_MAX_LRU_CACHE_SIZE",
 _REALTIME_BODY_CACHE_SIZE = 1000  # Keep realtime helper caches bounded; workloads rarely exceed 1k models/intents
 INITIAL_RETRY_DELAY: Final = float(os.getenv("INITIAL_RETRY_DELAY", 0.5))
 MAX_RETRY_DELAY: Final = float(os.getenv("MAX_RETRY_DELAY", 8.0))
-LENS_UPDATE_ATTEMPTS: Final = get_env_int("LENS_UPDATE_ATTEMPTS", 40)
-LENS_UPDATE_BACKOFF_SECONDS: Final = float(os.getenv("LENS_UPDATE_BACKOFF_SECONDS", 0.02))
-LENS_MODEL_RETRIES: Final = get_env_int("LENS_MODEL_RETRIES", 4)
-LENS_MODEL_RETRY_MAX_SECONDS: Final = float(os.getenv("LENS_MODEL_RETRY_MAX_SECONDS", 60))
-LENS_WORKER_SLOTS: Final = get_env_int("LENS_WORKER_SLOTS", 3)
-LENS_WORKER_POLL_SECONDS: Final = float(os.getenv("LENS_WORKER_POLL_SECONDS", 2))
 JITTER: Final = float(os.getenv("JITTER", 0.75))
 DEFAULT_IN_MEMORY_TTL = int(os.getenv("DEFAULT_IN_MEMORY_TTL", 5))  # default time to live for the in-memory cache
 DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE: Final = int(

litellm/proxy/lens/worker.py

@@ -9,13 +9,6 @@ from typing import Final
 import httpx
 from pydantic import BaseModel, ConfigDict, ValidationError
 
-from litellm.constants import (
-    LENS_MODEL_RETRIES,
-    LENS_MODEL_RETRY_MAX_SECONDS,
-    LENS_WORKER_POLL_SECONDS,
-    LENS_WORKER_SLOTS,
-)
-
 from .analysis import AnalysisResponseError, analyze_sample, validation_details
 from .models import (
     Claim,
@@ -32,6 +25,10 @@ from .models import (
 from .release import PROTOCOL_VERSION, release_tag
 
 logger: Final = logging.getLogger("litellm.lens.worker")
+MODEL_RETRIES: Final = 4
+MODEL_RETRY_MAX_SECONDS: Final = 60.0
+SLOTS: Final = 3
+POLL_SECONDS: Final = 2.0
 
 
 class ClaimedJobIdentity(BaseModel):
@@ -56,12 +53,12 @@ class ModelErrorEnvelope(BaseModel):
 
 
 def retry_delay(error: httpx.TransportError | httpx.HTTPStatusError, attempt: int) -> float:
-    backoff: Final = float(min(2**attempt, LENS_MODEL_RETRY_MAX_SECONDS))
+    backoff: Final = float(min(2**attempt, MODEL_RETRY_MAX_SECONDS))
     if not isinstance(error, httpx.HTTPStatusError):
         return backoff
     requested: Final = error.response.headers.get("retry-after", "")
     try:
-        return min(max(float(requested), backoff), LENS_MODEL_RETRY_MAX_SECONDS)
+        return min(max(float(requested), backoff), MODEL_RETRY_MAX_SECONDS)
     except ValueError:
         return backoff
 
@@ -145,7 +142,7 @@ class LensWorker:
                 503,
                 504,
             )
-            if not retryable or attempt >= LENS_MODEL_RETRIES:
+            if not retryable or attempt >= MODEL_RETRIES:
                 raise
             await self.sleep(retry_delay(exc, attempt))
             return await self.model_request(path, body, attempt + 1)
@@ -270,7 +267,7 @@ async def main() -> None:
     async with httpx.AsyncClient(
         base_url=url, headers=MappingProxyType({"Authorization": f"Bearer {token}"}), timeout=180
     ) as client:
-        await LensWorker(client).serve(LENS_WORKER_SLOTS, LENS_WORKER_POLL_SECONDS)
+        await LensWorker(client).serve(SLOTS, POLL_SECONDS)
 
 
 if __name__ == "__main__":

tests/unit/proxy/lens/test_worker.py

@@ -6,7 +6,6 @@ import httpx
 import pytest
 from pydantic import ValidationError
 
-from litellm.constants import LENS_MODEL_RETRIES, LENS_MODEL_RETRY_MAX_SECONDS
 from litellm.proxy.lens.models import (
     Claim,
     Execution,
@@ -19,7 +18,13 @@ from litellm.proxy.lens.models import (
     TracePart,
 )
 from litellm.proxy.lens.state import queue_job
-from litellm.proxy.lens.worker import LensWorker, failure_message, retry_delay
+from litellm.proxy.lens.worker import (
+    MODEL_RETRIES,
+    MODEL_RETRY_MAX_SECONDS,
+    LensWorker,
+    failure_message,
+    retry_delay,
+)
 from tests.unit.proxy.lens.test_state import NOW, lens
 
 
@@ -71,9 +76,9 @@ async def test_transient_retries_are_bounded() -> None:
             await LensWorker(client, sleep=sleep).model_request(
                 "/model", ModelRequest(purpose="extract", prompt="review")
             )
-    assert attempts.qsize() == LENS_MODEL_RETRIES + 1
+    assert attempts.qsize() == MODEL_RETRIES + 1
     assert tuple(delays.get_nowait() for _ in range(delays.qsize())) == tuple(
-        float(min(2**n, LENS_MODEL_RETRY_MAX_SECONDS)) for n in range(LENS_MODEL_RETRIES)
+        float(min(2**n, MODEL_RETRY_MAX_SECONDS)) for n in range(MODEL_RETRIES)
     )
 
 
@@ -102,7 +107,7 @@ async def test_rate_limited_model_waits_as_long_as_the_provider_asks_then_comple
 
 @pytest.mark.parametrize(
     ("retry_after", "attempt", "expected"),
-    (("", 1, 2), ("5", 0, 5), ("1", 3, 8), ("9999", 0, LENS_MODEL_RETRY_MAX_SECONDS), ("soon", 2, 4)),
+    (("", 1, 2), ("5", 0, 5), ("1", 3, 8), ("9999", 0, MODEL_RETRY_MAX_SECONDS), ("soon", 2, 4)),
 )
 def test_retry_delay_prefers_the_providers_wait_within_bounds(retry_after: str, attempt: int, expected: float) -> None:
     request: Final = httpx.Request("POST", "https://proxy.test/model")