From 26c414f285a50b68abd9b40d715ca8ceccfc912d Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 27 Feb 2026 16:01:23 +0530 Subject: [PATCH] Update the docs --- .../index.md | 90 +++++++++++-------- docs/my-website/docs/response_api.md | 34 +++---- 2 files changed, 62 insertions(+), 62 deletions(-) diff --git a/docs/my-website/blog/responses_api_encrypted_content_incident/index.md b/docs/my-website/blog/responses_api_encrypted_content_incident/index.md index 8422f6d3cd0..f229f9567da 100644 --- a/docs/my-website/blog/responses_api_encrypted_content_incident/index.md +++ b/docs/my-website/blog/responses_api_encrypted_content_incident/index.md @@ -119,47 +119,59 @@ Implemented a new `encrypted_content_affinity` pre-call check that intelligently ### Implementation -**1. New `EncryptedContentAffinityCheck` Class** ([`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py)) +**1. Encoding `model_id` into output item IDs** ([`responses/utils.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/utils.py)) + +The same approach used for `previous_response_id` affinity — no cache needed. When a response contains output items with `encrypted_content`, LiteLLM rewrites their IDs to embed the originating deployment's `model_id`: + +```python +# On response: rs_abc123 → encitem_{base64("litellm:model_id:{model_id};item_id:rs_abc123")} +def _build_encrypted_item_id(model_id: str, item_id: str) -> str: + assembled = f"litellm:model_id:{model_id};item_id:{item_id}" + encoded = base64.b64encode(assembled.encode("utf-8")).decode("utf-8") + return f"encitem_{encoded}" + +# On request: decode encitem_... → extract model_id for routing +def _decode_encrypted_item_id(encoded_id: str) -> Optional[Dict[str, str]]: + if not encoded_id.startswith("encitem_"): + return None + cleaned = encoded_id[len("encitem_"):] + missing = len(cleaned) % 4 + if missing: + cleaned += "=" * (4 - missing) # restore padding stripped in transit + decoded = base64.b64decode(cleaned).decode("utf-8") + model_id, item_id = decoded.split(";", 1) + return {"model_id": model_id.replace("litellm:model_id:", ""), + "item_id": item_id.replace("item_id:", "")} +``` + +Before forwarding to the upstream provider, LiteLLM restores the original item IDs so the provider never sees the encoded form: + +```python +# In responses/main.py — before calling the handler +input = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(input) +``` + +**2. `EncryptedContentAffinityCheck` — routing only** ([`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py)) + +No `async_log_success_event` or cache lookups — the `model_id` is decoded directly from the item ID: ```python class EncryptedContentAffinityCheck(CustomLogger): - """ - Routes follow-up Responses API requests to the deployment that produced - the encrypted output items they reference. - """ - - async def async_log_success_event(self, kwargs, response_obj, ...): - """Track: Extract item IDs from response output, cache item_id → deployment_id""" - output = self._get_output_from_response(response_obj) - item_ids = self._extract_item_ids_from_output(output) - model_id = self._get_model_id_from_kwargs(kwargs) - - for item_id in item_ids: - await self.cache.async_set_cache( - f"encrypted_content_affinity:v1:{item_id}", - model_id, - ttl=86400, # 24 hours - ) - async def async_filter_deployments(self, model, healthy_deployments, ...): - """Route: Check if input contains tracked items, pin to originating deployment""" - input_item_ids = self._extract_item_ids_from_input(request_kwargs.get("input")) - - for item_id in input_item_ids: - cached_model_id = await self.cache.async_get_cache(f"...:{item_id}") - if cached_model_id: + """Decode encitem_ IDs in input to extract model_id and pin to that deployment.""" + for item in request_kwargs.get("input", []): + decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(item.get("id", "")) + if decoded: deployment = self._find_deployment_by_model_id( - healthy_deployments, cached_model_id + healthy_deployments, decoded["model_id"] ) if deployment: - # Signal to bypass rate limits (encrypted content must go here) request_kwargs["_encrypted_content_affinity_pinned"] = True return [deployment] - - return healthy_deployments # Normal load balancing + return healthy_deployments ``` -**2. Rate Limit Bypass** ([`router.py#L8656-L8660`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py#L8656-L8660)) +**3. Rate Limit Bypass** ([`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py)) When encrypted content requires a specific deployment, RPM/TPM limits are bypassed (the request would fail on any other deployment anyway): @@ -185,9 +197,10 @@ router_settings: ### Key Benefits -✅ **No quota reduction**: Only pins requests containing tracked encrypted items +✅ **No quota reduction**: Only pins requests containing encrypted items ✅ **Bypasses rate limits**: When encrypted content requires a specific deployment, RPM/TPM limits don't block it -✅ **No `previous_response_id` required**: Works by tracking item IDs in response output +✅ **No `previous_response_id` required**: Works by encoding `model_id` directly into the item ID +✅ **No cache required**: `model_id` is decoded on-the-fly from the item ID — no Redis, no TTL ✅ **Globally safe**: Can be enabled for all models; non-Responses-API calls are unaffected ✅ **Surgical precision**: Normal requests continue to load balance freely @@ -197,12 +210,13 @@ router_settings: | # | Action | Status | Code | |---|---|---|---| -| 1 | Create `EncryptedContentAffinityCheck` class with tracking and routing logic | ✅ Done | [`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py) | -| 2 | Add `encrypted_content_affinity` to `OptionalPreCallChecks` type | ✅ Done | [`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/types/router.py) | -| 3 | Wire up check in `Router.add_optional_pre_call_checks` | ✅ Done | [`router.py#L8656-L8660`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py#L8656-L8660) | -| 4 | Implement rate limit bypass for affinity-pinned requests | ✅ Done | [`router.py#L8656-L8660`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py#L8656-L8660) | -| 5 | Unit tests: tracking, routing, no-op for non-Responses-API, RPM bypass | ✅ Done | [`test_encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py) | -| 6 | Documentation: Responses API guide, load balancing guide, config reference | ✅ Done | [Docs](https://docs.litellm.ai/docs/response_api#encrypted-content-affinity-multi-region-load-balancing) | +| 1 | Encode `model_id` into encrypted-content item IDs on response | ✅ Done | [`responses/utils.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/utils.py) | +| 2 | Restore original item IDs before forwarding to upstream provider | ✅ Done | [`responses/main.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/main.py) | +| 3 | `EncryptedContentAffinityCheck`: decode item IDs to route (no cache) | ✅ Done | [`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py) | +| 4 | Add `encrypted_content_affinity` to `OptionalPreCallChecks` type | ✅ Done | [`types/router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/types/router.py) | +| 5 | Implement rate limit bypass for affinity-pinned requests | ✅ Done | [`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py) | +| 6 | Unit tests: encoding/decoding utilities, routing, RPM bypass | ✅ Done | [`test_encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py) | +| 7 | Documentation: Responses API guide, load balancing guide, config reference | ✅ Done | [Docs](https://docs.litellm.ai/docs/response_api#encrypted-content-affinity-multi-region-load-balancing) | --- diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md index 85144a80245..a7cf61ef16a 100644 --- a/docs/my-website/docs/response_api.md +++ b/docs/my-website/docs/response_api.md @@ -1011,24 +1011,25 @@ This error occurs when: ### The Solution: `encrypted_content_affinity` -The `encrypted_content_affinity` pre-call check intelligently tracks encrypted content and routes follow-up requests to the originating deployment **only when necessary**. +The `encrypted_content_affinity` pre-call check routes follow-up requests containing encrypted items to the originating deployment **only when necessary** **Key Benefits:** -- ✅ **No quota reduction**: Unlike `deployment_affinity`, only pins requests that contain tracked encrypted items +- ✅ **No quota reduction**: Unlike `deployment_affinity`, only pins requests that contain encrypted items - ✅ **Bypasses rate limits**: When encrypted content requires a specific deployment, RPM/TPM limits are bypassed (the request would fail on any other deployment anyway) -- ✅ **No `previous_response_id` required**: Works by tracking item IDs in response output and matching them in request input +- ✅ **No `previous_response_id` required**: Works by encoding `model_id` directly into item IDs +- ✅ **No cache required**: `model_id` is decoded on-the-fly — no Redis dependency, no TTL to manage - ✅ **Globally safe**: Can be enabled for all models; non-Responses-API calls (chat, embeddings) are unaffected ### How It Works -1. **Tracking Phase** (after successful response): - - Extracts all item IDs from response `output` (e.g., `msg_abc`, `rs_xyz`) - - Caches mapping: `item_id` → `deployment_id` (default TTL: 24 hours) +1. **Encoding Phase** (on response): + - For each output item that contains `encrypted_content`, LiteLLM rewrites the item ID to embed the originating `model_id`: `rs_xyz` → `encitem_{base64("litellm:model_id:{model_id};item_id:rs_xyz")}` + - The original item ID is restored before forwarding the request to the upstream provider 2. **Routing Phase** (before request): - - Scans request `input` for item IDs - - If tracked item found → pins to originating deployment, bypasses rate limits - - If no tracked items → normal load balancing + - Scans request `input` for `encitem_` prefixed IDs + - If found → decodes `model_id`, pins to originating deployment, bypasses rate limits + - If no encoded items → normal load balancing ### Configuration @@ -1058,7 +1059,6 @@ router = Router( }, ], optional_pre_call_checks=["encrypted_content_affinity"], - deployment_affinity_ttl_seconds=86400, # 24 hours (default) ) # Initial request - routes to any deployment @@ -1104,7 +1104,6 @@ router_settings: enable_pre_call_checks: true optional_pre_call_checks: - encrypted_content_affinity - deployment_affinity_ttl_seconds: 86400 # Optional, default is 86400 (24 hours) ``` **Start proxy:** @@ -1124,19 +1123,6 @@ litellm --config config.yaml | `session_affinity` | Session-based applications | All requests with same `session_id` | ⚠️ Reduces quota by # of sessions | | `deployment_affinity` | Simple sticky sessions | All requests from same API key | ❌ Reduces quota by # of users | -### Multi-Instance Deployment (Redis) - -For multiple LiteLLM proxy instances, use Redis to share affinity state: - -```yaml -router_settings: - optional_pre_call_checks: - - encrypted_content_affinity - redis_host: redis.example.com - redis_port: 6379 - redis_password: your-password -``` - ## Calling non-Responses API endpoints (`/responses` to `/chat/completions` Bridge)