fix(proxy): keep polling working on a Redis that refuses scripts, keep the cancel ttl, and sanitize polling ids in logs

A Redis whose ACL denies scripting answered every polling state write with
an error, so background responses stayed queued and a cancel never stopped
the call. The write now falls back to GETRANGE then SET with the same
cancelled and missing checks when Redis refuses the script, and the
background task's cancel check is a plain GETRANGE.

The cancel route now writes with the configured polling ttl instead of the
1 hour default, and the polling id is sanitized in the state write logs.
This commit is contained in:
mateo-berri 2026-10-31 01:40:36 -07:00
parent 7b3e1f7bbf
commit 2f4c807e58
6 changed files with 200 additions and 25 deletions

View file

@ -91,6 +91,8 @@ class _AsyncRedisCommands(Protocol):
def lpop(self, name: str, count: int | None = None) -> Awaitable[object]: ...
def getrange(self, key: str, start: int, end: int) -> Awaitable[object]: ...
def pipeline(self, transaction: bool = True) -> "Pipeline[bytes]": ...
def eval(self, script: str, numkeys: int, *keys_and_args: str | bytes | float) -> Awaitable[object]: ...
@ -141,6 +143,7 @@ _INCREMENT_IF_PRESENT_LUA: Final = (
_LUA_COUNT: Final = TypeAdapter(int)
_LUA_OPTIONAL_TOTAL: Final[TypeAdapter[float | None]] = TypeAdapter(float | None)
_OPTIONAL_COUNTS: Final = TypeAdapter(tuple[int | None, ...])
_GETRANGE_REPLY: Final[TypeAdapter[bytes | str]] = TypeAdapter(bytes | str)
def _decoded_counts(values: Sequence[bytes | str | None]) -> tuple[int | None, ...]:
@ -1737,6 +1740,13 @@ class RedisCache(BaseCache):
_record_swallowed_redis_failure(self._circuit_breaker, e)
return key_value_dict
@_redis_circuit_breaker_guard
async def async_getrange(self, key: str, start: int, end: int) -> str:
value: Final = _GETRANGE_REPLY.validate_python(
await self._async_commands().getrange(self.check_and_fix_namespace(key=key), start, end)
)
return value.decode("utf-8", errors="replace") if isinstance(value, bytes) else value
@_redis_circuit_breaker_guard
async def async_get_cache(
self, key, parent_otel_span: Span | None = None, *, raise_on_error: bool = False, **kwargs

View file

@ -1790,6 +1790,7 @@ async def cancel_response(
from litellm.proxy.proxy_server import (
general_settings,
llm_router,
polling_cache_ttl,
proxy_config,
proxy_logging_obj,
read_request_body,
@ -1810,7 +1811,7 @@ async def cancel_response(
if not redis_usage_cache:
raise HTTPException(status_code=500, detail="Redis cache not configured.")
polling_handler: Final = ResponsePollingHandler(redis_cache=redis_usage_cache)
polling_handler: Final = ResponsePollingHandler(redis_cache=redis_usage_cache, ttl=polling_cache_ttl)
# Get current state to verify it exists
state: Final = await polling_handler.get_state(response_id)

View file

@ -8,11 +8,13 @@ from datetime import datetime, timezone
from typing import Final
from pydantic import TypeAdapter
from redis.exceptions import ResponseError
from litellm._internal_context import with_service_target
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid4
from litellm.caching.redis_cache import RedisCache
from litellm.proxy.litellm_pre_call_utils import sanitize_for_log
from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStatus
_RESPONSE_POLLING_TARGET: Final = "response_polling"
@ -39,13 +41,9 @@ redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2])
return 1
"""
_CANCELLED_OR_GONE_SCRIPT: Final = """
local head = redis.call('GETRANGE', KEYS[1], 0, string.len(ARGV[1]) - 1)
if head == '' or head == ARGV[1] then
return 1
end
return 0
"""
async def _stored_status_prefix(redis_cache: RedisCache, cache_key: str) -> str:
return await redis_cache.async_getrange(cache_key, 0, len(_CANCELLED_PREFIX) - 1)
class ResponsePollingHandler:
@ -181,7 +179,7 @@ class ResponsePollingHandler:
# Get current state
cached_state: Final = await self.redis_cache.async_get_cache(cache_key)
if not cached_state:
verbose_proxy_logger.warning("No cached state found for polling_id: %s", polling_id)
verbose_proxy_logger.warning("No cached state found for polling_id: %s", sanitize_for_log(polling_id))
return
# Parse existing ResponsesAPIResponse from cache
@ -240,20 +238,17 @@ class ResponsePollingHandler:
if store is not None:
state["store"] = store
write_unless_cancelled: Final = self.redis_cache.async_register_script(_WRITE_UNLESS_CANCELLED_SCRIPT)
try:
written: Final = _SCRIPT_REPLY.validate_python(
await write_unless_cancelled(
keys=(cache_key,),
args=(_encoded_state(state), self.redis_cache.get_ttl(ttl=self.ttl), _CANCELLED_PREFIX),
)
)
written: Final = await self._write_unless_cancelled(self.redis_cache, cache_key, state)
except Exception as write_error: # noqa: BLE001 # a Redis fault must not end the background stream
verbose_proxy_logger.error("Failed to write the polling state for %s: %s", polling_id, write_error)
verbose_proxy_logger.error(
"Failed to write the polling state for %s: %s", sanitize_for_log(polling_id), write_error
)
return
if written != _WRITTEN:
verbose_proxy_logger.debug(
"Skipped the polling state write for %s: the response was cancelled or has expired", polling_id
"Skipped the polling state write for %s: the response was cancelled or has expired",
sanitize_for_log(polling_id),
)
return
@ -276,13 +271,31 @@ class ResponsePollingHandler:
return None
async def _write_unless_cancelled(
self, redis_cache: RedisCache, cache_key: str, state: Mapping[str, object]
) -> int:
write_unless_cancelled: Final = redis_cache.async_register_script(_WRITE_UNLESS_CANCELLED_SCRIPT)
try:
return _SCRIPT_REPLY.validate_python(
await write_unless_cancelled(
keys=(cache_key,),
args=(_encoded_state(state), redis_cache.get_ttl(ttl=self.ttl), _CANCELLED_PREFIX),
)
)
except ResponseError as script_error:
verbose_proxy_logger.debug(
"Redis did not run the polling write script, writing with GETRANGE and SET: %s", script_error
)
if await _stored_status_prefix(redis_cache, cache_key) in ("", _CANCELLED_PREFIX):
return 0
await redis_cache.async_set_cache(cache_key, json.dumps({"status": state["status"], **state}), ttl=self.ttl)
return _WRITTEN
@with_service_target(_RESPONSE_POLLING_TARGET)
async def is_cancelled_or_gone(self, polling_id: str) -> bool:
if not self.redis_cache:
return False
cancelled_or_gone: Final = self.redis_cache.async_register_script(_CANCELLED_OR_GONE_SCRIPT)
reply: Final = await cancelled_or_gone(keys=(self.get_cache_key(polling_id),), args=(_CANCELLED_PREFIX,))
return _SCRIPT_REPLY.validate_python(reply) == 1
return await _stored_status_prefix(self.redis_cache, self.get_cache_key(polling_id)) in ("", _CANCELLED_PREFIX)
async def cancel_polling(self, polling_id: str) -> bool:
"""

View file

@ -76,6 +76,41 @@ async def test_async_delete_cache_applies_namespace(namespace, monkeypatch, redi
mock_redis_instance.delete.assert_awaited_once_with(expected_key)
@pytest.mark.parametrize("decode_responses", [False, True])
@pytest.mark.asyncio
async def test_async_getrange_reads_a_prefix_of_what_was_set_under_the_namespace(
decode_responses, monkeypatch, redis_no_ping
):
from fakeredis import FakeServer
from fakeredis.aioredis import FakeRedis
monkeypatch.setenv("REDIS_HOST", "https://my-test-host")
redis_cache: Final = RedisCache(namespace="litellm")
client: Final = FakeRedis(server=FakeServer(), decode_responses=decode_responses)
with patch.object(redis_cache, "init_async_client", return_value=client):
await redis_cache.async_set_cache("polling:1", {"status": "cancelled"}, ttl=60)
assert await redis_cache.async_getrange("polling:1", 0, 11) == '{"status": "'
assert await redis_cache.async_getrange("polling:never-set", 0, 11) == ""
await client.aclose()
@pytest.mark.asyncio
async def test_async_getrange_that_cuts_a_multibyte_character_does_not_raise(monkeypatch, redis_no_ping):
from fakeredis import FakeServer
from fakeredis.aioredis import FakeRedis
monkeypatch.setenv("REDIS_HOST", "https://my-test-host")
redis_cache: Final = RedisCache()
client: Final = FakeRedis(server=FakeServer())
await client.set("polling:1", "é".encode())
with patch.object(redis_cache, "init_async_client", return_value=client):
assert await redis_cache.async_getrange("polling:1", 0, 0) == "<EFBFBD>"
await client.aclose()
@pytest.mark.parametrize("namespace", [None, "litellm"])
def test_delete_cache_applies_namespace(namespace, monkeypatch, redis_no_ping):
"""delete_cache must prefix keys with the namespace, matching every other

View file

@ -7,7 +7,7 @@ import contextlib
import dataclasses
import json
import unittest
from collections.abc import Callable, Mapping
from collections.abc import Awaitable, Callable, Mapping, Sequence
from typing import Any, Final, Literal, cast
from unittest.mock import AsyncMock, MagicMock, call, patch
@ -19,6 +19,7 @@ from fastapi.testclient import TestClient
from httpx import Response
import litellm
from litellm.caching.base_cache import BaseCache
from litellm.constants import EMPTY_MAPPING
from litellm.proxy._types import ProxyException
from litellm.proxy.proxy_server import app
@ -1027,6 +1028,55 @@ async def test_a_finished_read_or_cancel_bills_usage_only_the_configured_deploym
assert row.status == "completed"
class _OnePollingStateRedis(BaseCache):
def __init__(self, state: Mapping[str, object]) -> None:
super().__init__()
self.state = json.dumps(state)
self.ttls: tuple[int, ...] = ()
async def async_get_cache(self, key: str, **kwargs: object) -> str:
return self.state
async def async_set_cache_pipeline(self, cache_list: object, **kwargs: object) -> None:
raise NotImplementedError
def async_register_script(self, script: str) -> Callable[..., Awaitable[int]]:
async def write(keys: Sequence[str], args: Sequence[str]) -> int:
self.state = json.loads(args[0])
self.ttls = (*self.ttls, int(args[1]))
return 1
return write
@pytest.mark.asyncio
async def test_cancelling_a_polling_response_keeps_the_configured_polling_ttl() -> None:
from fastapi import Response as FastAPIResponse
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.response_api_endpoints.endpoints import cancel_response
redis: Final = _OnePollingStateRedis({"id": "litellm_poll_ttl", "object": "response", "status": "in_progress"})
with (
patch(
"litellm.proxy.proxy_server.redis_usage_cache", redis
), # test-quality-ok: the endpoint reads the proxy singletons at call time
patch(
"litellm.proxy.proxy_server.polling_cache_ttl", 120
), # test-quality-ok: the endpoint reads the proxy singletons at call time
):
cancelled: Final = await cancel_response(
response_id="litellm_poll_ttl",
request=_request_for("POST", "/v1/responses/litellm_poll_ttl/cancel", b""),
fastapi_response=FastAPIResponse(),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-canceller"),
)
assert cancelled["status"] == "cancelled"
assert redis.ttls == (120,)
@pytest.mark.asyncio
async def test_deleting_a_finished_unbilled_background_response_bills_it_before_the_provider_forgets_it(
monkeypatch: pytest.MonkeyPatch,

View file

@ -16,6 +16,7 @@ following the OpenAI Response API format.
import asyncio
import json
import logging
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence
from datetime import datetime, timezone
from typing import Dict, Final, NoReturn, Optional
@ -24,7 +25,9 @@ from unittest.mock import AsyncMock, Mock, patch
import pytest
from fastapi import Request
from pydantic import ValidationError
from redis.exceptions import NoPermissionError
from litellm._logging import verbose_proxy_logger
from litellm.caching.base_cache import BaseCache
from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler
@ -55,6 +58,9 @@ class _RedisStandIn(BaseCache):
async def async_set_cache_pipeline(self, cache_list: Sequence[tuple[str, object]], **kwargs: object) -> NoReturn:
raise NotImplementedError
async def async_getrange(self, key: str, start: int, end: int) -> str:
return self.stored.get(key, "")[start : end + 1]
def async_register_script(self, script: str) -> Callable[..., Awaitable[int]]:
async def write_unless_cancelled(keys: Sequence[str], args: Sequence[object]) -> int:
if keys[0] not in self.stored or self.stored[keys[0]].startswith(args[2]):
@ -63,13 +69,21 @@ class _RedisStandIn(BaseCache):
self.ttls[keys[0]] = int(args[1])
return 1
async def cancelled_or_gone(keys: Sequence[str], args: Sequence[object]) -> int:
return int(self.stored.get(keys[0], "")[: len(args[0])] in ("", args[0]))
return write_unless_cancelled
return write_unless_cancelled if "'SET'" in script else cancelled_or_gone
class _ScriptRefusingRedisStandIn(_RedisStandIn):
def async_register_script(self, script: str) -> Callable[..., Awaitable[int]]:
async def refused(keys: Sequence[str], args: Sequence[object]) -> NoReturn:
raise NoPermissionError("this user has no permissions to run the 'evalsha' command")
return refused
class _UnreachableRedisStandIn(_RedisStandIn):
async def async_getrange(self, key: str, start: int, end: int) -> NoReturn:
raise ConnectionError("redis is unreachable")
def async_register_script(self, script: str) -> Callable[..., Awaitable[int]]:
async def write_fails(keys: Sequence[str], args: Sequence[object]) -> NoReturn:
raise ConnectionError("redis is unreachable")
@ -628,6 +642,27 @@ class TestResponsePollingHandler:
assert await handler.is_cancelled_or_gone("litellm_poll_test") is expected
assert await handler.is_cancelled_or_gone("litellm_poll_never_created") is True
@pytest.mark.asyncio
@pytest.mark.parametrize("redis_type", (_RedisStandIn, _ScriptRefusingRedisStandIn))
async def test_a_cancel_sticks_whether_or_not_redis_runs_scripts(self, redis_type: type[_RedisStandIn]) -> None:
redis_cache: Final = redis_type.holding(
{"id": "litellm_poll_test", "object": "response", "status": "in_progress", "output": []}
)
handler: Final = ResponsePollingHandler(redis_cache=redis_cache, ttl=120)
await handler.update_state(polling_id="litellm_poll_test", output=[{"id": "item_1"}])
await handler.cancel_polling("litellm_poll_test")
await handler.update_state(polling_id="litellm_poll_test", status="completed", output=[{"id": "item_2"}])
assert await handler.get_state("litellm_poll_test") == {
"id": "litellm_poll_test",
"object": "response",
"status": "cancelled",
"output": [{"id": "item_1"}],
}
assert await handler.is_cancelled_or_gone("litellm_poll_test") is True
assert redis_cache.ttls == {ResponsePollingHandler.get_cache_key("litellm_poll_test"): 120}
@pytest.mark.asyncio
async def test_a_failed_state_write_keeps_the_stored_state_and_does_not_raise(self) -> None:
state: Final = {
@ -643,6 +678,37 @@ class TestResponsePollingHandler:
assert await handler.get_state("litellm_poll_test") == state
@pytest.mark.asyncio
@pytest.mark.parametrize(
("redis_cache", "logged"),
(
(
_UnreachableRedisStandIn.holding({"status": "in_progress"}, polling_id="litellm_poll_a\nforged"),
"Failed to write the polling state for litellm_poll_aforged: redis is unreachable",
),
(
_RedisStandIn(),
"No cached state found for polling_id: litellm_poll_aforged",
),
(
_RedisStandIn.holding({"status": "cancelled"}, polling_id="litellm_poll_a\nforged"),
"Skipped the polling state write for litellm_poll_aforged: "
"the response was cancelled or has expired",
),
),
)
async def test_a_polling_id_with_a_line_break_cannot_forge_a_state_write_log_line(
self, redis_cache: _RedisStandIn, logged: str, caplog: pytest.LogCaptureFixture
) -> None:
handler: Final = ResponsePollingHandler(redis_cache=redis_cache)
with caplog.at_level(logging.DEBUG, logger=verbose_proxy_logger.name):
await handler.update_state(polling_id="litellm_poll_a\nforged", status="cancelled")
messages: Final = [record.getMessage() for record in caplog.records]
assert logged in messages
assert not [message for message in messages if "\n" in message]
@pytest.mark.asyncio
async def test_a_queued_response_reads_back_after_an_update(self) -> None:
handler: Final = ResponsePollingHandler(redis_cache=_RedisStandIn())