From 2f4c807e58e35f0b09c61ee0436ca816701cabd1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:40:36 -0700 Subject: [PATCH] fix(proxy): keep polling working on a Redis that refuses scripts, keep the cancel ttl, and sanitize polling ids in logs A Redis whose ACL denies scripting answered every polling state write with an error, so background responses stayed queued and a cancel never stopped the call. The write now falls back to GETRANGE then SET with the same cancelled and missing checks when Redis refuses the script, and the background task's cancel check is a plain GETRANGE. The cancel route now writes with the configured polling ttl instead of the 1 hour default, and the polling id is sanitized in the state write logs. --- litellm/caching/redis_cache.py | 10 +++ .../proxy/response_api_endpoints/endpoints.py | 3 +- .../proxy/response_polling/polling_handler.py | 53 ++++++++------ tests/unit/caching/test_redis_cache.py | 35 +++++++++ .../response_api_endpoints/test_endpoints.py | 52 +++++++++++++- .../test_response_polling_handler.py | 72 ++++++++++++++++++- 6 files changed, 200 insertions(+), 25 deletions(-) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index afe6f8a1eab..abd4d8472b9 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -91,6 +91,8 @@ class _AsyncRedisCommands(Protocol): def lpop(self, name: str, count: int | None = None) -> Awaitable[object]: ... + def getrange(self, key: str, start: int, end: int) -> Awaitable[object]: ... + def pipeline(self, transaction: bool = True) -> "Pipeline[bytes]": ... def eval(self, script: str, numkeys: int, *keys_and_args: str | bytes | float) -> Awaitable[object]: ... @@ -141,6 +143,7 @@ _INCREMENT_IF_PRESENT_LUA: Final = ( _LUA_COUNT: Final = TypeAdapter(int) _LUA_OPTIONAL_TOTAL: Final[TypeAdapter[float | None]] = TypeAdapter(float | None) _OPTIONAL_COUNTS: Final = TypeAdapter(tuple[int | None, ...]) +_GETRANGE_REPLY: Final[TypeAdapter[bytes | str]] = TypeAdapter(bytes | str) def _decoded_counts(values: Sequence[bytes | str | None]) -> tuple[int | None, ...]: @@ -1737,6 +1740,13 @@ class RedisCache(BaseCache): _record_swallowed_redis_failure(self._circuit_breaker, e) return key_value_dict + @_redis_circuit_breaker_guard + async def async_getrange(self, key: str, start: int, end: int) -> str: + value: Final = _GETRANGE_REPLY.validate_python( + await self._async_commands().getrange(self.check_and_fix_namespace(key=key), start, end) + ) + return value.decode("utf-8", errors="replace") if isinstance(value, bytes) else value + @_redis_circuit_breaker_guard async def async_get_cache( self, key, parent_otel_span: Span | None = None, *, raise_on_error: bool = False, **kwargs diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 4f8b1697f7e..bb800888f03 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -1790,6 +1790,7 @@ async def cancel_response( from litellm.proxy.proxy_server import ( general_settings, llm_router, + polling_cache_ttl, proxy_config, proxy_logging_obj, read_request_body, @@ -1810,7 +1811,7 @@ async def cancel_response( if not redis_usage_cache: raise HTTPException(status_code=500, detail="Redis cache not configured.") - polling_handler: Final = ResponsePollingHandler(redis_cache=redis_usage_cache) + polling_handler: Final = ResponsePollingHandler(redis_cache=redis_usage_cache, ttl=polling_cache_ttl) # Get current state to verify it exists state: Final = await polling_handler.get_state(response_id) diff --git a/litellm/proxy/response_polling/polling_handler.py b/litellm/proxy/response_polling/polling_handler.py index 946fd1f3f62..1dc1575de7a 100644 --- a/litellm/proxy/response_polling/polling_handler.py +++ b/litellm/proxy/response_polling/polling_handler.py @@ -8,11 +8,13 @@ from datetime import datetime, timezone from typing import Final from pydantic import TypeAdapter +from redis.exceptions import ResponseError from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid4 from litellm.caching.redis_cache import RedisCache +from litellm.proxy.litellm_pre_call_utils import sanitize_for_log from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStatus _RESPONSE_POLLING_TARGET: Final = "response_polling" @@ -39,13 +41,9 @@ redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2]) return 1 """ -_CANCELLED_OR_GONE_SCRIPT: Final = """ -local head = redis.call('GETRANGE', KEYS[1], 0, string.len(ARGV[1]) - 1) -if head == '' or head == ARGV[1] then - return 1 -end -return 0 -""" + +async def _stored_status_prefix(redis_cache: RedisCache, cache_key: str) -> str: + return await redis_cache.async_getrange(cache_key, 0, len(_CANCELLED_PREFIX) - 1) class ResponsePollingHandler: @@ -181,7 +179,7 @@ class ResponsePollingHandler: # Get current state cached_state: Final = await self.redis_cache.async_get_cache(cache_key) if not cached_state: - verbose_proxy_logger.warning("No cached state found for polling_id: %s", polling_id) + verbose_proxy_logger.warning("No cached state found for polling_id: %s", sanitize_for_log(polling_id)) return # Parse existing ResponsesAPIResponse from cache @@ -240,20 +238,17 @@ class ResponsePollingHandler: if store is not None: state["store"] = store - write_unless_cancelled: Final = self.redis_cache.async_register_script(_WRITE_UNLESS_CANCELLED_SCRIPT) try: - written: Final = _SCRIPT_REPLY.validate_python( - await write_unless_cancelled( - keys=(cache_key,), - args=(_encoded_state(state), self.redis_cache.get_ttl(ttl=self.ttl), _CANCELLED_PREFIX), - ) - ) + written: Final = await self._write_unless_cancelled(self.redis_cache, cache_key, state) except Exception as write_error: # noqa: BLE001 # a Redis fault must not end the background stream - verbose_proxy_logger.error("Failed to write the polling state for %s: %s", polling_id, write_error) + verbose_proxy_logger.error( + "Failed to write the polling state for %s: %s", sanitize_for_log(polling_id), write_error + ) return if written != _WRITTEN: verbose_proxy_logger.debug( - "Skipped the polling state write for %s: the response was cancelled or has expired", polling_id + "Skipped the polling state write for %s: the response was cancelled or has expired", + sanitize_for_log(polling_id), ) return @@ -276,13 +271,31 @@ class ResponsePollingHandler: return None + async def _write_unless_cancelled( + self, redis_cache: RedisCache, cache_key: str, state: Mapping[str, object] + ) -> int: + write_unless_cancelled: Final = redis_cache.async_register_script(_WRITE_UNLESS_CANCELLED_SCRIPT) + try: + return _SCRIPT_REPLY.validate_python( + await write_unless_cancelled( + keys=(cache_key,), + args=(_encoded_state(state), redis_cache.get_ttl(ttl=self.ttl), _CANCELLED_PREFIX), + ) + ) + except ResponseError as script_error: + verbose_proxy_logger.debug( + "Redis did not run the polling write script, writing with GETRANGE and SET: %s", script_error + ) + if await _stored_status_prefix(redis_cache, cache_key) in ("", _CANCELLED_PREFIX): + return 0 + await redis_cache.async_set_cache(cache_key, json.dumps({"status": state["status"], **state}), ttl=self.ttl) + return _WRITTEN + @with_service_target(_RESPONSE_POLLING_TARGET) async def is_cancelled_or_gone(self, polling_id: str) -> bool: if not self.redis_cache: return False - cancelled_or_gone: Final = self.redis_cache.async_register_script(_CANCELLED_OR_GONE_SCRIPT) - reply: Final = await cancelled_or_gone(keys=(self.get_cache_key(polling_id),), args=(_CANCELLED_PREFIX,)) - return _SCRIPT_REPLY.validate_python(reply) == 1 + return await _stored_status_prefix(self.redis_cache, self.get_cache_key(polling_id)) in ("", _CANCELLED_PREFIX) async def cancel_polling(self, polling_id: str) -> bool: """ diff --git a/tests/unit/caching/test_redis_cache.py b/tests/unit/caching/test_redis_cache.py index 2bdc7b9549b..8a16c55ef4b 100644 --- a/tests/unit/caching/test_redis_cache.py +++ b/tests/unit/caching/test_redis_cache.py @@ -76,6 +76,41 @@ async def test_async_delete_cache_applies_namespace(namespace, monkeypatch, redi mock_redis_instance.delete.assert_awaited_once_with(expected_key) +@pytest.mark.parametrize("decode_responses", [False, True]) +@pytest.mark.asyncio +async def test_async_getrange_reads_a_prefix_of_what_was_set_under_the_namespace( + decode_responses, monkeypatch, redis_no_ping +): + from fakeredis import FakeServer + from fakeredis.aioredis import FakeRedis + + monkeypatch.setenv("REDIS_HOST", "https://my-test-host") + redis_cache: Final = RedisCache(namespace="litellm") + client: Final = FakeRedis(server=FakeServer(), decode_responses=decode_responses) + + with patch.object(redis_cache, "init_async_client", return_value=client): + await redis_cache.async_set_cache("polling:1", {"status": "cancelled"}, ttl=60) + + assert await redis_cache.async_getrange("polling:1", 0, 11) == '{"status": "' + assert await redis_cache.async_getrange("polling:never-set", 0, 11) == "" + await client.aclose() + + +@pytest.mark.asyncio +async def test_async_getrange_that_cuts_a_multibyte_character_does_not_raise(monkeypatch, redis_no_ping): + from fakeredis import FakeServer + from fakeredis.aioredis import FakeRedis + + monkeypatch.setenv("REDIS_HOST", "https://my-test-host") + redis_cache: Final = RedisCache() + client: Final = FakeRedis(server=FakeServer()) + await client.set("polling:1", "é".encode()) + + with patch.object(redis_cache, "init_async_client", return_value=client): + assert await redis_cache.async_getrange("polling:1", 0, 0) == "�" + await client.aclose() + + @pytest.mark.parametrize("namespace", [None, "litellm"]) def test_delete_cache_applies_namespace(namespace, monkeypatch, redis_no_ping): """delete_cache must prefix keys with the namespace, matching every other diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 7760b6b7fcc..b09e070664d 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -7,7 +7,7 @@ import contextlib import dataclasses import json import unittest -from collections.abc import Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from typing import Any, Final, Literal, cast from unittest.mock import AsyncMock, MagicMock, call, patch @@ -19,6 +19,7 @@ from fastapi.testclient import TestClient from httpx import Response import litellm +from litellm.caching.base_cache import BaseCache from litellm.constants import EMPTY_MAPPING from litellm.proxy._types import ProxyException from litellm.proxy.proxy_server import app @@ -1027,6 +1028,55 @@ async def test_a_finished_read_or_cancel_bills_usage_only_the_configured_deploym assert row.status == "completed" +class _OnePollingStateRedis(BaseCache): + def __init__(self, state: Mapping[str, object]) -> None: + super().__init__() + self.state = json.dumps(state) + self.ttls: tuple[int, ...] = () + + async def async_get_cache(self, key: str, **kwargs: object) -> str: + return self.state + + async def async_set_cache_pipeline(self, cache_list: object, **kwargs: object) -> None: + raise NotImplementedError + + def async_register_script(self, script: str) -> Callable[..., Awaitable[int]]: + async def write(keys: Sequence[str], args: Sequence[str]) -> int: + self.state = json.loads(args[0]) + self.ttls = (*self.ttls, int(args[1])) + return 1 + + return write + + +@pytest.mark.asyncio +async def test_cancelling_a_polling_response_keeps_the_configured_polling_ttl() -> None: + from fastapi import Response as FastAPIResponse + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.response_api_endpoints.endpoints import cancel_response + + redis: Final = _OnePollingStateRedis({"id": "litellm_poll_ttl", "object": "response", "status": "in_progress"}) + + with ( + patch( + "litellm.proxy.proxy_server.redis_usage_cache", redis + ), # test-quality-ok: the endpoint reads the proxy singletons at call time + patch( + "litellm.proxy.proxy_server.polling_cache_ttl", 120 + ), # test-quality-ok: the endpoint reads the proxy singletons at call time + ): + cancelled: Final = await cancel_response( + response_id="litellm_poll_ttl", + request=_request_for("POST", "/v1/responses/litellm_poll_ttl/cancel", b""), + fastapi_response=FastAPIResponse(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-canceller"), + ) + + assert cancelled["status"] == "cancelled" + assert redis.ttls == (120,) + + @pytest.mark.asyncio async def test_deleting_a_finished_unbilled_background_response_bills_it_before_the_provider_forgets_it( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 28806ff2acd..b1b2810f8df 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -16,6 +16,7 @@ following the OpenAI Response API format. import asyncio import json +import logging from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence from datetime import datetime, timezone from typing import Dict, Final, NoReturn, Optional @@ -24,7 +25,9 @@ from unittest.mock import AsyncMock, Mock, patch import pytest from fastapi import Request from pydantic import ValidationError +from redis.exceptions import NoPermissionError +from litellm._logging import verbose_proxy_logger from litellm.caching.base_cache import BaseCache from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler @@ -55,6 +58,9 @@ class _RedisStandIn(BaseCache): async def async_set_cache_pipeline(self, cache_list: Sequence[tuple[str, object]], **kwargs: object) -> NoReturn: raise NotImplementedError + async def async_getrange(self, key: str, start: int, end: int) -> str: + return self.stored.get(key, "")[start : end + 1] + def async_register_script(self, script: str) -> Callable[..., Awaitable[int]]: async def write_unless_cancelled(keys: Sequence[str], args: Sequence[object]) -> int: if keys[0] not in self.stored or self.stored[keys[0]].startswith(args[2]): @@ -63,13 +69,21 @@ class _RedisStandIn(BaseCache): self.ttls[keys[0]] = int(args[1]) return 1 - async def cancelled_or_gone(keys: Sequence[str], args: Sequence[object]) -> int: - return int(self.stored.get(keys[0], "")[: len(args[0])] in ("", args[0])) + return write_unless_cancelled - return write_unless_cancelled if "'SET'" in script else cancelled_or_gone + +class _ScriptRefusingRedisStandIn(_RedisStandIn): + def async_register_script(self, script: str) -> Callable[..., Awaitable[int]]: + async def refused(keys: Sequence[str], args: Sequence[object]) -> NoReturn: + raise NoPermissionError("this user has no permissions to run the 'evalsha' command") + + return refused class _UnreachableRedisStandIn(_RedisStandIn): + async def async_getrange(self, key: str, start: int, end: int) -> NoReturn: + raise ConnectionError("redis is unreachable") + def async_register_script(self, script: str) -> Callable[..., Awaitable[int]]: async def write_fails(keys: Sequence[str], args: Sequence[object]) -> NoReturn: raise ConnectionError("redis is unreachable") @@ -628,6 +642,27 @@ class TestResponsePollingHandler: assert await handler.is_cancelled_or_gone("litellm_poll_test") is expected assert await handler.is_cancelled_or_gone("litellm_poll_never_created") is True + @pytest.mark.asyncio + @pytest.mark.parametrize("redis_type", (_RedisStandIn, _ScriptRefusingRedisStandIn)) + async def test_a_cancel_sticks_whether_or_not_redis_runs_scripts(self, redis_type: type[_RedisStandIn]) -> None: + redis_cache: Final = redis_type.holding( + {"id": "litellm_poll_test", "object": "response", "status": "in_progress", "output": []} + ) + handler: Final = ResponsePollingHandler(redis_cache=redis_cache, ttl=120) + + await handler.update_state(polling_id="litellm_poll_test", output=[{"id": "item_1"}]) + await handler.cancel_polling("litellm_poll_test") + await handler.update_state(polling_id="litellm_poll_test", status="completed", output=[{"id": "item_2"}]) + + assert await handler.get_state("litellm_poll_test") == { + "id": "litellm_poll_test", + "object": "response", + "status": "cancelled", + "output": [{"id": "item_1"}], + } + assert await handler.is_cancelled_or_gone("litellm_poll_test") is True + assert redis_cache.ttls == {ResponsePollingHandler.get_cache_key("litellm_poll_test"): 120} + @pytest.mark.asyncio async def test_a_failed_state_write_keeps_the_stored_state_and_does_not_raise(self) -> None: state: Final = { @@ -643,6 +678,37 @@ class TestResponsePollingHandler: assert await handler.get_state("litellm_poll_test") == state + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("redis_cache", "logged"), + ( + ( + _UnreachableRedisStandIn.holding({"status": "in_progress"}, polling_id="litellm_poll_a\nforged"), + "Failed to write the polling state for litellm_poll_aforged: redis is unreachable", + ), + ( + _RedisStandIn(), + "No cached state found for polling_id: litellm_poll_aforged", + ), + ( + _RedisStandIn.holding({"status": "cancelled"}, polling_id="litellm_poll_a\nforged"), + "Skipped the polling state write for litellm_poll_aforged: " + "the response was cancelled or has expired", + ), + ), + ) + async def test_a_polling_id_with_a_line_break_cannot_forge_a_state_write_log_line( + self, redis_cache: _RedisStandIn, logged: str, caplog: pytest.LogCaptureFixture + ) -> None: + handler: Final = ResponsePollingHandler(redis_cache=redis_cache) + + with caplog.at_level(logging.DEBUG, logger=verbose_proxy_logger.name): + await handler.update_state(polling_id="litellm_poll_a\nforged", status="cancelled") + + messages: Final = [record.getMessage() for record in caplog.records] + assert logged in messages + assert not [message for message in messages if "\n" in message] + @pytest.mark.asyncio async def test_a_queued_response_reads_back_after_an_update(self) -> None: handler: Final = ResponsePollingHandler(redis_cache=_RedisStandIn())