diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index afe6f8a1eab..abd4d8472b9 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -91,6 +91,8 @@ class _AsyncRedisCommands(Protocol): def lpop(self, name: str, count: int | None = None) -> Awaitable[object]: ... + def getrange(self, key: str, start: int, end: int) -> Awaitable[object]: ... + def pipeline(self, transaction: bool = True) -> "Pipeline[bytes]": ... def eval(self, script: str, numkeys: int, *keys_and_args: str | bytes | float) -> Awaitable[object]: ... @@ -141,6 +143,7 @@ _INCREMENT_IF_PRESENT_LUA: Final = ( _LUA_COUNT: Final = TypeAdapter(int) _LUA_OPTIONAL_TOTAL: Final[TypeAdapter[float | None]] = TypeAdapter(float | None) _OPTIONAL_COUNTS: Final = TypeAdapter(tuple[int | None, ...]) +_GETRANGE_REPLY: Final[TypeAdapter[bytes | str]] = TypeAdapter(bytes | str) def _decoded_counts(values: Sequence[bytes | str | None]) -> tuple[int | None, ...]: @@ -1737,6 +1740,13 @@ class RedisCache(BaseCache): _record_swallowed_redis_failure(self._circuit_breaker, e) return key_value_dict + @_redis_circuit_breaker_guard + async def async_getrange(self, key: str, start: int, end: int) -> str: + value: Final = _GETRANGE_REPLY.validate_python( + await self._async_commands().getrange(self.check_and_fix_namespace(key=key), start, end) + ) + return value.decode("utf-8", errors="replace") if isinstance(value, bytes) else value + @_redis_circuit_breaker_guard async def async_get_cache( self, key, parent_otel_span: Span | None = None, *, raise_on_error: bool = False, **kwargs diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 4f8b1697f7e..bb800888f03 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -1790,6 +1790,7 @@ async def cancel_response( from litellm.proxy.proxy_server import ( general_settings, llm_router, + polling_cache_ttl, proxy_config, proxy_logging_obj, read_request_body, @@ -1810,7 +1811,7 @@ async def cancel_response( if not redis_usage_cache: raise HTTPException(status_code=500, detail="Redis cache not configured.") - polling_handler: Final = ResponsePollingHandler(redis_cache=redis_usage_cache) + polling_handler: Final = ResponsePollingHandler(redis_cache=redis_usage_cache, ttl=polling_cache_ttl) # Get current state to verify it exists state: Final = await polling_handler.get_state(response_id) diff --git a/litellm/proxy/response_polling/polling_handler.py b/litellm/proxy/response_polling/polling_handler.py index 946fd1f3f62..1dc1575de7a 100644 --- a/litellm/proxy/response_polling/polling_handler.py +++ b/litellm/proxy/response_polling/polling_handler.py @@ -8,11 +8,13 @@ from datetime import datetime, timezone from typing import Final from pydantic import TypeAdapter +from redis.exceptions import ResponseError from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid4 from litellm.caching.redis_cache import RedisCache +from litellm.proxy.litellm_pre_call_utils import sanitize_for_log from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStatus _RESPONSE_POLLING_TARGET: Final = "response_polling" @@ -39,13 +41,9 @@ redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2]) return 1 """ -_CANCELLED_OR_GONE_SCRIPT: Final = """ -local head = redis.call('GETRANGE', KEYS[1], 0, string.len(ARGV[1]) - 1) -if head == '' or head == ARGV[1] then - return 1 -end -return 0 -""" + +async def _stored_status_prefix(redis_cache: RedisCache, cache_key: str) -> str: + return await redis_cache.async_getrange(cache_key, 0, len(_CANCELLED_PREFIX) - 1) class ResponsePollingHandler: @@ -181,7 +179,7 @@ class ResponsePollingHandler: # Get current state cached_state: Final = await self.redis_cache.async_get_cache(cache_key) if not cached_state: - verbose_proxy_logger.warning("No cached state found for polling_id: %s", polling_id) + verbose_proxy_logger.warning("No cached state found for polling_id: %s", sanitize_for_log(polling_id)) return # Parse existing ResponsesAPIResponse from cache @@ -240,20 +238,17 @@ class ResponsePollingHandler: if store is not None: state["store"] = store - write_unless_cancelled: Final = self.redis_cache.async_register_script(_WRITE_UNLESS_CANCELLED_SCRIPT) try: - written: Final = _SCRIPT_REPLY.validate_python( - await write_unless_cancelled( - keys=(cache_key,), - args=(_encoded_state(state), self.redis_cache.get_ttl(ttl=self.ttl), _CANCELLED_PREFIX), - ) - ) + written: Final = await self._write_unless_cancelled(self.redis_cache, cache_key, state) except Exception as write_error: # noqa: BLE001 # a Redis fault must not end the background stream - verbose_proxy_logger.error("Failed to write the polling state for %s: %s", polling_id, write_error) + verbose_proxy_logger.error( + "Failed to write the polling state for %s: %s", sanitize_for_log(polling_id), write_error + ) return if written != _WRITTEN: verbose_proxy_logger.debug( - "Skipped the polling state write for %s: the response was cancelled or has expired", polling_id + "Skipped the polling state write for %s: the response was cancelled or has expired", + sanitize_for_log(polling_id), ) return @@ -276,13 +271,31 @@ class ResponsePollingHandler: return None + async def _write_unless_cancelled( + self, redis_cache: RedisCache, cache_key: str, state: Mapping[str, object] + ) -> int: + write_unless_cancelled: Final = redis_cache.async_register_script(_WRITE_UNLESS_CANCELLED_SCRIPT) + try: + return _SCRIPT_REPLY.validate_python( + await write_unless_cancelled( + keys=(cache_key,), + args=(_encoded_state(state), redis_cache.get_ttl(ttl=self.ttl), _CANCELLED_PREFIX), + ) + ) + except ResponseError as script_error: + verbose_proxy_logger.debug( + "Redis did not run the polling write script, writing with GETRANGE and SET: %s", script_error + ) + if await _stored_status_prefix(redis_cache, cache_key) in ("", _CANCELLED_PREFIX): + return 0 + await redis_cache.async_set_cache(cache_key, json.dumps({"status": state["status"], **state}), ttl=self.ttl) + return _WRITTEN + @with_service_target(_RESPONSE_POLLING_TARGET) async def is_cancelled_or_gone(self, polling_id: str) -> bool: if not self.redis_cache: return False - cancelled_or_gone: Final = self.redis_cache.async_register_script(_CANCELLED_OR_GONE_SCRIPT) - reply: Final = await cancelled_or_gone(keys=(self.get_cache_key(polling_id),), args=(_CANCELLED_PREFIX,)) - return _SCRIPT_REPLY.validate_python(reply) == 1 + return await _stored_status_prefix(self.redis_cache, self.get_cache_key(polling_id)) in ("", _CANCELLED_PREFIX) async def cancel_polling(self, polling_id: str) -> bool: """ diff --git a/tests/unit/caching/test_redis_cache.py b/tests/unit/caching/test_redis_cache.py index 2bdc7b9549b..8a16c55ef4b 100644 --- a/tests/unit/caching/test_redis_cache.py +++ b/tests/unit/caching/test_redis_cache.py @@ -76,6 +76,41 @@ async def test_async_delete_cache_applies_namespace(namespace, monkeypatch, redi mock_redis_instance.delete.assert_awaited_once_with(expected_key) +@pytest.mark.parametrize("decode_responses", [False, True]) +@pytest.mark.asyncio +async def test_async_getrange_reads_a_prefix_of_what_was_set_under_the_namespace( + decode_responses, monkeypatch, redis_no_ping +): + from fakeredis import FakeServer + from fakeredis.aioredis import FakeRedis + + monkeypatch.setenv("REDIS_HOST", "https://my-test-host") + redis_cache: Final = RedisCache(namespace="litellm") + client: Final = FakeRedis(server=FakeServer(), decode_responses=decode_responses) + + with patch.object(redis_cache, "init_async_client", return_value=client): + await redis_cache.async_set_cache("polling:1", {"status": "cancelled"}, ttl=60) + + assert await redis_cache.async_getrange("polling:1", 0, 11) == '{"status": "' + assert await redis_cache.async_getrange("polling:never-set", 0, 11) == "" + await client.aclose() + + +@pytest.mark.asyncio +async def test_async_getrange_that_cuts_a_multibyte_character_does_not_raise(monkeypatch, redis_no_ping): + from fakeredis import FakeServer + from fakeredis.aioredis import FakeRedis + + monkeypatch.setenv("REDIS_HOST", "https://my-test-host") + redis_cache: Final = RedisCache() + client: Final = FakeRedis(server=FakeServer()) + await client.set("polling:1", "é".encode()) + + with patch.object(redis_cache, "init_async_client", return_value=client): + assert await redis_cache.async_getrange("polling:1", 0, 0) == "�" + await client.aclose() + + @pytest.mark.parametrize("namespace", [None, "litellm"]) def test_delete_cache_applies_namespace(namespace, monkeypatch, redis_no_ping): """delete_cache must prefix keys with the namespace, matching every other diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 7760b6b7fcc..b09e070664d 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -7,7 +7,7 @@ import contextlib import dataclasses import json import unittest -from collections.abc import Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from typing import Any, Final, Literal, cast from unittest.mock import AsyncMock, MagicMock, call, patch @@ -19,6 +19,7 @@ from fastapi.testclient import TestClient from httpx import Response import litellm +from litellm.caching.base_cache import BaseCache from litellm.constants import EMPTY_MAPPING from litellm.proxy._types import ProxyException from litellm.proxy.proxy_server import app @@ -1027,6 +1028,55 @@ async def test_a_finished_read_or_cancel_bills_usage_only_the_configured_deploym assert row.status == "completed" +class _OnePollingStateRedis(BaseCache): + def __init__(self, state: Mapping[str, object]) -> None: + super().__init__() + self.state = json.dumps(state) + self.ttls: tuple[int, ...] = () + + async def async_get_cache(self, key: str, **kwargs: object) -> str: + return self.state + + async def async_set_cache_pipeline(self, cache_list: object, **kwargs: object) -> None: + raise NotImplementedError + + def async_register_script(self, script: str) -> Callable[..., Awaitable[int]]: + async def write(keys: Sequence[str], args: Sequence[str]) -> int: + self.state = json.loads(args[0]) + self.ttls = (*self.ttls, int(args[1])) + return 1 + + return write + + +@pytest.mark.asyncio +async def test_cancelling_a_polling_response_keeps_the_configured_polling_ttl() -> None: + from fastapi import Response as FastAPIResponse + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.response_api_endpoints.endpoints import cancel_response + + redis: Final = _OnePollingStateRedis({"id": "litellm_poll_ttl", "object": "response", "status": "in_progress"}) + + with ( + patch( + "litellm.proxy.proxy_server.redis_usage_cache", redis + ), # test-quality-ok: the endpoint reads the proxy singletons at call time + patch( + "litellm.proxy.proxy_server.polling_cache_ttl", 120 + ), # test-quality-ok: the endpoint reads the proxy singletons at call time + ): + cancelled: Final = await cancel_response( + response_id="litellm_poll_ttl", + request=_request_for("POST", "/v1/responses/litellm_poll_ttl/cancel", b""), + fastapi_response=FastAPIResponse(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-canceller"), + ) + + assert cancelled["status"] == "cancelled" + assert redis.ttls == (120,) + + @pytest.mark.asyncio async def test_deleting_a_finished_unbilled_background_response_bills_it_before_the_provider_forgets_it( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 28806ff2acd..b1b2810f8df 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -16,6 +16,7 @@ following the OpenAI Response API format. import asyncio import json +import logging from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence from datetime import datetime, timezone from typing import Dict, Final, NoReturn, Optional @@ -24,7 +25,9 @@ from unittest.mock import AsyncMock, Mock, patch import pytest from fastapi import Request from pydantic import ValidationError +from redis.exceptions import NoPermissionError +from litellm._logging import verbose_proxy_logger from litellm.caching.base_cache import BaseCache from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler @@ -55,6 +58,9 @@ class _RedisStandIn(BaseCache): async def async_set_cache_pipeline(self, cache_list: Sequence[tuple[str, object]], **kwargs: object) -> NoReturn: raise NotImplementedError + async def async_getrange(self, key: str, start: int, end: int) -> str: + return self.stored.get(key, "")[start : end + 1] + def async_register_script(self, script: str) -> Callable[..., Awaitable[int]]: async def write_unless_cancelled(keys: Sequence[str], args: Sequence[object]) -> int: if keys[0] not in self.stored or self.stored[keys[0]].startswith(args[2]): @@ -63,13 +69,21 @@ class _RedisStandIn(BaseCache): self.ttls[keys[0]] = int(args[1]) return 1 - async def cancelled_or_gone(keys: Sequence[str], args: Sequence[object]) -> int: - return int(self.stored.get(keys[0], "")[: len(args[0])] in ("", args[0])) + return write_unless_cancelled - return write_unless_cancelled if "'SET'" in script else cancelled_or_gone + +class _ScriptRefusingRedisStandIn(_RedisStandIn): + def async_register_script(self, script: str) -> Callable[..., Awaitable[int]]: + async def refused(keys: Sequence[str], args: Sequence[object]) -> NoReturn: + raise NoPermissionError("this user has no permissions to run the 'evalsha' command") + + return refused class _UnreachableRedisStandIn(_RedisStandIn): + async def async_getrange(self, key: str, start: int, end: int) -> NoReturn: + raise ConnectionError("redis is unreachable") + def async_register_script(self, script: str) -> Callable[..., Awaitable[int]]: async def write_fails(keys: Sequence[str], args: Sequence[object]) -> NoReturn: raise ConnectionError("redis is unreachable") @@ -628,6 +642,27 @@ class TestResponsePollingHandler: assert await handler.is_cancelled_or_gone("litellm_poll_test") is expected assert await handler.is_cancelled_or_gone("litellm_poll_never_created") is True + @pytest.mark.asyncio + @pytest.mark.parametrize("redis_type", (_RedisStandIn, _ScriptRefusingRedisStandIn)) + async def test_a_cancel_sticks_whether_or_not_redis_runs_scripts(self, redis_type: type[_RedisStandIn]) -> None: + redis_cache: Final = redis_type.holding( + {"id": "litellm_poll_test", "object": "response", "status": "in_progress", "output": []} + ) + handler: Final = ResponsePollingHandler(redis_cache=redis_cache, ttl=120) + + await handler.update_state(polling_id="litellm_poll_test", output=[{"id": "item_1"}]) + await handler.cancel_polling("litellm_poll_test") + await handler.update_state(polling_id="litellm_poll_test", status="completed", output=[{"id": "item_2"}]) + + assert await handler.get_state("litellm_poll_test") == { + "id": "litellm_poll_test", + "object": "response", + "status": "cancelled", + "output": [{"id": "item_1"}], + } + assert await handler.is_cancelled_or_gone("litellm_poll_test") is True + assert redis_cache.ttls == {ResponsePollingHandler.get_cache_key("litellm_poll_test"): 120} + @pytest.mark.asyncio async def test_a_failed_state_write_keeps_the_stored_state_and_does_not_raise(self) -> None: state: Final = { @@ -643,6 +678,37 @@ class TestResponsePollingHandler: assert await handler.get_state("litellm_poll_test") == state + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("redis_cache", "logged"), + ( + ( + _UnreachableRedisStandIn.holding({"status": "in_progress"}, polling_id="litellm_poll_a\nforged"), + "Failed to write the polling state for litellm_poll_aforged: redis is unreachable", + ), + ( + _RedisStandIn(), + "No cached state found for polling_id: litellm_poll_aforged", + ), + ( + _RedisStandIn.holding({"status": "cancelled"}, polling_id="litellm_poll_a\nforged"), + "Skipped the polling state write for litellm_poll_aforged: " + "the response was cancelled or has expired", + ), + ), + ) + async def test_a_polling_id_with_a_line_break_cannot_forge_a_state_write_log_line( + self, redis_cache: _RedisStandIn, logged: str, caplog: pytest.LogCaptureFixture + ) -> None: + handler: Final = ResponsePollingHandler(redis_cache=redis_cache) + + with caplog.at_level(logging.DEBUG, logger=verbose_proxy_logger.name): + await handler.update_state(polling_id="litellm_poll_a\nforged", status="cancelled") + + messages: Final = [record.getMessage() for record in caplog.records] + assert logged in messages + assert not [message for message in messages if "\n" in message] + @pytest.mark.asyncio async def test_a_queued_response_reads_back_after_an_update(self) -> None: handler: Final = ResponsePollingHandler(redis_cache=_RedisStandIn())