Merge branch 'main' into litellm_claude_code_identity_scrub

.circleci/config.yml

@@ -6,6 +6,9 @@ parameters:
   migration_candidate_image:
     type: string
     default: ""
+  migration_baseline_image:
+    type: string
+    default: "ghcr.io/berriai/litellm-database:v1.102.0"
   migration_source_sha:
     type: string
     default: ""
@@ -1508,7 +1511,7 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver"
+            uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not legacy_resolver"
 
   installing_litellm_on_python_3_13:
     docker:
@@ -1532,7 +1535,7 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver"
+            uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not legacy_resolver"
 
   installing_litellm_on_python_v2_migration_resolver:
     docker:
@@ -1561,10 +1564,11 @@ jobs:
           url: tcp://localhost:5432
           timeout: "60"
       - run:
-          name: Run v2 migration resolver proxy smoke test
+          name: Run both migration resolvers against Postgres
           command: |
             uv run --no-sync python -m pytest -vv \
-              tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver
+              tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings \
+              tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_legacy_resolver
 
   helm_chart_testing:
     machine:
@@ -2945,7 +2949,10 @@ jobs:
     parameters:
       suite:
         type: enum
-        enum: [startup, recovery, legacy]
+        enum: [startup, recovery, legacy, upgrade, shaped]
+      baseline:
+        type: boolean
+        default: false
     machine:
       image: ubuntu-2204:2024.04.1
     resource_class: large
@@ -2953,6 +2960,7 @@ jobs:
     environment:
       LITELLM_MIGRATION_TESTS: "1"
       LITELLM_MIGRATION_TEST_IMAGE: litellm-docker-database:ci
+      LITELLM_MIGRATION_BASELINE_IMAGE: << pipeline.parameters.migration_baseline_image >>
       MIGRATION_TEST_ADMIN_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres
       MIGRATION_TEST_CONTAINER_ADMIN_URL: postgresql://postgres:postgres@host.docker.internal:5432/postgres
       MIGRATION_TEST_OUTPUT: /tmp/migration-results
@@ -2980,6 +2988,16 @@ jobs:
       - wait_for_service:
           url: tcp://localhost:5432
           timeout: "60"
+      - when:
+          condition: << parameters.baseline >>
+          steps:
+            - run:
+                name: Pull the baseline release the upgrade starts from
+                environment:
+                  BASELINE_IMAGE: << pipeline.parameters.migration_baseline_image >>
+                command: |
+                  [[ "$BASELINE_IMAGE" =~ ^ghcr.io/berriai/[a-z0-9._/-]+(@sha256:[0-9a-f]{64}|:v[0-9][0-9a-z.-]*)$ ]] || exit 1
+                  docker pull "$BASELINE_IMAGE"
       - run:
           name: Run migration startup regressions
           environment:
@@ -3032,28 +3050,29 @@ jobs:
       - run:
           name: Run Docker container with bad DATABASE_URL
           command: |
+            set +e
             docker run --name my-app \
               -p 4000:4000 \
               -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
               -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
               -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
               myapp:latest \
-              --port 4000 > docker_output.log 2>&1 || true
+              --port 4000 > docker_output.log 2>&1
+            echo "$?" > docker_exit_code
+            set -e
       - run:
           name: Display Docker logs
           command: cat docker_output.log
       - run:
-          name: Check for expected error
+          name: Proxy must refuse to serve on an unreachable database
           command: |
-            if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \
-               (grep -q "Database setup failed after multiple retries" docker_output.log || \
-                grep -q "ERROR:    Application startup failed. Exiting." docker_output.log); then
-              echo "Expected error found. Test passed."
-            else
-              echo "Expected error not found. Test failed."
-              cat docker_output.log
-              exit 1
-            fi
+            fail() { echo "FAILED: $1"; cat docker_output.log; exit 1; }
+            exit_code="$(cat docker_exit_code)"
+            [ "$exit_code" -ne 0 ] || fail "proxy exited 0 with an unreachable database"
+            grep -q "P1001" docker_output.log || fail "log does not name the unreachable database server"
+            ! grep -q "Application startup complete" docker_output.log || fail "proxy reached serving state"
+            ! docker exec my-app true 2>/dev/null || fail "container is still running"
+            echo "Proxy refused to serve (exit $exit_code) and never reached startup. Test passed."
 
   provider_replay_harness:
     docker:
@@ -3187,6 +3206,16 @@ workflows:
           name: migration-legacy-and-pooling
           suite: legacy
           requires: [build_docker_database_image]
+      - migration_startup_tests:
+          name: migration-upgrade
+          suite: upgrade
+          baseline: true
+          requires: [build_docker_database_image]
+      - migration_startup_tests:
+          name: migration-upgrade-shaped
+          suite: shaped
+          baseline: true
+          requires: [build_docker_database_image]
   migration_startup_scheduled:
     triggers:
       - schedule:

.circleci/scripts/run_integration.sh

@@ -121,6 +121,10 @@ start_proxy() {
       "LITELLM_MODEL_COST_MAP_URL=$INTEGRATION_UPSTREAM_URL/_cost_map"
       "MODEL_COST_MAP_MIN_MODEL_COUNT=1"
       "MODEL_COST_MAP_MAX_SHRINK_RATIO=0"
+      "GEMINI_API_BASE=$INTEGRATION_UPSTREAM_URL"
+      "ANTHROPIC_API_BASE=$INTEGRATION_UPSTREAM_URL"
+      "GEMINI_API_KEY=sk-scripted-provider"
+      "ANTHROPIC_API_KEY=sk-scripted-provider"
     )
   else
     cost_map_env=("LITELLM_LOCAL_MODEL_COST_MAP=True")

.circleci/scripts/run_migration_tests.py

@@ -13,6 +13,8 @@ SUITES: Final = {
     "startup": (("test_startup.py",), 12),
     "recovery": (("test_recovery.py",), 15),
     "legacy": (("test_legacy.py", "test_pooling.py"), 11),
+    "upgrade": (("test_upgrade.py", "test_rolling_upgrade.py"), 5),
+    "shaped": (("test_shaped_database.py",), 1),
 }
 
 
@@ -93,6 +95,7 @@ def main() -> int:
             {
                 **metadata,
                 "suite": suite,
+                "baseline_image": os.environ.get("LITELLM_MIGRATION_BASELINE_IMAGE", ""),
                 "expected_cases": expected,
                 "passed": passed,
                 "pytest_exit_code": result.returncode,

.github/actions/cache-cargo-build/action.yml

@@ -15,6 +15,12 @@ description: >-
   cache the same directory for different workloads, and a shared key would let
   whichever ran first deny the others a save.
 
+inputs:
+  profile:
+    description: "Cargo profile the build uses (dev or release)"
+    required: false
+    default: "dev"
+
 runs:
   using: composite
   steps:
@@ -25,6 +31,6 @@ runs:
           ~/.cargo/registry
           ~/.cargo/git
           litellm-rust/target
-        key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }}
+        key: ${{ runner.os }}-maturin-${{ inputs.profile }}-${{ hashFiles('litellm-rust/Cargo.lock') }}
         restore-keys: |
-          ${{ runner.os }}-maturin-dev-
+          ${{ runner.os }}-maturin-${{ inputs.profile }}-

.github/e2e-stack/redact_output.py

@@ -0,0 +1,83 @@
+import argparse
+import os
+import sys
+from functools import reduce
+from pathlib import Path
+from typing import Final
+from xml.sax.saxutils import escape
+
+from pydantic import JsonValue, TypeAdapter, ValidationError
+from secrets_to_env import MIN_MASKED_LENGTH
+
+REDACTED: Final = "***"
+json_adapter: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
+
+
+def string_leaves(node: JsonValue) -> tuple[str, ...]:
+    match node:
+        case str():
+            return (node,)
+        case list():
+            return tuple(leaf for child in node for leaf in string_leaves(child))
+        case dict():
+            return tuple(leaf for child in node.values() for leaf in string_leaves(child))
+    return ()
+
+
+def field_lines(value: str) -> tuple[str, ...]:
+    try:
+        return tuple(line for leaf in string_leaves(json_adapter.validate_json(value)) for line in leaf.splitlines())
+    except ValidationError:
+        return ()
+
+
+def masked_values(values_files: tuple[Path, ...]) -> tuple[str, ...]:
+    values: Final = frozenset(
+        line.split("=", 1)[1].strip().strip("'")
+        for path in values_files
+        for line in path.read_text().splitlines()
+        if "=" in line
+    )
+    texts: Final = frozenset(text for value in values for text in (value, *field_lines(value)))
+    renderings: Final = frozenset(
+        rendering
+        for text in texts
+        if len(text) >= MIN_MASKED_LENGTH
+        for rendering in (text, escape(text), escape(text, {'"': """}))
+    )
+    return tuple(sorted(renderings, key=lambda rendering: (-len(rendering), rendering)))
+
+
+def redact(text: str, values: tuple[str, ...]) -> str:
+    return reduce(lambda redacted, value: redacted.replace(value, REDACTED), values, text)
+
+
+def write_redacted(source: Path, out_dir: Path, values: tuple[str, ...]) -> None:
+    target: Final = out_dir / source.name
+    with os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "w") as handle:
+        _ = handle.write(redact(source.read_text(errors="replace"), values))
+
+
+def main() -> int:
+    parser: Final = argparse.ArgumentParser()
+    _ = parser.add_argument("--values", action="append", type=Path, required=True)
+    _ = parser.add_argument("--out", type=Path, required=True)
+    _ = parser.add_argument("files", nargs="*", type=Path)
+    args: Final = parser.parse_args()
+    values_files: Final = tuple(args.values)
+    out_dir: Final[Path] = args.out
+    sources: Final = tuple(args.files)
+    try:
+        values: Final = masked_values(values_files)
+        out_dir.mkdir(mode=0o700, exist_ok=True)
+        for source in sources:
+            write_redacted(source, out_dir, values)
+    except OSError as error:
+        _ = sys.stderr.write(f"could not redact {error.filename}\n")
+        return 1
+    _ = sys.stdout.write(f"redacted {len(sources)} file(s) into {out_dir}\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/secrets_to_env.py

@@ -9,6 +9,7 @@ from pydantic import TypeAdapter, ValidationError
 secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
 ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
 MIN_MASKED_LENGTH: Final = 8
+ACTIONS_RUNNER_FLAG: Final = "GITHUB_ACTIONS"
 
 
 def main() -> int:
@@ -30,10 +31,15 @@ def main() -> int:
             f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n"
         )
         return 1
-    for value in secrets.values():
-        if len(value) >= MIN_MASKED_LENGTH:
-            _ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n")
-    sys.stdout.flush()
+    if os.environ.get(ACTIONS_RUNNER_FLAG) == "true":
+        _ = sys.stdout.write(
+            "".join(
+                f"::add-mask::{value.replace('%', '%25')}\n"
+                for value in secrets.values()
+                if len(value) >= MIN_MASKED_LENGTH
+            )
+        )
+        sys.stdout.flush()
     lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
     try:
         with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle:

.github/e2e-stack/select_tests.py

@@ -9,6 +9,7 @@ UNSUPPORTED: Final = re.compile(
     r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
     r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
     r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
+    r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$"
 )
 HARNESS: Final = re.compile(
     r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"

.github/e2e-stack/up.sh

@@ -143,7 +143,7 @@ env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/m
 
 start_server() {
   local name="$1"; shift
-  env "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
+  env -u AWS_ROLE_NAME "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
   echo $! > "${PIDS_DIR}/${name}.pid"
 }
 

.github/scripts/verify_linux_native_wheel.py

@@ -134,7 +134,16 @@ def main(
         uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members)
         native_path: Final = wheel.parent / "native" / Path(native_member.filename).name
         native_path.parent.mkdir(parents=True, exist_ok=True)
-        native_path.write_bytes(archive.read(native_member))
+        native_bytes: Final = archive.read(native_member)
+        native_path.write_bytes(native_bytes)
+        duplicated_vocabularies: Final = tuple(
+            member.filename
+            for member in wheel_members
+            if member.filename.startswith("litellm/litellm_core_utils/tokenizers/")
+            and re.fullmatch(r"[0-9a-f]{40}", PurePosixPath(member.filename).name)
+            and member.file_size > 0
+            and archive.read(member) in native_bytes
+        )
 
     wheel_metadata_tags_match: Final = (
         len(wheel_metadata_tags) == len(expanded_filename_tags)
@@ -205,7 +214,7 @@ def main(
     native_module: Final = load_native_module(native_path)
     native_module_loads: Final = native_module is not None
     panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
-    native_size_limit: Final = 25_000_000
+    native_size_limit: Final = 35_000_000
     native_size_within_limit: Final = native_member.file_size <= native_size_limit
     validations: Final = (
         (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
@@ -222,7 +231,8 @@ def main(
         ("Python extension entry point is present", extension_entry_point_present),
         ("Native module loads", native_module_loads),
         ("Production module omits the panic test hook", panic_test_hook_absent),
-        ("Native extension does not exceed 25 MB", native_size_within_limit),
+        (f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit),
+        ("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies),
         ("Wheel contents are valid", not unexpected_members),
     )
 
@@ -267,7 +277,8 @@ def main(
             ),
             (
                 not native_size_within_limit,
-                f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB",
+                f"native extension exceeds {native_size_limit / 1_000_000:.0f} MB: "
+                f"{native_member.file_size / 1_000_000:.2f} MB",
             ),
             (bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"),
         )

.github/workflows/codspeed.yml

@@ -13,6 +13,7 @@ on:
       - ".github/workflows/codspeed.yml"
       - ".github/actions/setup-uv-with-retries/**"
       - ".github/actions/cache-cargo-build/**"
+      - ".github/scripts/uv_sync_with_retries.sh"
   pull_request:
     branches:
       - main
@@ -25,6 +26,7 @@ on:
       - ".github/workflows/codspeed.yml"
       - ".github/actions/setup-uv-with-retries/**"
       - ".github/actions/cache-cargo-build/**"
+      - ".github/scripts/uv_sync_with_retries.sh"
   # Allow CodSpeed to trigger backtest performance analysis
   # in order to generate initial data
   workflow_dispatch:
@@ -59,19 +61,27 @@ jobs:
 
       - name: Cache the Rust build
         uses: ./.github/actions/cache-cargo-build
+        with:
+          profile: release
 
       # Build the wheel and resolve every dependency outside the CodSpeed
       # runner: the same maturin build took 42 minutes inside `codspeed run`
       # versus under 3 minutes as a plain step (LIT-6183)
-      - name: Build environment
+      - name: Build the release wheel
+        run: uv build --wheel --out-dir dist
+
+      - name: Install the wheel into the benchmark environment
+        run: |
+          UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/benchmark-venv" .github/scripts/uv_sync_with_retries.sh --frozen --no-default-groups --group benchmarks --no-install-project --python 3.12
+          uv pip install --python "${RUNNER_TEMP}/benchmark-venv/bin/python" --no-deps dist/*.whl
+
+      - name: Collect benchmarks
+        env:
+          PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
+          LITELLM_REQUIRE_INSTALLED_WHEEL: "1"
         run: >
-          env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
-          uv run --frozen --no-default-groups
-          --with pytest==8.3.5
-          --with pytest-codspeed==4.3.0
-          --with "mcp>=2.2.0,<3.0"
-          --with "a2a-sdk>=1.1.0,<2.0"
-          pytest
+          "${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
+          --import-mode=importlib
           -p pytest_codspeed.plugin
           tests/benchmarks/
           --codspeed
@@ -82,13 +92,9 @@ jobs:
         with:
           mode: simulation
           run: >
-            env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
-            uv run --frozen --no-default-groups
-            --with pytest==8.3.5
-            --with pytest-codspeed==4.3.0
-            --with "mcp>=2.2.0,<3.0"
-            --with "a2a-sdk>=1.1.0,<2.0"
-            pytest
+            env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 LITELLM_REQUIRE_INSTALLED_WHEEL=1
+            "${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
+            --import-mode=importlib
             -p pytest_codspeed.plugin
             tests/benchmarks/
             --codspeed

.github/workflows/test-e2e-changed.yml

@@ -175,6 +175,8 @@ jobs:
         env:
           TESTS: ${{ needs.detect.outputs.tests }}
           E2E_FIXTURE_MODE: live
+          E2E_PROVIDER_EDGE_HOST_REACHABLE: '1'
+          COLUMNS: '400'
         run: |
           umask 077
           read -r -a test_files <<< "${TESTS}"
@@ -189,6 +191,7 @@ jobs:
             uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"
             verified=$?
             set -e
+            grep -E '^(FAILED|ERROR) ' "${log}" || true
             grep -E '^=+ .* in [0-9.]+s( \([0-9:]+\))? =+$' "${log}" | tail -n 1
             echo "::endgroup::"
             if [ "${status}" = "5" ]; then
@@ -206,6 +209,24 @@ jobs:
             echo "pass ${pass} of 3 passed"
           done
 
+      - name: Redact the pytest output
+        if: always() && steps.boot.outcome == 'success'
+        run: |
+          umask 077
+          shopt -s nullglob
+          uv run --no-sync python .github/e2e-stack/redact_output.py \
+            --values tests/e2e/.env --values "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" \
+            --out "${RUNNER_TEMP}/e2e-redacted" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
+
+      - name: Keep the redacted pytest output
+        if: always() && steps.boot.outcome == 'success'
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: e2e-changed-pytest-output-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/e2e-redacted
+          retention-days: 14
+          if-no-files-found: ignore
+
       - name: Stop the stack
         if: always() && steps.boot.outcome != 'skipped'
         run: bash .github/e2e-stack/down.sh
@@ -214,7 +235,7 @@ jobs:
         if: always()
         run: |
           rm -f tests/e2e/.env "${RUNNER_TEMP}/e2e-boot.log" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
-          rm -rf "${RUNNER_TEMP}/litellm-e2e-stack"
+          rm -rf "${RUNNER_TEMP}/litellm-e2e-stack" "${RUNNER_TEMP}/e2e-redacted"
 
   gate:
     name: e2e-changed-tests

.github/workflows/test-linting.yml

@@ -130,6 +130,10 @@ jobs:
           echo "File content around line 43:"
           head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
 
+      - name: Check MCP operation boundary
+        if: steps.changes.outputs.decision != 'skip'
+        run: uv run --no-sync python scripts/check_mcp_operation_boundary.py
+
       - name: Run Ruff linting
         if: steps.changes.outputs.decision != 'skip'
         run: |

.github/workflows/test-rust.yml

@@ -130,7 +130,7 @@ jobs:
       - name: Test secret manager feature combinations
         run: |
           cargo test -p litellm-auth-gcp --locked --no-default-features
-          for features in '' aws google aws,google; do
+          for features in '' aws google hashicorp azure cyberark aws,google aws,azure google,azure aws,google,azure aws,google,cyberark aws,google,azure,cyberark aws,google,hashicorp,azure,cyberark; do
             cargo test -p litellm-secrets --locked --no-default-features --features "$features"
           done
 

Makefile

@@ -164,6 +164,7 @@ lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
 
 # Linting targets
 lint-ruff: $(LINT_DEP_INSTALL)
+	$(UV_RUN) python scripts/check_mcp_operation_boundary.py
 	cd litellm && $(UV_RUN) ruff check . && cd ..
 	$(UV_RUN) ruff check --config ruff-tests.toml tests
 

README.md

@@ -307,6 +307,7 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse
 | [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ |  |  | ✅ |  |  |  |  |
 | [DeepInfra (`deepinfra`)](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |
 | [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ |  |  |  |
 | [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ |  | ✅ |  |  |  |  |  |
@@ -356,7 +357,7 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse
 | [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Pinstripes (`pinstripes`)](https://docs.litellm.ai/docs/providers/pinstripes) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Predibase (`predibase`)](https://docs.litellm.ai/docs/providers/predibase) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
-| [Qwen AI Platform (`qwen_ai_platform`)](https://docs.litellm.ai/docs/providers/qwencloud) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  | ✅ |
+| [Qianwen AI Platform (`qwen_ai_platform`)](https://docs.litellm.ai/docs/providers/qwencloud) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  | ✅ |
 | [QwenCloud (`qwencloud`)](https://docs.litellm.ai/docs/providers/qwencloud) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  | ✅ |
 | [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) |  |  |  |  | ✅ |  |  |  |  |  |
 | [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |

backend/Dockerfile

@@ -61,6 +61,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra saml \
         --python python3.13
 
+RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
+
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
     npm_config_cache=/root/.npm \
     prisma generate --schema=./schema.prisma

backend/routes/allowlist.py

@@ -51,6 +51,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/cache_settings",
     "/coordination_redis/",
     "/cost_tracking",
+    "/cost_optimization/",
     "/cost/",
     "/credentials",
     "/credential",

ci_cd/cost_map_guard.py

@@ -1,8 +1,11 @@
 """Guard the cost map on pull requests.
 
-Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file,
-and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named
-litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models.
+Every pull request whose diff against its merge base touches one of the three cost map files gets the file
+checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the
+map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies
+and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot
+(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three
+files and may only add or update models.
 """
 
 from __future__ import annotations
@@ -108,20 +111,37 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
     )
 
 
+def touches_cost_map(changed_files: Sequence[str]) -> bool:
+    return any(path in GUARDED_PATHS for path in changed_files)
+
+
+def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
+    if bot:
+        return "bot contract enforced"
+    return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
+
+
 def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]:
+    if not bot and not touches_cost_map(changed_files):
+        return ()
     head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH)
     if isinstance(head_map, str):
         return (head_map,)
     return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ()))
 
 
-def _git(*args: str) -> str:
+def _git(*args: str) -> str | None:
     result: Final = subprocess.run(("git", *args), check=False, capture_output=True, text=True)
-    return result.stdout if result.returncode == 0 else ""
+    return result.stdout if result.returncode == 0 else None
 
 
 def snapshot(revision: str) -> Snapshot:
-    return Snapshot(*(_git("show", f"{revision}:{path}") for path in GUARDED_PATHS))
+    return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS))
+
+
+def changed_files(base: str, head: str) -> tuple[str, ...] | None:
+    diff: Final = _git("diff", "--name-only", "--no-renames", base, head)
+    return None if diff is None else tuple(diff.splitlines())
 
 
 def main(argv: Sequence[str]) -> int:
@@ -131,9 +151,12 @@ def main(argv: Sequence[str]) -> int:
     parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
     args: Final = parser.parse_args(argv)
     bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
-    changed_files: Final = tuple(_git("diff", "--name-only", args.base, args.head).splitlines())
-    failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed_files, bot)
-    contract: Final = "bot contract enforced" if bot else "human PR, file checks only"
+    changed: Final = changed_files(args.base, args.head)
+    if changed is None:
+        print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown")
+        return 1
+    failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot)
+    contract: Final = contract_for(bot, changed)
     if failures:
         print(f"cost map guard failed ({contract}):")
         print("\n".join(f"- {failure}" for failure in failures))

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json

@@ -6267,6 +6267,63 @@
       ],
       "title": "Spend update queue sizes (litellm_<queue>_size)",
       "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "description": "Requests that carried usage but were logged at $0 on a model whose pricing entry has a non-zero rate, by requested model and reason",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineWidth": 1,
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 430
+      },
+      "id": 110,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(litellm_zero_cost_requests_total[$__rate_interval])) by (requested_model, reason)",
+          "legendFormat": "{{requested_model}} / {{reason}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "litellm_zero_cost_requests rate",
+      "type": "timeseries"
     }
   ],
   "preload": false,

docker/README.md

@@ -2,6 +2,17 @@
 
 This guide provides instructions for building and running the LiteLLM application using Docker and Docker Compose.
 
+> **Just want to run LiteLLM?** This guide builds from source. To run the published
+> image instead, use `docker-compose.quickstart.yml` in this directory — the
+> two-service stack (gateway + Postgres) that the
+> [Docker quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start) documents:
+>
+> ```bash
+> curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml
+> printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env
+> docker compose -f docker-compose.quickstart.yml up -d
+> ```
+
 ## Prerequisites
 
 - Docker

docker/docker-compose.quickstart.yml

@@ -0,0 +1,41 @@
+# LiteLLM quickstart stack: the gateway plus a Postgres database that stores
+# models, virtual keys, and spend logs. Used by
+# https://docs.litellm.ai/docs/proxy/docker_quick_start
+#
+#   curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml
+#   printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env
+#   docker compose -f docker-compose.quickstart.yml up -d
+#
+# Compose reads .env from this directory. Keep it: regenerating LITELLM_SALT_KEY
+# makes credentials already stored in the database unreadable. For anything
+# beyond local evaluation, pin the image to a specific release tag.
+services:
+  litellm:
+    image: docker.litellm.ai/berriai/litellm:main-stable
+    ports:
+      - "4000:4000"
+    environment:
+      LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?set it in .env - see the header of this file}
+      LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?set it in .env - see the header of this file}
+      DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
+      STORE_MODEL_IN_DB: "True"
+    depends_on:
+      db:
+        condition: service_healthy
+
+  db:
+    image: postgres:16
+    environment:
+      POSTGRES_USER: litellm
+      POSTGRES_PASSWORD: litellm
+      POSTGRES_DB: litellm
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U litellm"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+
+volumes:
+  postgres_data:

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -2,10 +2,11 @@
 Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if the cost has been tracked.
 """
 
+from collections.abc import Sequence
 from dataclasses import replace as dataclasses_replace
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, List, Literal, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tuple, cast
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -18,8 +19,8 @@ if TYPE_CHECKING:
     from prisma import models as prisma_models
 
     from litellm.integrations.prometheus import PrometheusLogger
-    from litellm.proxy._types import LiteLLM_ManagedObjectTable
     from litellm.proxy.utils import PrismaClient, ProxyLogging
+    from litellm.repositories.prisma_protocols import TableActions
     from litellm.router import Router
     from litellm.types.router import Deployment
     from litellm.types.utils import LiteLLMBatch
@@ -41,6 +42,42 @@ TERMINAL_MANAGED_OBJECT_STATUSES: Final[Tuple[str, ...]] = (
 )
 
 
+class _ManagedObjectRow(Protocol):
+    @property
+    def id(self) -> str: ...
+
+    @property
+    def unified_object_id(self) -> str: ...
+
+    @property
+    def created_by(self) -> str | None: ...
+
+    @property
+    def file_object(self) -> object: ...
+
+
+def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]":
+    table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable
+    return table
+
+
+def _user_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_UserTable]":
+    table: Final[TableActions[prisma_models.LiteLLM_UserTable]] = prisma_client.db.litellm_usertable
+    return table
+
+
+def _token_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_VerificationToken]":
+    table: Final[TableActions[prisma_models.LiteLLM_VerificationToken]] = (
+        prisma_client.db.litellm_verificationtoken
+    )
+    return table
+
+
+def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_TeamTable]":
+    table: Final[TableActions[prisma_models.LiteLLM_TeamTable]] = prisma_client.db.litellm_teamtable
+    return table
+
+
 class CheckBatchCost:
     def __init__(
         self,
@@ -73,7 +110,7 @@ class CheckBatchCost:
         inline for a batch the first poll cycle then accounts again.
         """
         try:
-            await self.prisma_client.db.litellm_managedobjecttable.find_first(
+            await _managed_object_table(self.prisma_client).find_first(
                 where={"file_purpose": "batch", "batch_processed": False}
             )
         except Exception as probe_err:
@@ -97,10 +134,8 @@ class CheckBatchCost:
         if not user_id:
             return {}
         try:
-            user_row: prisma_models.LiteLLM_UserTable | None = (
-                await self.prisma_client.db.litellm_usertable.find_unique(
-                    where={"user_id": user_id}
-                )
+            user_row: prisma_models.LiteLLM_UserTable | None = await _user_table(self.prisma_client).find_unique(
+                where={"user_id": user_id}
             )
             if user_row is None:
                 return {}
@@ -117,11 +152,9 @@ class CheckBatchCost:
         if not api_key:
             return None
         try:
-            key_row: prisma_models.LiteLLM_VerificationToken | None = (
-                await self.prisma_client.db.litellm_verificationtoken.find_unique(
-                    where={"token": api_key}
-                )
-            )
+            key_row: prisma_models.LiteLLM_VerificationToken | None = await _token_table(
+                self.prisma_client
+            ).find_unique(where={"token": api_key})
             return getattr(key_row, "key_alias", None) if key_row is not None else None
         except Exception as e:
             verbose_proxy_logger.error(f"CheckBatchCost: could not look up key alias for batch {batch_id}: {e}")
@@ -132,17 +165,15 @@ class CheckBatchCost:
         if not team_id:
             return None
         try:
-            team_row: prisma_models.LiteLLM_TeamTable | None = (
-                await self.prisma_client.db.litellm_teamtable.find_unique(
-                    where={"team_id": team_id}
-                )
+            team_row: prisma_models.LiteLLM_TeamTable | None = await _team_table(self.prisma_client).find_unique(
+                where={"team_id": team_id}
             )
             return getattr(team_row, "team_alias", None) if team_row is not None else None
         except Exception as e:
             verbose_proxy_logger.error(f"CheckBatchCost: could not look up team alias for team {team_id}: {e}")
             return None
 
-    async def _get_org_id(self, job: "LiteLLM_ManagedObjectTable", batch_id: str) -> str | None:
+    async def _get_org_id(self, job: "_ManagedObjectRow", batch_id: str) -> str | None:
         org_id = getattr(job, "org_id", None)
         if org_id:
             return org_id
@@ -150,11 +181,9 @@ class CheckBatchCost:
         team_id = getattr(job, "team_id", None)
         if api_key:
             try:
-                key_row: prisma_models.LiteLLM_VerificationToken | None = (
-                    await self.prisma_client.db.litellm_verificationtoken.find_unique(
-                        where={"token": api_key}
-                    )
-                )
+                key_row: prisma_models.LiteLLM_VerificationToken | None = await _token_table(
+                    self.prisma_client
+                ).find_unique(where={"token": api_key})
                 key_org_id = getattr(key_row, "organization_id", None) if key_row is not None else None
                 if key_org_id:
                     return key_org_id
@@ -166,10 +195,8 @@ class CheckBatchCost:
         if not team_id:
             return None
         try:
-            team_row: prisma_models.LiteLLM_TeamTable | None = (
-                await self.prisma_client.db.litellm_teamtable.find_unique(
-                    where={"team_id": team_id}
-                )
+            team_row: prisma_models.LiteLLM_TeamTable | None = await _team_table(self.prisma_client).find_unique(
+                where={"team_id": team_id}
             )
             return getattr(team_row, "organization_id", None) if team_row is not None else None
         except Exception as e:
@@ -177,7 +204,7 @@ class CheckBatchCost:
             return None
 
     async def _build_creator_attribution_metadata(
-        self, job: "LiteLLM_ManagedObjectTable", batch_id: str
+        self, job: "_ManagedObjectRow", batch_id: str
     ) -> dict[str, object]:
         """
         Rebuild the spend-tracking metadata for the key, team, and tags that created the
@@ -225,7 +252,7 @@ class CheckBatchCost:
         should not be polled.
         """
         cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
-        result: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
+        result: Final = await _managed_object_table(self.prisma_client).update_many(
             where={
                 "file_purpose": "batch",
                 "status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)},
@@ -244,7 +271,7 @@ class CheckBatchCost:
 
         # A row already in a terminal status is never rewritten by the sweep above, so
         # without this it keeps a poll-page slot forever and starves newer batches.
-        retired: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
+        retired: Final = await _managed_object_table(self.prisma_client).update_many(
             where={
                 "file_purpose": "batch",
                 "batch_processed": False,
@@ -259,9 +286,9 @@ class CheckBatchCost:
                 f"{MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days that were never costed"
             )
 
-    async def _fallback_find_jobs(self) -> list:
+    async def _fallback_find_jobs(self) -> "Sequence[_ManagedObjectRow]":
         """Query batch jobs without the batch_processed filter (for older schemas)."""
-        return await self.prisma_client.db.litellm_managedobjecttable.find_many(
+        return await _managed_object_table(self.prisma_client).find_many(
             where={
                 "file_purpose": "batch",
                 "status": {
@@ -279,7 +306,7 @@ class CheckBatchCost:
             order={"created_at": "asc"},
         )
 
-    async def _retire_job(self, job: "LiteLLM_ManagedObjectTable", reason: str) -> None:
+    async def _retire_job(self, job: "_ManagedObjectRow", reason: str) -> None:
         """
         Take a row that can never be costed out of the poll page. Leaving it selectable
         would burn one of the MAX_OBJECTS_PER_POLL_CYCLE slots on every future cycle, and
@@ -292,7 +319,7 @@ class CheckBatchCost:
             else {"status": "stale_expired"}
         )
         try:
-            await self.prisma_client.db.litellm_managedobjecttable.update(
+            await _managed_object_table(self.prisma_client).update(
                 where={"id": job.id},
                 data=data,
             )
@@ -306,7 +333,7 @@ class CheckBatchCost:
             "so it will no longer be polled"
         )
 
-    async def _claim_job_for_costing(self, job: "LiteLLM_ManagedObjectTable") -> bool:
+    async def _claim_job_for_costing(self, job: "_ManagedObjectRow") -> bool:
         """
         Atomically flip batch_processed from false to true, returning whether this pod won
         the row. Every pod and uvicorn worker schedules its own poller against the shared
@@ -321,7 +348,7 @@ class CheckBatchCost:
         if not self._has_batch_processed_column:
             return True
         try:
-            claimed: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
+            claimed: Final = await _managed_object_table(self.prisma_client).update_many(
                 where={"id": job.id, "batch_processed": False},
                 data={"batch_processed": True},
             )
@@ -332,7 +359,7 @@ class CheckBatchCost:
             return False
         return claimed > 0
 
-    async def _release_job_claim(self, job: "LiteLLM_ManagedObjectTable") -> None:
+    async def _release_job_claim(self, job: "_ManagedObjectRow") -> None:
         """Give a claimed row back once billing it failed, so a later poll cycle retries it.
 
         Safe to match on batch_processed=True: while this poller is active the retrieve
@@ -342,7 +369,7 @@ class CheckBatchCost:
         if not self._has_batch_processed_column:
             return
         try:
-            await self.prisma_client.db.litellm_managedobjecttable.update_many(
+            await _managed_object_table(self.prisma_client).update_many(
                 where={"id": job.id, "batch_processed": True},
                 data={"batch_processed": False},
             )
@@ -353,7 +380,7 @@ class CheckBatchCost:
             )
 
     @staticmethod
-    def _has_unified_id_without_model(job: "LiteLLM_ManagedObjectTable") -> bool:
+    def _has_unified_id_without_model(job: "_ManagedObjectRow") -> bool:
         """A unified id that decodes but carries no model_id can never be routed."""
         from litellm.proxy.openai_files_endpoints.common_utils import (
             convert_b64_uid_to_unified_uid,
@@ -402,7 +429,7 @@ class CheckBatchCost:
         return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
 
     async def _finalize_unbilled_terminal_job(
-        self, job: "prisma_models.LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
+        self, job: "_ManagedObjectRow", response: "LiteLLMBatch"
     ) -> None:
         """Persist a terminal batch that has nothing billable, converting any raw
         provider file ids to managed ids, and take it out of the poll page."""
@@ -426,7 +453,7 @@ class CheckBatchCost:
                 "file_object": response.model_dump_json(),
                 **({"batch_processed": True} if self._has_batch_processed_column else {}),
             }
-            await self.prisma_client.db.litellm_managedobjecttable.update(
+            await _managed_object_table(self.prisma_client).update(
                 where={"id": job.id},
                 data=update_data,
             )
@@ -447,7 +474,7 @@ class CheckBatchCost:
 
     def _resolve_job_routing(
         self,
-        job: "LiteLLM_ManagedObjectTable",
+        job: "_ManagedObjectRow",
         prom_logger: Optional["PrometheusLogger"],
     ) -> Optional[Tuple[str, str]]:
         """
@@ -524,7 +551,7 @@ class CheckBatchCost:
 
     def _resolve_unmanaged_provider_routing(
         self,
-        job: "LiteLLM_ManagedObjectTable",
+        job: "_ManagedObjectRow",
         prom_logger: Optional["PrometheusLogger"],
         llm_provider: str,
         bare_model_name: str,
@@ -620,7 +647,7 @@ class CheckBatchCost:
     @classmethod
     def _get_managed_file_model_name(
         cls,
-        job: "LiteLLM_ManagedObjectTable",
+        job: "_ManagedObjectRow",
         deployment_info: "Deployment",
     ) -> Optional[str]:
         """
@@ -640,7 +667,7 @@ class CheckBatchCost:
         )
 
     @staticmethod
-    def _get_input_file_id(job: "LiteLLM_ManagedObjectTable") -> Optional[str]:
+    def _get_input_file_id(job: "_ManagedObjectRow") -> Optional[str]:
         import json
 
         from litellm.types.utils import LiteLLMBatch
@@ -660,7 +687,7 @@ class CheckBatchCost:
 
     async def _track_completed_batch_cost(
         self,
-        job: "LiteLLM_ManagedObjectTable",
+        job: "_ManagedObjectRow",
         response: "LiteLLMBatch",
         model_id: str,
         batch_id: str,
@@ -936,7 +963,7 @@ class CheckBatchCost:
                 # endpoint may transition a batch to "complete" before
                 # CheckBatchCost runs.  The batch_processed=False filter
                 # already prevents reprocessing finished batches.
-                jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many(
+                jobs = await _managed_object_table(self.prisma_client).find_many(
                     where={
                         "file_purpose": "batch",
                         "batch_processed": False,
@@ -1038,7 +1065,7 @@ class CheckBatchCost:
                     }
                     if self._has_batch_processed_column:
                         update_data["batch_processed"] = True
-                    await self.prisma_client.db.litellm_managedobjecttable.update(
+                    await _managed_object_table(self.prisma_client).update(
                         where={"id": job.id},
                         data=update_data,
                     )

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -6,7 +6,7 @@ same route are non-inference and free.
 """
 
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Dict, Optional, cast
+from typing import TYPE_CHECKING, Dict, Final, Optional, Protocol, cast
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -22,11 +22,31 @@ from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
 
 if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient, ProxyLogging
+    from litellm.repositories.prisma_protocols import TableActions
     from litellm.router import Router
 
 TERMINAL_RESPONSE_STATUSES = frozenset({"completed", "failed", "cancelled", "incomplete"})
 
 
+class _ManagedObjectRow(Protocol):
+    @property
+    def id(self) -> str: ...
+
+    @property
+    def unified_object_id(self) -> str: ...
+
+    @property
+    def created_by(self) -> str | None: ...
+
+    @property
+    def file_object(self) -> object: ...
+
+
+def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]":
+    table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable
+    return table
+
+
 class CheckResponsesCost:
     def __init__(
         self,
@@ -128,7 +148,7 @@ class CheckResponsesCost:
                 f"CheckResponsesCost: stale cleanup failed (poll will continue): {cleanup_err}"
             )
 
-        jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many(
+        jobs = await _managed_object_table(self.prisma_client).find_many(
             where={
                 "status": {"in": ["queued", "in_progress"]},
                 "file_purpose": "response",
@@ -138,7 +158,7 @@ class CheckResponsesCost:
         )
         
         verbose_proxy_logger.debug(f"Found {len(jobs)} response jobs to check")
-        completed_jobs = []
+        completed_jobs: Final[list[_ManagedObjectRow]] = []
 
         for job in jobs:
             unified_object_id = job.unified_object_id
@@ -189,7 +209,7 @@ class CheckResponsesCost:
 
         # Mark completed jobs in the database
         if len(completed_jobs) > 0:
-            await self.prisma_client.db.litellm_managedobjecttable.update_many(
+            await _managed_object_table(self.prisma_client).update_many(
                 where={"id": {"in": [job.id for job in completed_jobs]}},
                 data={"status": "completed"},
             )

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -481,10 +481,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         """
         if self.prisma_client is None:
             return
-        managed_object = (
-            await self.prisma_client.db.litellm_managedobjecttable.find_first(
-                where={"OR": [{"unified_object_id": object_id}, {"model_object_id": object_id}]}
-            )
+        managed_object = await _managed_object_table(self.prisma_client).find_first(
+            where={"OR": [{"unified_object_id": object_id}, {"model_object_id": object_id}]}
         )
         if managed_object is None:
             return
@@ -509,10 +507,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         """
         if self.prisma_client is None:
             return
-        managed_file = (
-            await self.prisma_client.db.litellm_managedfiletable.find_first(
-                where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]}
-            )
+        managed_file = await _managed_file_table(self.prisma_client).find_first(
+            where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]}
         )
         if managed_file is None:
             return
@@ -535,8 +531,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         provider_file_ids = tuple(
             file_id
             for file_id in (
-                getattr(response, "output_file_id", None),
-                getattr(response, "error_file_id", None),
+                response.output_file_id,
+                response.error_file_id,
             )
             if file_id and not _is_base64_encoded_unified_file_id(file_id)
         )
@@ -544,10 +540,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             return
         if self.prisma_client is None:
             return
-        batch_row = (
-            await self.prisma_client.db.litellm_managedobjecttable.find_first(
-                where={"unified_object_id": response.id}
-            )
+        batch_row = await _managed_object_table(self.prisma_client).find_first(
+            where={"unified_object_id": response.id}
         )
         if batch_row is None or (
             batch_row.created_by is None and batch_row.team_id is None

gateway/routes/allowlist.py

@@ -96,16 +96,19 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/assemblyai/",
     "/eu.assemblyai/",
     "/deepgram/",
+    "/fal_ai/",
     "/langfuse/",
     "/vllm/",
     "/mistral/",
     "/typesafe/",
+    "/openrouter/",
     "/nvidia_nim/",
     "/groq/",
     "/voyage/",
     "/cursor/",
     "/milvus/",
     "/openai_passthrough/",
+    "/tinyfish/",
     # Dynamic provider / toolset passthrough (path templates)
     "/{provider}/",
     "/toolset/",

helm/litellm/templates/backend/deployment.yaml

@@ -7,6 +7,9 @@ metadata:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: backend
 spec:
+  {{- if and (not .Values.backend.hpa.enabled) (not (kindIs "invalid" .Values.backend.replicaCount)) }}
+  replicas: {{ .Values.backend.replicaCount }}
+  {{- end }}
   {{- with .Values.backend.strategy }}
   strategy:
     {{- toYaml . | nindent 4 }}

helm/litellm/templates/gateway/deployment.yaml

@@ -7,6 +7,9 @@ metadata:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: gateway
 spec:
+  {{- if and (not .Values.gateway.hpa.enabled) (not (kindIs "invalid" .Values.gateway.replicaCount)) }}
+  replicas: {{ .Values.gateway.replicaCount }}
+  {{- end }}
   {{- with .Values.gateway.strategy }}
   strategy:
     {{- toYaml . | nindent 4 }}

helm/litellm/templates/ui/deployment.yaml

@@ -7,6 +7,9 @@ metadata:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: ui
 spec:
+  {{- if and (not .Values.ui.hpa.enabled) (not (kindIs "invalid" .Values.ui.replicaCount)) }}
+  replicas: {{ .Values.ui.replicaCount }}
+  {{- end }}
   {{- with .Values.ui.strategy }}
   strategy:
     {{- toYaml . | nindent 4 }}

helm/litellm/tests/replica_count_tests.yaml

@@ -0,0 +1,100 @@
+suite: test fixed replica count when HPA is disabled
+templates:
+  - gateway/deployment.yaml
+  - gateway/configmap.yaml
+  - backend/deployment.yaml
+  - ui/deployment.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: gateway renders replicaCount into spec.replicas when its HPA is disabled
+    template: gateway/deployment.yaml
+    set:
+      gateway.hpa.enabled: false
+      gateway.replicaCount: 3
+    asserts:
+      - isKind:
+          of: Deployment
+      - equal:
+          path: spec.replicas
+          value: 3
+
+  - it: backend renders replicaCount into spec.replicas when its HPA is disabled
+    template: backend/deployment.yaml
+    set:
+      backend.hpa.enabled: false
+      backend.replicaCount: 2
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 2
+
+  - it: ui renders replicaCount into spec.replicas when its HPA is disabled
+    template: ui/deployment.yaml
+    set:
+      ui.hpa.enabled: false
+      ui.replicaCount: 2
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 2
+
+  - it: replicaCount 0 scales the gateway to zero instead of being treated as unset
+    template: gateway/deployment.yaml
+    set:
+      gateway.hpa.enabled: false
+      gateway.replicaCount: 0
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 0
+
+  - it: a component with HPA disabled but no replicaCount set keeps omitting spec.replicas, so upgrades do not reset a hand-scaled Deployment
+    set:
+      gateway.hpa.enabled: false
+      backend.hpa.enabled: false
+      ui.hpa.enabled: false
+    asserts:
+      - notExists:
+          path: spec.replicas
+        template: gateway/deployment.yaml
+      - notExists:
+          path: spec.replicas
+        template: backend/deployment.yaml
+      - notExists:
+          path: spec.replicas
+        template: ui/deployment.yaml
+
+  - it: every component omits spec.replicas when its HPA is enabled, so the autoscaler owns the count
+    set:
+      gateway.hpa.enabled: true
+      gateway.replicaCount: 3
+      backend.hpa.enabled: true
+      backend.replicaCount: 3
+      ui.hpa.enabled: true
+      ui.replicaCount: 3
+    asserts:
+      - notExists:
+          path: spec.replicas
+        template: gateway/deployment.yaml
+      - notExists:
+          path: spec.replicas
+        template: backend/deployment.yaml
+      - notExists:
+          path: spec.replicas
+        template: ui/deployment.yaml
+
+  - it: a component with HPA disabled renders replicas while a sibling with HPA enabled does not
+    set:
+      gateway.hpa.enabled: false
+      gateway.replicaCount: 4
+      backend.hpa.enabled: true
+      backend.replicaCount: 4
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 4
+        template: gateway/deployment.yaml
+      - notExists:
+          path: spec.replicas
+        template: backend/deployment.yaml

helm/litellm/values.yaml

@@ -397,6 +397,11 @@ gateway:
   #   failureThreshold: 30
   #   periodSeconds: 10
   startupProbe: {}
+  # Optional fixed pod count, rendered into the Deployment's spec.replicas only
+  # when hpa.enabled is false. Unset by default so an existing Deployment keeps
+  # its current count; with the HPA on, the autoscaler owns the count, e.g.:
+  #   replicaCount: 3
+  replicaCount:
   hpa:
     enabled: true
     minReplicas: 1
@@ -524,6 +529,8 @@ backend:
   strategy: {}
   # Optional startupProbe; same shape as gateway.startupProbe. Empty by default.
   startupProbe: {}
+  # Same semantics as gateway.replicaCount.
+  replicaCount:
   hpa:
     enabled: true
     minReplicas: 1
@@ -590,6 +597,8 @@ ui:
   strategy: {}
   # Optional startupProbe; same shape as gateway.startupProbe. Empty by default.
   startupProbe: {}
+  # Same semantics as gateway.replicaCount.
+  replicaCount:
   hpa:
     enabled: false
     minReplicas: 1

litellm-proxy-extras/litellm_proxy_extras/migrations/20260917000000_add_agent_access_group_ids/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN IF NOT EXISTS "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[];

litellm-proxy-extras/litellm_proxy_extras/migrations/20260919000000_add_autorouter_user_session_rollup/migration.sql

@@ -0,0 +1,42 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_AutoRouterUserSession" (
+    "user_id" TEXT NOT NULL,
+    "api_key" TEXT NOT NULL,
+    "session_id" TEXT NOT NULL,
+    "router_name" TEXT NOT NULL,
+    "router_type" TEXT NOT NULL,
+    "first_turn_at" TIMESTAMP(3) NOT NULL,
+    "last_turn_at" TIMESTAMP(3) NOT NULL,
+    "last_model" TEXT NOT NULL,
+    "models" JSONB NOT NULL DEFAULT '{}',
+    "turns" INTEGER NOT NULL DEFAULT 0,
+    "unordered_turns" INTEGER NOT NULL DEFAULT 0,
+    "covered_turns" INTEGER NOT NULL DEFAULT 0,
+    "cache_hits" INTEGER NOT NULL DEFAULT 0,
+    "same_model_turns" INTEGER NOT NULL DEFAULT 0,
+    "same_model_hits" INTEGER NOT NULL DEFAULT 0,
+    "first_visit_turns" INTEGER NOT NULL DEFAULT 0,
+    "first_visit_hits" INTEGER NOT NULL DEFAULT 0,
+    "return_turns" INTEGER NOT NULL DEFAULT 0,
+    "return_hits" INTEGER NOT NULL DEFAULT 0,
+    "return_expired_misses" INTEGER NOT NULL DEFAULT 0,
+    "return_within_ttl_misses" INTEGER NOT NULL DEFAULT 0,
+    "ttl_5m_turns" INTEGER NOT NULL DEFAULT 0,
+    "ttl_1h_turns" INTEGER NOT NULL DEFAULT 0,
+    "total_tokens" BIGINT NOT NULL DEFAULT 0,
+    "spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "savings_estimated_turns" INTEGER NOT NULL DEFAULT 0,
+    "savings_estimated_actual_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "savings_estimated_saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "savings_estimated_baseline_models" JSONB NOT NULL DEFAULT '{}',
+    "classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "classifier_cost_recorded_turns" INTEGER NOT NULL DEFAULT 0,
+    "tier_turns" JSONB NOT NULL DEFAULT '{}',
+    "baseline_models" JSONB NOT NULL DEFAULT '{}',
+
+    CONSTRAINT "LiteLLM_AutoRouterUserSession_pkey" PRIMARY KEY ("user_id", "api_key", "session_id", "router_name")
+);
+
+CREATE INDEX IF NOT EXISTS "idx_autorouter_user_session_last_turn" ON "LiteLLM_AutoRouterUserSession"("last_turn_at");
+
+CREATE INDEX IF NOT EXISTS "idx_autorouter_user_session_user_last_turn" ON "LiteLLM_AutoRouterUserSession"("user_id", "last_turn_at");

litellm-proxy-extras/litellm_proxy_extras/migrations/20260920041500_add_policy_attachment_is_default/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "LiteLLM_PolicyAttachmentTable" ADD COLUMN IF NOT EXISTS "is_default" BOOLEAN NOT NULL DEFAULT false;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260921000000_add_password_reset_columns/migration.sql

@@ -0,0 +1,3 @@
+ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "password_reset_required" BOOLEAN;
+
+ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "last_breach_check_at" TIMESTAMP(3);

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -73,6 +73,7 @@ model LiteLLM_AgentsTable {
   static_headers       Json?          @default("{}")
   extra_headers        String[]       @default([])
   agent_access_groups String[]       @default([])
+  access_group_ids     String[]       @default([])
   object_permission_id String?
   object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
   spend      Float    @default(0.0)
@@ -246,6 +247,8 @@ model LiteLLM_UserTable {
     organization_id String?
     object_permission_id String?
     password  String?
+    password_reset_required Boolean?
+    last_breach_check_at DateTime?
     teams    String[] @default([])
     user_role  String?
 		max_budget Float?
@@ -1419,6 +1422,7 @@ model LiteLLM_PolicyAttachmentTable {
   models           String[]  @default([])  // Model names or patterns
   tags             String[]  @default([])  // Tag patterns (e.g., ["healthcare", "prod-*"])
   priority         Int?                    // Explicit execution order
+  is_default       Boolean   @default(false)  // Applied only when no non-default attachment matches
   created_at       DateTime  @default(now())
   created_by       String?
   updated_at       DateTime  @default(now()) @updatedAt
@@ -1620,6 +1624,47 @@ model LiteLLM_AutoRouterSession {
   @@index([last_turn_at], map: "idx_autorouter_session_last_turn")
 }
 
+model LiteLLM_AutoRouterUserSession {
+  user_id               String
+  api_key               String
+  session_id            String
+  router_name           String
+  router_type           String
+  first_turn_at         DateTime
+  last_turn_at          DateTime
+  last_model            String
+  models                Json     @default("{}")
+  turns                 Int      @default(0)
+  unordered_turns       Int      @default(0)
+  covered_turns         Int      @default(0)
+  cache_hits            Int      @default(0)
+  same_model_turns      Int      @default(0)
+  same_model_hits       Int      @default(0)
+  first_visit_turns     Int      @default(0)
+  first_visit_hits      Int      @default(0)
+  return_turns          Int      @default(0)
+  return_hits           Int      @default(0)
+  return_expired_misses Int      @default(0)
+  return_within_ttl_misses  Int      @default(0)
+  ttl_5m_turns          Int      @default(0)
+  ttl_1h_turns          Int      @default(0)
+  total_tokens          BigInt   @default(0)
+  spend                 Float    @default(0)
+  saved_spend           Float    @default(0)
+  savings_estimated_turns Int    @default(0)
+  savings_estimated_actual_spend Float @default(0)
+  savings_estimated_saved_spend Float @default(0)
+  savings_estimated_baseline_models Json @default("{}")
+  classifier_cost       Float    @default(0)
+  classifier_cost_recorded_turns Int @default(0)
+  tier_turns            Json     @default("{}")
+  baseline_models       Json     @default("{}")
+
+  @@id([user_id, api_key, session_id, router_name])
+  @@index([last_turn_at], map: "idx_autorouter_user_session_last_turn")
+  @@index([user_id, last_turn_at], map: "idx_autorouter_user_session_user_last_turn")
+}
+
 // Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in
 // either direction. forward duplicates the requests the keys did not route through the
 // router through it, answering whether they should adopt it; reverse duplicates the

litellm-rust/Cargo.toml

@@ -22,12 +22,23 @@ litellm-secrets = { path = "crates/secrets" }
 litellm-secrets-types = { path = "crates/secrets-types" }
 litellm-secrets-aws = { path = "crates/secrets-aws" }
 litellm-secrets-google = { path = "crates/secrets-google" }
+litellm-secrets-hashicorp = { path = "crates/secrets-hashicorp" }
+litellm-secrets-azure = { path = "crates/secrets-azure" }
+litellm-secrets-cyberark = { path = "crates/secrets-cyberark" }
 litellm-http = { path = "crates/http" }
 litellm-llms = { path = "crates/llms" }
 litellm-types = { path = "crates/types" }
 litellm-core-utils = { path = "crates/core-utils" }
 litellm-cache = { path = "crates/cache" }
+litellm-cache-azure-blob = { path = "crates/cache-azure-blob" }
 litellm-cache-memory = { path = "crates/cache-memory" }
+litellm-cache-redis = { path = "crates/cache-redis" }
+litellm-cache-s3 = { path = "crates/cache-s3" }
+litellm-cache-gcs = { path = "crates/cache-gcs" }
+litellm-cache-disk = { path = "crates/cache-disk" }
+litellm-cache-redis-semantic = { path = "crates/cache-redis-semantic" }
+litellm-cache-response = { path = "crates/cache-response" }
+litellm-cache-qdrant-semantic = { path = "crates/cache-qdrant-semantic" }
 litellm-token-counter = { path = "crates/token-counter" }
 litellm-token-counter-fast = { path = "crates/token-counter-fast" }
 litellm-token-counter-huggingface = { path = "crates/token-counter-huggingface" }
@@ -45,9 +56,14 @@ pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] }
 pythonize = "0.29.0"
 rand = "0.8"
 reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls", "http2", "stream"] }
+qdrant-client = { version = "1.19.0", default-features = false }
+uuid = { version = "1", features = ["v4"] }
 rstest = "0.26.1"
 rstest_reuse = "0.7.0"
 rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] }
+rustify = "=0.7.0"
+rustify_derive = "=0.5.5"
+vaultrs = { version = "=0.8.0", default-features = false, features = ["rustls"] }
 rustls-native-certs = "0.8"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["float_roundtrip"] }
@@ -64,6 +80,7 @@ base64 = "0.22"
 moka = { version = "0.12.16", features = ["future"] }
 strum = { version = "0.28.0", features = ["derive"] }
 url = "2.5.8"
+percent-encoding = "2.3"
 webpki-roots = "1"
 time = { version = "0.3.53", features = ["parsing"] }
 criterion = "0.8.2"
@@ -72,7 +89,7 @@ veil = "0.3.0"
 
 [profile.release]
 opt-level = 3
-lto = "thin"
+lto = "fat"
 codegen-units = 1
 panic = "unwind"
 debug = false

litellm-rust/crates/auth-aws/src/aws.rs

@@ -621,6 +621,26 @@ mod tests {
         None
     }
 
+    #[test]
+    fn secret_names_cover_environment_reads() {
+        let seen = std::sync::Arc::new(std::sync::Mutex::new(
+            std::collections::BTreeSet::<String>::new(),
+        ));
+        let recorded = seen.clone();
+        let env = |name: &str| {
+            recorded.lock().unwrap().insert(name.to_string());
+            None
+        };
+        resolve_aws_region(None, &Map::new(), &env);
+        aws_auth_config(&Map::new(), &env);
+        assert!(
+            seen.lock()
+                .unwrap()
+                .iter()
+                .all(|name| crate::constants::SECRET_NAMES.contains(&name.as_str()))
+        );
+    }
+
     #[test]
     fn a_region_comes_from_the_call_then_the_model_then_the_environment() {
         let params = Map::from_iter([("aws_region_name".to_string(), Value::from("eu-west-1"))]);

litellm-rust/crates/auth-aws/src/constants.rs

@@ -14,6 +14,19 @@ pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
 pub const AWS_STS_ENDPOINT: &str = "AWS_STS_ENDPOINT";
 pub const AWS_EXTERNAL_ID: &str = "AWS_EXTERNAL_ID";
 pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK";
+pub const SECRET_NAMES: &[&str] = &[
+    AWS_ACCESS_KEY_ID,
+    AWS_SECRET_ACCESS_KEY,
+    AWS_SESSION_TOKEN,
+    AWS_REGION_NAME,
+    AWS_REGION,
+    AWS_SESSION_NAME,
+    AWS_PROFILE_NAME,
+    AWS_ROLE_NAME,
+    AWS_WEB_IDENTITY_TOKEN,
+    AWS_STS_ENDPOINT,
+    AWS_EXTERNAL_ID,
+];
 
 /// Headers SigV4 covers, beyond the `x-amz-` / `x-amzn-` prefixes. Mirrors
 /// Python's `_filter_headers_for_aws_signature` allowlist.

litellm-rust/crates/auth-azure/src/lib.rs

@@ -3,5 +3,5 @@ mod native;
 mod resolve;
 mod types;
 
-pub use resolve::AzureAuthService;
-pub use types::AzureAuthInputs;
+pub use resolve::{AzureAuthService, SECRET_NAMES};
+pub use types::{AzureAuthInputs, ConfigValue};

litellm-rust/crates/auth-azure/src/resolve.rs

@@ -19,6 +19,17 @@ const AZURE_AUTHORITY_HOST_ENV: &str = "AZURE_AUTHORITY_HOST";
 const AZURE_CREDENTIAL_ENV: &str = "AZURE_CREDENTIAL";
 const AZURE_FEDERATED_TOKEN_FILE_ENV: &str = "AZURE_FEDERATED_TOKEN_FILE";
 
+pub const SECRET_NAMES: &[&str] = &[
+    AZURE_AD_TOKEN_ENV,
+    AZURE_TENANT_ID_ENV,
+    AZURE_CLIENT_ID_ENV,
+    AZURE_CLIENT_SECRET_ENV,
+    AZURE_SCOPE_ENV,
+    AZURE_AUTHORITY_HOST_ENV,
+    AZURE_CREDENTIAL_ENV,
+    AZURE_FEDERATED_TOKEN_FILE_ENV,
+];
+
 #[derive(Clone, Debug)]
 pub(crate) enum AzureCredentialPlan {
     Supplied(Sourced<ResolvedCredential>),
@@ -440,13 +451,14 @@ fn non_empty_reference(value: &str, kind: &str) -> Result<String, Error> {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::BTreeSet;
     use std::future::Future;
     use std::sync::{Arc, Mutex};
 
     use serde_json::json;
 
     use super::{
-        AzureAuthService, AzureCredentialPlan, AzureTokenAcquirer, oidc_reference,
+        AzureAuthService, AzureCredentialPlan, AzureTokenAcquirer, SECRET_NAMES, oidc_reference,
         resolve_reference, select_auth_plan,
     };
     use crate::native::ValidatedAzureRequest;
@@ -517,6 +529,24 @@ mod tests {
         assert!(matches!(plan, AzureCredentialPlan::Native(_)));
     }
 
+    #[test]
+    fn secret_names_cover_environment_reads() {
+        let seen = std::sync::Arc::new(std::sync::Mutex::new(BTreeSet::<String>::new()));
+        let recorded = seen.clone();
+        let inputs = AzureAuthInputs::default();
+        select_auth_plan(&inputs, &|name| {
+            recorded.lock().unwrap().insert(name.to_string());
+            None
+        })
+        .unwrap();
+        assert!(
+            seen.lock()
+                .unwrap()
+                .iter()
+                .all(|name| SECRET_NAMES.contains(&name.as_str()))
+        );
+    }
+
     #[test]
     fn supplied_token_does_not_require_refresh() {
         let params = json!({"azure_ad_token": "token"});

litellm-rust/crates/auth-azure/src/types.rs

@@ -51,6 +51,21 @@ pub struct AzureAuthInputs {
 }
 
 impl AzureAuthInputs {
+    pub fn default_credential_for_scope(scope: &str) -> Self {
+        Self {
+            azure_scope: ConfigValue::Value(Sourced::new(
+                scope.to_string(),
+                InputSource::Deployment,
+            )),
+            azure_credential: ConfigValue::Value(Sourced::new(
+                "DefaultAzureCredential".to_string(),
+                InputSource::Deployment,
+            )),
+            enable_azure_ad_token_refresh: Sourced::new(true, InputSource::Deployment),
+            ..Self::default()
+        }
+    }
+
     pub fn or_configured_token_refresh(self, enabled: bool) -> Self {
         if *self.enable_azure_ad_token_refresh.value() || !enabled {
             return self;

litellm-rust/crates/auth-gcp/src/lib.rs

@@ -23,6 +23,16 @@ const VERTEXAI_PROJECT_ENV: &str = "VERTEXAI_PROJECT";
 const VERTEXAI_LOCATION_ENV: &str = "VERTEXAI_LOCATION";
 const VERTEX_LOCATION_ENV: &str = "VERTEX_LOCATION";
 
+pub const SECRET_NAMES: &[&str] = &[
+    VERTEX_AI_API_KEY_ENV,
+    VERTEXAI_API_KEY_ENV,
+    VERTEXAI_CREDENTIALS_ENV,
+    GOOGLE_APPLICATION_CREDENTIALS_ENV,
+    VERTEXAI_PROJECT_ENV,
+    VERTEXAI_LOCATION_ENV,
+    VERTEX_LOCATION_ENV,
+];
+
 #[derive(Clone, Debug, Default)]
 pub struct VertexConfig {
     credentials: Option<Sourced<SecretValue>>,
@@ -128,6 +138,14 @@ impl VertexAuth {
         }
     }
 
+    pub async fn access_token(
+        &self,
+        config: &VertexConfig,
+        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
+    ) -> Result<String, Error> {
+        self.load_provider(config, env_lookup).await?.token().await
+    }
+
     pub async fn validate_environment(
         &self,
         headers: Vec<(String, String)>,
@@ -398,6 +416,7 @@ fn auth_acquisition_error(error: gcp_auth::Error) -> Error {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::BTreeSet;
     use std::sync::atomic::{AtomicUsize, Ordering};
 
     use serde_json::json;
@@ -468,6 +487,27 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn secret_names_cover_environment_reads() {
+        let seen = Arc::new(std::sync::Mutex::new(BTreeSet::<String>::new()));
+        let recorded = seen.clone();
+        let env = |name: &str| {
+            recorded.lock().unwrap().insert(name.to_string());
+            None
+        };
+        let auth = auth(Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
+        auth.validate_environment(Vec::new(), None, &VertexConfig::default(), &env)
+            .await
+            .unwrap();
+        get_vertex_ai_location(&VertexConfig::default(), &env);
+        assert!(
+            seen.lock()
+                .unwrap()
+                .iter()
+                .all(|name| SECRET_NAMES.contains(&name.as_str()))
+        );
+    }
+
     #[test]
     fn empty_primary_values_fall_back_to_python_aliases() {
         let config = config(json!({

litellm-rust/crates/auth-types/src/secret.rs

@@ -1,4 +1,5 @@
 use serde::Deserialize;
+use std::hash::{Hash, Hasher};
 use veil::Redact;
 
 #[derive(Redact, Clone, Deserialize)]
@@ -23,6 +24,12 @@ impl PartialEq for SecretValue {
 
 impl Eq for SecretValue {}
 
+impl Hash for SecretValue {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.0.hash(state);
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::SecretValue;

litellm-rust/crates/cache-azure-blob/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "litellm-cache-azure-blob"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-auth-azure.workspace = true
+litellm-auth-types.workspace = true
+litellm-cache.workspace = true
+
+async-trait = "0.1"
+azure_core = "1.1.0"
+azure_storage_blob = "1.1.0"
+futures-util.workspace = true
+tokio.workspace = true
+url.workspace = true
+
+[dev-dependencies]
+litellm-cache-response.workspace = true
+serde_json.workspace = true

litellm-rust/crates/cache-azure-blob/src/cache.rs

@@ -0,0 +1,254 @@
+use std::{sync::Arc, time::Duration};
+
+use azure_core::{
+    credentials::TokenCredential,
+    error::ErrorKind,
+    http::{ClientOptions, RequestContent},
+};
+use azure_storage_blob::{
+    BlobContainerClient, BlobContainerClientOptions,
+    models::{BlobClientUploadOptions, StorageErrorCode},
+};
+use futures_util::{TryStreamExt, future::try_join_all};
+use litellm_cache::{
+    BaseCache, BatchCache, CacheCodec, CacheConnectionResult, CacheConnectionStatus, Error,
+    ExactCacheContext, FlushCache,
+};
+use tokio::runtime::Handle;
+use url::Url;
+
+use crate::credential::AzureBlobCredential;
+
+pub struct AzureBlobCache<C> {
+    container: BlobContainerClient,
+    codec: C,
+    runtime: Handle,
+    account_url: String,
+    container_name: String,
+}
+
+impl<C: CacheCodec> AzureBlobCache<C> {
+    pub async fn connect(
+        account_url: &str,
+        container: &str,
+        codec: C,
+        runtime: Handle,
+    ) -> Result<Self, Error> {
+        Self::connect_with_options(
+            account_url,
+            container,
+            Some(Arc::new(AzureBlobCredential::default())),
+            ClientOptions::default(),
+            codec,
+            runtime,
+        )
+        .await
+    }
+
+    pub async fn connect_with_options(
+        account_url: &str,
+        container: &str,
+        credential: Option<Arc<dyn TokenCredential>>,
+        client_options: ClientOptions,
+        codec: C,
+        runtime: Handle,
+    ) -> Result<Self, Error> {
+        let parsed = Url::parse(account_url).map_err(|_| Error::Unavailable)?;
+        let account_url = parsed.as_str().trim_end_matches('/').to_string();
+        let container_url = {
+            let mut url = parsed;
+            url.path_segments_mut()
+                .map_err(|()| Error::Unavailable)?
+                .pop_if_empty()
+                .push(container);
+            url
+        };
+        let client = BlobContainerClient::new(
+            container_url,
+            credential,
+            Some(BlobContainerClientOptions {
+                client_options,
+                ..BlobContainerClientOptions::default()
+            }),
+        )
+        .map_err(|_| Error::Unavailable)?;
+        let cache = Self {
+            container: client,
+            codec,
+            runtime,
+            account_url,
+            container_name: container.to_string(),
+        };
+        cache.create_container().await?;
+        Ok(cache)
+    }
+
+    pub fn account_url(&self) -> &str {
+        &self.account_url
+    }
+
+    pub fn container_name(&self) -> &str {
+        &self.container_name
+    }
+
+    async fn create_container(&self) -> Result<(), Error> {
+        match self.container.create(None).await {
+            Ok(_) => Ok(()),
+            Err(error) if is_storage_error(&error, StorageErrorCode::ContainerAlreadyExists) => {
+                Ok(())
+            }
+            Err(_) => Err(Error::Unavailable),
+        }
+    }
+
+    async fn upload(&self, key: &str, value: &C::Value, overwrite: bool) -> Result<(), Error> {
+        let payload = self.codec.encode(value)?;
+        let options = (!overwrite).then(|| BlobClientUploadOptions::default().if_not_exists());
+        match self
+            .container
+            .blob_client(key)
+            .upload(RequestContent::from(payload), options)
+            .await
+        {
+            Ok(_) => Ok(()),
+            Err(error) if !overwrite && is_already_present(&error) => Ok(()),
+            Err(_) => Err(Error::Unavailable),
+        }
+    }
+
+    async fn download(&self, key: &str) -> Result<Option<C::Value>, Error> {
+        let response = match self.container.blob_client(key).download(None).await {
+            Ok(response) => response,
+            Err(error) if is_storage_error(&error, StorageErrorCode::BlobNotFound) => {
+                return Ok(None);
+            }
+            Err(_) => return Err(Error::Unavailable),
+        };
+        let bytes = response
+            .body
+            .collect()
+            .await
+            .map_err(|_| Error::Unavailable)?;
+        self.codec.decode(&bytes).map(Some)
+    }
+
+    async fn delete_all_blobs(&self) -> Result<(), Error> {
+        let mut pages = self
+            .container
+            .list_blobs(None)
+            .map_err(|_| Error::Unavailable)?
+            .into_pages();
+        while let Some(page) = pages.try_next().await.map_err(|_| Error::Unavailable)? {
+            let page = page.into_model().map_err(|_| Error::Unavailable)?;
+            for name in page.blob_items.into_iter().filter_map(|item| item.name) {
+                self.container
+                    .blob_client(&name)
+                    .delete(None)
+                    .await
+                    .map_err(|_| Error::Unavailable)?;
+            }
+        }
+        Ok(())
+    }
+
+    fn block_on<T>(&self, future: impl Future<Output = T>) -> T {
+        self.runtime.block_on(future)
+    }
+}
+
+fn is_already_present(error: &azure_core::Error) -> bool {
+    is_storage_error(error, StorageErrorCode::BlobAlreadyExists)
+        || is_storage_error(error, StorageErrorCode::ConditionNotMet)
+}
+
+fn is_storage_error(error: &azure_core::Error, code: StorageErrorCode) -> bool {
+    matches!(
+        error.kind(),
+        ErrorKind::HttpResponse {
+            error_code: Some(error_code),
+            ..
+        } if error_code == code.as_ref()
+    )
+}
+
+impl<C: CacheCodec> BaseCache for AzureBlobCache<C> {
+    type Value = C::Value;
+    type Context = ExactCacheContext;
+
+    fn get_ttl(&self, _: &ExactCacheContext) -> Option<Duration> {
+        None
+    }
+
+    fn set_cache(&self, key: &str, value: C::Value, _: &ExactCacheContext) -> Result<(), Error> {
+        self.block_on(self.upload(key, &value, false))
+    }
+
+    fn get_cache(&self, key: &str, _: &ExactCacheContext) -> Result<Option<C::Value>, Error> {
+        self.block_on(self.download(key))
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: C::Value,
+        _: ExactCacheContext,
+    ) -> Result<(), Error> {
+        self.upload(key, &value, true).await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        _: &ExactCacheContext,
+    ) -> Result<Option<C::Value>, Error> {
+        self.download(key).await
+    }
+
+    async fn async_set_cache_pipeline(
+        &self,
+        entries: Vec<(String, C::Value)>,
+        _: ExactCacheContext,
+    ) -> Result<(), Error> {
+        try_join_all(
+            entries
+                .iter()
+                .map(|(key, value)| self.upload(key, value, true)),
+        )
+        .await
+        .map(drop)
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        Ok(match self.container.get_properties(None).await {
+            Ok(_) => CacheConnectionResult {
+                status: CacheConnectionStatus::Success,
+                message: "Azure Blob cache connection test successful".into(),
+                error: None,
+            },
+            Err(error) => CacheConnectionResult {
+                status: CacheConnectionStatus::Failed,
+                message: format!("Azure Blob connection failed: {error}"),
+                error: Some(error.to_string()),
+            },
+        })
+    }
+}
+
+impl<C: CacheCodec> BatchCache for AzureBlobCache<C> {}
+
+impl<C: CacheCodec> FlushCache for AzureBlobCache<C> {
+    fn flush_cache(&self) -> Result<(), Error> {
+        self.block_on(self.delete_all_blobs())
+    }
+
+    async fn async_flush_cache(&self) -> Result<(), Error> {
+        self.delete_all_blobs().await
+    }
+}
+
+#[cfg(test)]
+mod tests;

litellm-rust/crates/cache-azure-blob/src/cache/tests.rs

@@ -0,0 +1,746 @@
+use std::{
+    collections::BTreeMap,
+    sync::{Arc, Mutex},
+    time::Duration,
+};
+
+use azure_core::http::{
+    AsyncRawResponse, Body, ClientOptions, HttpClient, Method, Request, StatusCode, Transport,
+    headers::{HeaderName, Headers},
+};
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheConnectionStatus, Error, ExactCacheContext, FlushCache,
+};
+use litellm_cache_response::{
+    CacheEntry, CacheKeyField, CacheKeyInput, ResponseCache, ResponseCacheCodec,
+    ResponseCacheRequest, cache_key,
+};
+use serde_json::json;
+use tokio::runtime::Runtime;
+
+use super::AzureBlobCache;
+
+const ACCOUNT_URL: &str = "https://example.blob.core.windows.net";
+const CONTAINER: &str = "litellm-cache";
+const IF_NONE_MATCH: HeaderName = HeaderName::from_static("if-none-match");
+const ERROR_CODE: HeaderName = HeaderName::from_static("x-ms-error-code");
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct RecordedRequest {
+    method: Method,
+    path: String,
+    query: String,
+    if_none_match: Option<String>,
+}
+
+#[derive(Default)]
+struct FakeState {
+    container_exists: bool,
+    blobs: BTreeMap<String, Vec<u8>>,
+    requests: Vec<RecordedRequest>,
+    failing: bool,
+    precondition_conflicts: bool,
+}
+
+#[derive(Clone, Default)]
+struct FakeBlobService {
+    state: Arc<Mutex<FakeState>>,
+}
+
+impl std::fmt::Debug for FakeBlobService {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("FakeBlobService")
+    }
+}
+
+impl FakeBlobService {
+    fn with_existing_container() -> Self {
+        let service = Self::default();
+        service.state.lock().unwrap().container_exists = true;
+        service
+    }
+
+    fn blob(&self, name: &str) -> Option<Vec<u8>> {
+        self.state.lock().unwrap().blobs.get(name).cloned()
+    }
+
+    fn blob_names(&self) -> Vec<String> {
+        self.state.lock().unwrap().blobs.keys().cloned().collect()
+    }
+
+    fn seed_blob(&self, name: &str, bytes: &[u8]) {
+        self.state
+            .lock()
+            .unwrap()
+            .blobs
+            .insert(name.to_string(), bytes.to_vec());
+    }
+
+    fn set_failing(&self, failing: bool) {
+        self.state.lock().unwrap().failing = failing;
+    }
+
+    fn set_precondition_conflicts(&self, enabled: bool) {
+        self.state.lock().unwrap().precondition_conflicts = enabled;
+    }
+
+    fn requests(&self) -> Vec<RecordedRequest> {
+        self.state.lock().unwrap().requests.clone()
+    }
+
+    fn container_exists(&self) -> bool {
+        self.state.lock().unwrap().container_exists
+    }
+
+    fn respond(status: StatusCode, error_code: Option<&str>, body: Vec<u8>) -> AsyncRawResponse {
+        let mut headers = Headers::new();
+        if let Some(code) = error_code {
+            headers.insert(ERROR_CODE, code.to_string());
+        }
+        AsyncRawResponse::from_bytes(status, headers, body)
+    }
+
+    fn list_body(state: &FakeState) -> Vec<u8> {
+        let mut xml = String::from(
+            r#"<?xml version="1.0" encoding="utf-8"?><EnumerationResults ServiceEndpoint="https://example.blob.core.windows.net/" ContainerName="litellm-cache"><Blobs>"#,
+        );
+        for name in state.blobs.keys() {
+            xml.push_str(&format!(
+                "<Blob><Name>{name}</Name><Properties><BlobType>BlockBlob</BlobType></Properties></Blob>"
+            ));
+        }
+        xml.push_str("</Blobs><NextMarker /></EnumerationResults>");
+        xml.into_bytes()
+    }
+}
+
+#[async_trait::async_trait]
+impl HttpClient for FakeBlobService {
+    async fn execute_request(&self, request: &Request) -> azure_core::Result<AsyncRawResponse> {
+        let mut state = self.state.lock().unwrap();
+        let path = request.url().path().to_string();
+        let query = request.url().query().unwrap_or_default().to_string();
+        let if_none_match = request
+            .headers()
+            .get_optional_str(&IF_NONE_MATCH)
+            .map(str::to_owned);
+        state.requests.push(RecordedRequest {
+            method: request.method(),
+            path: path.clone(),
+            query: query.clone(),
+            if_none_match: if_none_match.clone(),
+        });
+        if state.failing {
+            return Ok(Self::respond(
+                StatusCode::Forbidden,
+                Some("AuthorizationFailure"),
+                Vec::new(),
+            ));
+        }
+        let container_path = format!("/{CONTAINER}");
+        let blob_name = path
+            .strip_prefix(&format!("{container_path}/"))
+            .map(str::to_owned);
+        let is_container = path == container_path && query.contains("restype=container");
+        let response = match (request.method(), is_container, blob_name) {
+            (Method::Put, true, None) if state.container_exists => Self::respond(
+                StatusCode::Conflict,
+                Some("ContainerAlreadyExists"),
+                Vec::new(),
+            ),
+            (Method::Put, true, None) => {
+                state.container_exists = true;
+                Self::respond(StatusCode::Created, None, Vec::new())
+            }
+            (Method::Get, true, None) if query.contains("comp=list") => {
+                Self::respond(StatusCode::Ok, None, Self::list_body(&state))
+            }
+            (Method::Get, true, None) if state.container_exists => {
+                Self::respond(StatusCode::Ok, None, Vec::new())
+            }
+            (Method::Get, true, None) => {
+                Self::respond(StatusCode::NotFound, Some("ContainerNotFound"), Vec::new())
+            }
+            (Method::Put, false, Some(name)) => {
+                if if_none_match.as_deref() == Some("*") && state.blobs.contains_key(&name) {
+                    if state.precondition_conflicts {
+                        Self::respond(
+                            StatusCode::PreconditionFailed,
+                            Some("ConditionNotMet"),
+                            Vec::new(),
+                        )
+                    } else {
+                        Self::respond(StatusCode::Conflict, Some("BlobAlreadyExists"), Vec::new())
+                    }
+                } else {
+                    let bytes = match request.body() {
+                        Body::Bytes(bytes) => bytes.to_vec(),
+                        Body::SeekableStream(_) => panic!("unexpected streaming upload"),
+                    };
+                    state.blobs.insert(name, bytes);
+                    Self::respond(StatusCode::Created, None, Vec::new())
+                }
+            }
+            (Method::Get, false, Some(name)) => match state.blobs.get(&name) {
+                Some(bytes) => Self::respond(StatusCode::Ok, None, bytes.clone()),
+                None => Self::respond(StatusCode::NotFound, Some("BlobNotFound"), Vec::new()),
+            },
+            (Method::Delete, false, Some(name)) => match state.blobs.remove(&name) {
+                Some(_) => Self::respond(StatusCode::Accepted, None, Vec::new()),
+                None => Self::respond(StatusCode::NotFound, Some("BlobNotFound"), Vec::new()),
+            },
+            (method, _, _) => panic!("unexpected request {method:?} {path}?{query}"),
+        };
+        Ok(response)
+    }
+}
+
+struct Fixture {
+    runtime: Runtime,
+    service: FakeBlobService,
+    cache: Arc<AzureBlobCache<ResponseCacheCodec>>,
+}
+
+impl Fixture {
+    fn new(service: FakeBlobService) -> Self {
+        let runtime = Runtime::new().unwrap();
+        let cache = runtime
+            .block_on(Self::connect(&service, runtime.handle().clone()))
+            .unwrap();
+        Self {
+            runtime,
+            service,
+            cache: Arc::new(cache),
+        }
+    }
+
+    async fn connect(
+        service: &FakeBlobService,
+        handle: tokio::runtime::Handle,
+    ) -> Result<AzureBlobCache<ResponseCacheCodec>, Error> {
+        AzureBlobCache::connect_with_options(
+            ACCOUNT_URL,
+            CONTAINER,
+            None,
+            ClientOptions {
+                transport: Some(Transport::new(Arc::new(service.clone()))),
+                ..ClientOptions::default()
+            },
+            ResponseCacheCodec,
+            handle,
+        )
+        .await
+    }
+
+    fn response_cache(&self) -> ResponseCache<AzureBlobCache<ResponseCacheCodec>> {
+        ResponseCache::new(self.cache.clone())
+    }
+
+    fn stored_json(&self, key: &str) -> serde_json::Value {
+        serde_json::from_slice(&self.service.blob(key).expect("blob should exist")).unwrap()
+    }
+}
+
+fn request(model: &str) -> ResponseCacheRequest {
+    ResponseCacheRequest::new(CacheKeyInput {
+        fields: vec![CacheKeyField {
+            name: "model".into(),
+            value: Some(model.into()),
+            api_parameter: true,
+            internal_parameter: false,
+        }],
+        preset: None,
+        namespace: None,
+        include_provider_parameters: false,
+    })
+}
+
+fn now() -> Duration {
+    Duration::from_secs(1_700_000_000)
+}
+
+fn entry(value: serde_json::Value) -> CacheEntry {
+    CacheEntry {
+        timestamp: Some(1_700_000_000.5),
+        response: value,
+    }
+}
+
+fn no_ttl() -> ExactCacheContext {
+    ExactCacheContext::default()
+}
+
+fn with_ttl(seconds: u64) -> ExactCacheContext {
+    ExactCacheContext {
+        ttl: Some(Duration::from_secs(seconds)),
+    }
+}
+
+#[test]
+fn connect_creates_the_container_once() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    assert!(fixture.service.container_exists());
+    assert_eq!(
+        fixture.service.requests(),
+        vec![RecordedRequest {
+            method: Method::Put,
+            path: format!("/{CONTAINER}"),
+            query: "restype=container".into(),
+            if_none_match: None,
+        }]
+    );
+    assert_eq!(fixture.cache.account_url(), ACCOUNT_URL);
+    assert_eq!(fixture.cache.container_name(), CONTAINER);
+}
+
+#[test]
+fn connect_accepts_an_existing_container() {
+    let fixture = Fixture::new(FakeBlobService::with_existing_container());
+    assert!(fixture.service.container_exists());
+    assert_eq!(fixture.service.requests().len(), 1);
+}
+
+#[test]
+fn connect_accepts_account_urls_with_trailing_slash() {
+    let runtime = Runtime::new().unwrap();
+    let service = FakeBlobService::default();
+    let cache = runtime
+        .block_on(AzureBlobCache::connect_with_options(
+            "https://example.blob.core.windows.net/",
+            CONTAINER,
+            None,
+            ClientOptions {
+                transport: Some(Transport::new(Arc::new(service.clone()))),
+                ..ClientOptions::default()
+            },
+            ResponseCacheCodec,
+            runtime.handle().clone(),
+        ))
+        .unwrap();
+    assert_eq!(service.requests()[0].path, format!("/{CONTAINER}"));
+    assert_eq!(cache.account_url(), "https://example.blob.core.windows.net");
+}
+
+#[test]
+fn connect_keeps_account_url_query_parameters_on_the_container_path() {
+    let runtime = Runtime::new().unwrap();
+    let service = FakeBlobService::default();
+    runtime
+        .block_on(AzureBlobCache::connect_with_options(
+            "https://example.blob.core.windows.net/?sv=2024-01-01&sig=abc",
+            CONTAINER,
+            None,
+            ClientOptions {
+                transport: Some(Transport::new(Arc::new(service.clone()))),
+                ..ClientOptions::default()
+            },
+            ResponseCacheCodec,
+            runtime.handle().clone(),
+        ))
+        .unwrap();
+    let create = &service.requests()[0];
+    assert_eq!(create.path, format!("/{CONTAINER}"));
+    assert!(create.query.contains("sig=abc"));
+}
+
+#[test]
+fn connect_surfaces_service_failures() {
+    let runtime = Runtime::new().unwrap();
+    let service = FakeBlobService::default();
+    service.set_failing(true);
+    let result = runtime.block_on(Fixture::connect(&service, runtime.handle().clone()));
+    assert!(matches!(result, Err(Error::Unavailable)));
+}
+
+#[test]
+fn sync_set_and_get_round_trip_python_json_shape() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    let value = entry(json!({"choices": [{"message": {"content": "héllo 🌍"}}]}));
+    fixture
+        .cache
+        .set_cache("key-1", value.clone(), &no_ttl())
+        .unwrap();
+
+    assert_eq!(
+        fixture.stored_json("key-1"),
+        json!({
+            "timestamp": 1_700_000_000.5,
+            "response": {"choices": [{"message": {"content": "héllo 🌍"}}]}
+        })
+    );
+    assert_eq!(
+        fixture.cache.get_cache("key-1", &no_ttl()).unwrap(),
+        Some(value)
+    );
+}
+
+#[test]
+fn sync_set_does_not_overwrite_an_existing_blob() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture
+        .cache
+        .set_cache("key", entry(json!({"v": "first"})), &no_ttl())
+        .unwrap();
+    fixture
+        .cache
+        .set_cache("key", entry(json!({"v": "second"})), &no_ttl())
+        .unwrap();
+
+    assert_eq!(
+        fixture.stored_json("key")["response"],
+        json!({"v": "first"})
+    );
+    let uploads: Vec<_> = fixture
+        .service
+        .requests()
+        .into_iter()
+        .filter(|request| request.method == Method::Put && request.path.ends_with("/key"))
+        .collect();
+    assert_eq!(uploads.len(), 2);
+    assert!(
+        uploads
+            .iter()
+            .all(|request| request.if_none_match.as_deref() == Some("*"))
+    );
+}
+
+#[test]
+fn sync_set_treats_a_precondition_conflict_as_an_existing_blob() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture.service.set_precondition_conflicts(true);
+    fixture
+        .cache
+        .set_cache("key", entry(json!({"v": "first"})), &no_ttl())
+        .unwrap();
+    fixture
+        .cache
+        .set_cache("key", entry(json!({"v": "second"})), &no_ttl())
+        .unwrap();
+
+    assert_eq!(
+        fixture.stored_json("key")["response"],
+        json!({"v": "first"})
+    );
+}
+
+#[test]
+fn async_set_overwrites_an_existing_blob() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture.runtime.block_on(async {
+        fixture
+            .cache
+            .async_set_cache("key", entry(json!({"v": "first"})), no_ttl())
+            .await
+            .unwrap();
+        fixture
+            .cache
+            .async_set_cache("key", entry(json!({"v": "second"})), no_ttl())
+            .await
+            .unwrap();
+        assert_eq!(
+            fixture
+                .cache
+                .async_get_cache("key", &no_ttl())
+                .await
+                .unwrap(),
+            Some(entry(json!({"v": "second"})))
+        );
+    });
+    assert_eq!(
+        fixture.stored_json("key")["response"],
+        json!({"v": "second"})
+    );
+    assert!(
+        fixture
+            .service
+            .requests()
+            .iter()
+            .filter(|request| request.method == Method::Put && request.path.ends_with("/key"))
+            .all(|request| request.if_none_match.is_none())
+    );
+}
+
+#[test]
+fn missing_blobs_are_misses() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    assert_eq!(fixture.cache.get_cache("absent", &no_ttl()).unwrap(), None);
+    assert_eq!(
+        fixture
+            .runtime
+            .block_on(fixture.cache.async_get_cache("absent", &no_ttl()))
+            .unwrap(),
+        None
+    );
+}
+
+#[test]
+fn ttl_is_ignored_and_entries_never_expire() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    assert_eq!(fixture.cache.get_ttl(&with_ttl(1)), None);
+    assert_eq!(fixture.cache.get_ttl(&no_ttl()), None);
+
+    fixture
+        .cache
+        .set_cache("key", entry(json!("value")), &with_ttl(1))
+        .unwrap();
+    std::thread::sleep(Duration::from_millis(1100));
+    assert_eq!(
+        fixture.cache.get_cache("key", &with_ttl(1)).unwrap(),
+        Some(entry(json!("value")))
+    );
+    assert!(
+        fixture
+            .service
+            .requests()
+            .iter()
+            .all(|request| !request.query.contains("expiry"))
+    );
+}
+
+#[test]
+fn malformed_blobs_are_invalid_entries_and_response_cache_misses() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture.service.seed_blob("broken-json", b"{not json");
+    fixture
+        .service
+        .seed_blob("broken-utf8", &[0xff, 0xfe, 0x22]);
+    fixture
+        .service
+        .seed_blob("wrong-shape", br#"{"timestamp": "yesterday"}"#);
+
+    for key in ["broken-json", "broken-utf8", "wrong-shape"] {
+        assert!(matches!(
+            fixture.cache.get_cache(key, &no_ttl()),
+            Err(Error::InvalidEntry)
+        ));
+    }
+
+    let response_cache = fixture.response_cache();
+    let broken = request("broken");
+    fixture
+        .service
+        .seed_blob(&cache_key(&broken.key), b"{not json");
+    assert_eq!(response_cache.lookup(&broken, now()).unwrap(), None);
+    assert_eq!(
+        fixture
+            .runtime
+            .block_on(response_cache.async_lookup(&broken, now()))
+            .unwrap(),
+        None
+    );
+}
+
+#[test]
+fn batch_get_preserves_order_and_marks_misses_and_invalid_entries() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture
+        .cache
+        .set_cache("a", entry(json!("A")), &no_ttl())
+        .unwrap();
+    fixture
+        .cache
+        .set_cache("c", entry(json!("C")), &no_ttl())
+        .unwrap();
+    fixture.service.seed_blob("bad", b"nope");
+    let keys = ["c", "missing", "a", "bad"].map(String::from);
+
+    let sync = fixture.cache.batch_get_cache(&keys, &no_ttl()).unwrap();
+    assert_eq!(
+        sync,
+        vec![
+            BatchEntry::Hit(entry(json!("C"))),
+            BatchEntry::Miss,
+            BatchEntry::Hit(entry(json!("A"))),
+            BatchEntry::Invalid,
+        ]
+    );
+
+    let asynchronous = fixture
+        .runtime
+        .block_on(fixture.cache.async_batch_get_cache(keys.to_vec(), no_ttl()))
+        .unwrap();
+    assert_eq!(asynchronous, sync);
+
+    let response_cache = fixture.response_cache();
+    let requests = [request("hit"), request("missing"), request("bad")];
+    response_cache
+        .store(&requests[0], json!("HIT"), now())
+        .unwrap();
+    fixture
+        .service
+        .seed_blob(&cache_key(&requests[2].key), b"nope");
+    let hits = response_cache.lookup_batch(&requests, now()).unwrap();
+    assert_eq!(hits.values, vec![Some(json!("HIT")), None, None]);
+    assert_eq!(hits.missing_indices, vec![1, 2]);
+    let async_hits = fixture
+        .runtime
+        .block_on(response_cache.async_lookup_batch(&requests, now()))
+        .unwrap();
+    assert_eq!(async_hits.values, hits.values);
+}
+
+#[test]
+fn async_pipeline_writes_every_entry_with_overwrite() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture.service.seed_blob("k2", b"stale");
+    fixture
+        .runtime
+        .block_on(fixture.cache.async_set_cache_pipeline(
+            vec![
+                ("k1".into(), entry(json!({"n": 1}))),
+                ("k2".into(), entry(json!({"n": 2}))),
+                ("k3".into(), entry(json!({"n": 3}))),
+            ],
+            with_ttl(30),
+        ))
+        .unwrap();
+    assert_eq!(fixture.service.blob_names(), ["k1", "k2", "k3"]);
+    assert_eq!(fixture.stored_json("k2")["response"], json!({"n": 2}));
+}
+
+#[test]
+fn flush_deletes_every_blob_in_the_container() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    for key in ["x", "y", "z"] {
+        fixture
+            .cache
+            .set_cache(key, entry(json!(key)), &no_ttl())
+            .unwrap();
+    }
+    fixture.cache.flush_cache().unwrap();
+    assert!(fixture.service.blob_names().is_empty());
+    assert!(fixture.service.container_exists());
+
+    fixture
+        .cache
+        .set_cache("again", entry(json!(1)), &no_ttl())
+        .unwrap();
+    fixture
+        .runtime
+        .block_on(fixture.cache.async_flush_cache())
+        .unwrap();
+    assert!(fixture.service.blob_names().is_empty());
+}
+
+#[test]
+fn service_failures_map_to_unavailable() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture.service.set_failing(true);
+    assert!(matches!(
+        fixture.cache.get_cache("key", &no_ttl()),
+        Err(Error::Unavailable)
+    ));
+    assert!(matches!(
+        fixture.cache.set_cache("key", entry(json!(1)), &no_ttl()),
+        Err(Error::Unavailable)
+    ));
+    assert!(matches!(
+        fixture.cache.flush_cache(),
+        Err(Error::Unavailable)
+    ));
+    assert!(matches!(
+        fixture.runtime.block_on(
+            fixture
+                .cache
+                .async_set_cache_pipeline(vec![("k".into(), entry(json!(1)))], no_ttl())
+        ),
+        Err(Error::Unavailable)
+    ));
+}
+
+#[test]
+fn test_connection_reports_container_reachability() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    let ok = fixture
+        .runtime
+        .block_on(fixture.cache.test_connection())
+        .unwrap();
+    assert_eq!(ok.status, CacheConnectionStatus::Success);
+    assert!(ok.error.is_none());
+
+    fixture.service.set_failing(true);
+    let failed = fixture
+        .runtime
+        .block_on(fixture.cache.test_connection())
+        .unwrap();
+    assert_eq!(failed.status, CacheConnectionStatus::Failed);
+    assert!(failed.error.is_some());
+}
+
+#[test]
+fn disconnect_is_idempotent_and_keeps_data() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture
+        .cache
+        .set_cache("key", entry(json!(1)), &no_ttl())
+        .unwrap();
+    fixture.runtime.block_on(async {
+        fixture.cache.disconnect().await.unwrap();
+        fixture.cache.disconnect().await.unwrap();
+    });
+    assert_eq!(
+        fixture.cache.get_cache("key", &no_ttl()).unwrap(),
+        Some(entry(json!(1)))
+    );
+}
+
+#[test]
+fn response_cache_stores_and_reads_through_the_backend() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    let response_cache = fixture.response_cache();
+    let mut request = request("gpt");
+    request.context = with_ttl(60);
+    let response = json!({"id": "chatcmpl-1"});
+    response_cache
+        .store(&request, response.clone(), now())
+        .unwrap();
+    assert_eq!(
+        fixture.stored_json(&cache_key(&request.key)),
+        json!({"timestamp": 1_700_000_000.0, "response": {"id": "chatcmpl-1"}})
+    );
+    assert_eq!(
+        response_cache
+            .lookup(&request, now() + Duration::from_secs(3600))
+            .unwrap(),
+        Some(response.clone())
+    );
+    assert_eq!(
+        fixture
+            .runtime
+            .block_on(response_cache.async_lookup(&request, now() + Duration::from_secs(3600)))
+            .unwrap(),
+        Some(response.clone())
+    );
+    fixture.runtime.block_on(async {
+        response_cache
+            .async_store(&request, json!("replaced"), now())
+            .await
+            .unwrap();
+        assert_eq!(
+            response_cache.async_lookup(&request, now()).await.unwrap(),
+            Some(json!("replaced"))
+        );
+        response_cache.async_flush().await.unwrap();
+        assert_eq!(
+            response_cache.async_lookup(&request, now()).await.unwrap(),
+            None
+        );
+    });
+}
+
+#[test]
+fn non_object_responses_are_written_serialized_like_python() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture
+        .cache
+        .set_cache("s", entry(json!("plain")), &no_ttl())
+        .unwrap();
+    assert_eq!(
+        fixture.stored_json("s"),
+        json!({"timestamp": 1_700_000_000.5, "response": "\"plain\""})
+    );
+    assert_eq!(
+        fixture.cache.get_cache("s", &no_ttl()).unwrap(),
+        Some(entry(json!("plain")))
+    );
+}

litellm-rust/crates/cache-azure-blob/src/credential.rs

@@ -0,0 +1,84 @@
+use std::{
+    fmt,
+    sync::Arc,
+    time::{Duration, SystemTime},
+};
+
+use azure_core::{
+    credentials::{AccessToken, TokenCredential, TokenRequestOptions},
+    error::ErrorKind,
+    time::OffsetDateTime,
+};
+use litellm_auth_azure::{AzureAuthInputs, AzureAuthService};
+use litellm_auth_types::ResolvedCredential;
+
+const STATIC_TOKEN_LIFETIME: Duration = Duration::from_secs(300);
+const LLM_TOKEN_ENV: &str = "AZURE_AD_TOKEN";
+
+type EnvLookup = Arc<dyn Fn(&str) -> Option<String> + Send + Sync>;
+
+pub struct AzureBlobCredential {
+    service: AzureAuthService,
+    env_lookup: EnvLookup,
+}
+
+impl fmt::Debug for AzureBlobCredential {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str("AzureBlobCredential")
+    }
+}
+
+impl Default for AzureBlobCredential {
+    fn default() -> Self {
+        Self::new(
+            AzureAuthService::default(),
+            Arc::new(|name| std::env::var(name).ok()),
+        )
+    }
+}
+
+impl AzureBlobCredential {
+    pub fn new(service: AzureAuthService, env_lookup: EnvLookup) -> Self {
+        Self {
+            service,
+            env_lookup,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl TokenCredential for AzureBlobCredential {
+    async fn get_token(
+        &self,
+        scopes: &[&str],
+        _options: Option<TokenRequestOptions<'_>>,
+    ) -> azure_core::Result<AccessToken> {
+        let env_lookup = &self.env_lookup;
+        let lookup = move |name: &str| (name != LLM_TOKEN_ENV).then(|| env_lookup(name)).flatten();
+        let credential = self
+            .service
+            .get_azure_ad_token(
+                &AzureAuthInputs::default_credential_for_scope(&scopes.join(" ")),
+                &lookup,
+            )
+            .await
+            .map_err(|error| {
+                azure_core::Error::with_message(ErrorKind::Credential, error.to_string())
+            })?
+            .ok_or_else(|| {
+                azure_core::Error::with_message(
+                    ErrorKind::Credential,
+                    "no Azure credential is available for blob storage",
+                )
+            })?;
+        let (token, expires_on) = match credential.into_value() {
+            ResolvedCredential::AccessToken { token, expires_on } => (token, expires_on),
+            ResolvedCredential::Static(token) => (token, None),
+        };
+        let expires_on = expires_on.unwrap_or_else(|| SystemTime::now() + STATIC_TOKEN_LIFETIME);
+        Ok(AccessToken::new(
+            token.expose().to_string(),
+            OffsetDateTime::from(expires_on),
+        ))
+    }
+}

litellm-rust/crates/cache-azure-blob/src/lib.rs

@@ -0,0 +1,5 @@
+mod cache;
+mod credential;
+
+pub use cache::AzureBlobCache;
+pub use credential::AzureBlobCredential;

litellm-rust/crates/cache-disk/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "litellm-cache-disk"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-cache.workspace = true
+py_literal = "0.4.0"
+rand.workspace = true
+rusqlite = { version = "0.40", features = ["bundled"] }
+serde-pickle = "1.2"
+serde_json.workspace = true
+tokio.workspace = true
+
+[dev-dependencies]
+rstest.workspace = true
+tempfile = "3.27.0"

litellm-rust/crates/cache-disk/src/adapter.rs

@@ -0,0 +1,10 @@
+use litellm_cache::Error;
+
+use crate::StoredValue;
+
+pub trait ValueAdapter: Send + Sync + 'static {
+    fn read(&self, value: StoredValue) -> Result<Option<Vec<u8>>, Error>;
+    fn write(&self, payload: Vec<u8>) -> StoredValue;
+    fn counter_seed(&self, value: Option<StoredValue>) -> Result<f64, Error>;
+    fn counter_value(&self, value: f64) -> StoredValue;
+}

litellm-rust/crates/cache-disk/src/cache.rs

@@ -0,0 +1,301 @@
+use std::{
+    path::Path,
+    sync::Arc,
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, CacheConnectionStatus,
+    CounterCache, DeleteCache, Error, ExactCacheContext, FlushCache,
+};
+
+use crate::{DiskStore, DiskcacheSqliteStore, PythonDiskCacheAdapter, StoredValue, ValueAdapter};
+
+pub struct DiskCache<S, D = DiskcacheSqliteStore, A = PythonDiskCacheAdapter> {
+    store: Arc<D>,
+    adapter: Arc<A>,
+    codec: S,
+}
+
+impl<S: CacheCodec> DiskCache<S> {
+    pub fn open(directory: impl AsRef<Path>, codec: S) -> Result<Self, Error> {
+        Ok(Self {
+            store: Arc::new(DiskcacheSqliteStore::open(directory)?),
+            adapter: Arc::new(PythonDiskCacheAdapter),
+            codec,
+        })
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore> DiskCache<S, D, PythonDiskCacheAdapter> {
+    pub fn with_store(store: D, codec: S) -> Self {
+        Self {
+            store: Arc::new(store),
+            adapter: Arc::new(PythonDiskCacheAdapter),
+            codec,
+        }
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> DiskCache<S, D, A> {
+    pub fn with_adapter(store: D, adapter: A, codec: S) -> Self {
+        Self {
+            store: Arc::new(store),
+            adapter: Arc::new(adapter),
+            codec,
+        }
+    }
+
+    pub fn directory(&self) -> &Path {
+        self.store.directory()
+    }
+
+    fn decode_stored(&self, value: StoredValue) -> Result<Option<S::Value>, Error> {
+        let Some(bytes) = self.adapter.read(value)? else {
+            return Ok(None);
+        };
+        self.codec.decode(&bytes).map(Some)
+    }
+
+    async fn run_blocking<T, F>(store: Arc<D>, operation: F) -> Result<T, Error>
+    where
+        T: Send + 'static,
+        F: FnOnce(&D) -> Result<T, Error> + Send + 'static,
+    {
+        tokio::task::spawn_blocking(move || operation(&store))
+            .await
+            .map_err(|_| Error::Unavailable)?
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> BaseCache for DiskCache<S, D, A> {
+    type Value = S::Value;
+    type Context = ExactCacheContext;
+
+    fn get_ttl(&self, context: &Self::Context) -> Option<Duration> {
+        context.ttl
+    }
+
+    fn set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: &Self::Context,
+    ) -> Result<(), Error> {
+        let value = self.adapter.write(self.codec.encode(&value)?);
+        let expire_time = context.ttl.map(|ttl| unix_now() + ttl.as_secs_f64());
+        self.store.set(key, value, expire_time, unix_now())
+    }
+
+    fn get_cache(&self, key: &str, _: &Self::Context) -> Result<Option<Self::Value>, Error> {
+        self.store
+            .get(key, unix_now())?
+            .map(|value| self.decode_stored(value))
+            .transpose()
+            .map(|value| value.flatten())
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: ExactCacheContext,
+    ) -> Result<(), Error> {
+        let value = self.adapter.write(self.codec.encode(&value)?);
+        let ttl = context.ttl;
+        let key = key.to_string();
+        Self::run_blocking(Arc::clone(&self.store), move |store| {
+            let expire_time = ttl.map(|ttl| unix_now() + ttl.as_secs_f64());
+            store.set(&key, value, expire_time, unix_now())
+        })
+        .await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        _: &ExactCacheContext,
+    ) -> Result<Option<Self::Value>, Error> {
+        let key = key.to_string();
+        let value = Self::run_blocking(Arc::clone(&self.store), move |store| {
+            store.get(&key, unix_now())
+        })
+        .await?;
+        value
+            .map(|value| self.decode_stored(value))
+            .transpose()
+            .map(|value| value.flatten())
+    }
+
+    async fn async_set_cache_pipeline(
+        &self,
+        entries: Vec<(String, Self::Value)>,
+        context: ExactCacheContext,
+    ) -> Result<(), Error> {
+        let entries = entries
+            .into_iter()
+            .map(|(key, value)| {
+                self.codec
+                    .encode(&value)
+                    .map(|value| (key, self.adapter.write(value)))
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+        let expire_after = context.ttl;
+        Self::run_blocking(Arc::clone(&self.store), move |store| {
+            for (key, value) in entries {
+                let expire_time = expire_after.map(|ttl| unix_now() + ttl.as_secs_f64());
+                store.set(&key, value, expire_time, unix_now())?;
+            }
+            Ok(())
+        })
+        .await
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        let result = Self::run_blocking(Arc::clone(&self.store), |store| {
+            store.probe().map(|_| CacheConnectionResult {
+                status: CacheConnectionStatus::Success,
+                message: "Disk cache connection test successful".into(),
+                error: None,
+            })
+        })
+        .await;
+        Ok(match result {
+            Ok(result) => result,
+            Err(error) => CacheConnectionResult {
+                status: CacheConnectionStatus::Failed,
+                message: format!("Disk cache connection failed: {error}"),
+                error: Some(error.to_string()),
+            },
+        })
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> BatchCache for DiskCache<S, D, A> {
+    fn batch_get_cache(
+        &self,
+        keys: &[String],
+        context: &ExactCacheContext,
+    ) -> Result<Vec<BatchEntry<Self::Value>>, Error> {
+        keys.iter()
+            .map(|key| match self.get_cache(key, context) {
+                Ok(Some(value)) => Ok(BatchEntry::Hit(value)),
+                Ok(None) => Ok(BatchEntry::Miss),
+                Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid),
+                Err(error) => Err(error),
+            })
+            .collect()
+    }
+
+    async fn async_batch_get_cache(
+        &self,
+        keys: Vec<String>,
+        _: ExactCacheContext,
+    ) -> Result<Vec<BatchEntry<Self::Value>>, Error> {
+        let values = Self::run_blocking(Arc::clone(&self.store), move |store| {
+            keys.into_iter()
+                .map(|key| store.get(&key, unix_now()).map(|value| (key, value)))
+                .collect::<Result<Vec<_>, _>>()
+        })
+        .await?;
+        values
+            .into_iter()
+            .map(|(_, value)| match value {
+                None => Ok(BatchEntry::Miss),
+                Some(value) => match self.decode_stored(value) {
+                    Ok(Some(value)) => Ok(BatchEntry::Hit(value)),
+                    Ok(None) => Ok(BatchEntry::Miss),
+                    Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid),
+                    Err(error) => Err(error),
+                },
+            })
+            .collect()
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> DeleteCache for DiskCache<S, D, A> {
+    fn delete_cache(&self, key: &str) -> Result<(), Error> {
+        self.store.pop(key, unix_now()).map(|_| ())
+    }
+
+    async fn async_delete_cache(&self, key: &str) -> Result<(), Error> {
+        let key = key.to_string();
+        Self::run_blocking(Arc::clone(&self.store), move |store| {
+            store.pop(&key, unix_now()).map(|_| ())
+        })
+        .await
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> FlushCache for DiskCache<S, D, A> {
+    fn flush_cache(&self) -> Result<(), Error> {
+        self.store.clear()
+    }
+
+    async fn async_flush_cache(&self) -> Result<(), Error> {
+        Self::run_blocking(Arc::clone(&self.store), |store| store.clear()).await
+    }
+}
+
+impl<S: CacheCodec<Value = f64>, D: DiskStore, A: ValueAdapter> CounterCache
+    for DiskCache<S, D, A>
+{
+    fn increment_cache(
+        &self,
+        key: &str,
+        amount: f64,
+        context: ExactCacheContext,
+    ) -> Result<f64, Error> {
+        increment(
+            self.adapter.as_ref(),
+            self.store.as_ref(),
+            key,
+            amount,
+            context.ttl,
+        )
+    }
+
+    async fn async_increment(
+        &self,
+        key: &str,
+        amount: f64,
+        context: ExactCacheContext,
+    ) -> Result<f64, Error> {
+        let key = key.to_string();
+        let adapter = Arc::clone(&self.adapter);
+        Self::run_blocking(Arc::clone(&self.store), move |store| {
+            increment(adapter.as_ref(), store, &key, amount, context.ttl)
+        })
+        .await
+    }
+}
+
+fn increment<A: ValueAdapter, D: DiskStore>(
+    adapter: &A,
+    store: &D,
+    key: &str,
+    amount: f64,
+    ttl: Option<Duration>,
+) -> Result<f64, Error> {
+    let mut result = None;
+    let mut apply = |current: Option<StoredValue>| {
+        let initial = adapter.counter_seed(current)?;
+        let value = initial + amount;
+        let stored = adapter.counter_value(value);
+        result = Some(value);
+        Ok((stored, ttl.map(|ttl| unix_now() + ttl.as_secs_f64())))
+    };
+    store.update(key, unix_now(), &mut apply)?;
+    result.ok_or(Error::InvalidEntry)
+}
+
+fn unix_now() -> f64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs_f64()
+}

litellm-rust/crates/cache-disk/src/lib.rs

@@ -0,0 +1,11 @@
+mod adapter;
+mod cache;
+mod python;
+mod sqlite;
+mod store;
+
+pub use adapter::ValueAdapter;
+pub use cache::DiskCache;
+pub use python::PythonDiskCacheAdapter;
+pub use sqlite::DiskcacheSqliteStore;
+pub use store::{DiskStore, StoredValue};

litellm-rust/crates/cache-disk/src/python/mod.rs

@@ -0,0 +1,77 @@
+mod value;
+
+use litellm_cache::Error;
+use py_literal::Value;
+
+use crate::{StoredValue, ValueAdapter};
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct PythonDiskCacheAdapter;
+
+impl PythonDiskCacheAdapter {
+    fn python_get_cache(value: StoredValue) -> Result<Option<Value>, Error> {
+        let value = match value {
+            StoredValue::Bytes(value) => Value::Bytes(value),
+            StoredValue::Text(value) => Value::String(value),
+            StoredValue::Integer(value) => Value::Integer(value.into()),
+            StoredValue::Float(value) => Value::Float(value),
+            StoredValue::Pickle(value) => value::from_pickle(&value)?,
+        };
+        if !value::is_truthy(&value) {
+            return Ok(None);
+        }
+        match value {
+            Value::String(text) => Ok(Some(
+                value::from_json_text(&text).unwrap_or(Value::String(text)),
+            )),
+            Value::Bytes(bytes) => match std::str::from_utf8(&bytes) {
+                Ok(text) => Ok(Some(
+                    value::from_json_text(text).unwrap_or(Value::Bytes(bytes)),
+                )),
+                Err(_) => Ok(Some(Value::Bytes(bytes))),
+            },
+            value => Ok(Some(value)),
+        }
+    }
+}
+
+impl ValueAdapter for PythonDiskCacheAdapter {
+    fn read(&self, value: StoredValue) -> Result<Option<Vec<u8>>, Error> {
+        match value {
+            StoredValue::Text(value) => Ok((!value.is_empty()).then(|| value.into_bytes())),
+            StoredValue::Bytes(value) => Ok((!value.is_empty()).then_some(value)),
+            value => {
+                let Some(value) = Self::python_get_cache(value)? else {
+                    return Ok(None);
+                };
+                value::to_json(&value).map(Some)
+            }
+        }
+    }
+
+    fn write(&self, payload: Vec<u8>) -> StoredValue {
+        StoredValue::Bytes(payload)
+    }
+
+    fn counter_seed(&self, value: Option<StoredValue>) -> Result<f64, Error> {
+        let Some(value) = value else {
+            return Ok(0.0);
+        };
+        let Some(value) = Self::python_get_cache(value)? else {
+            return Ok(0.0);
+        };
+        Ok(if value::is_int(&value) {
+            value::to_f64(&value).unwrap_or(0.0)
+        } else {
+            0.0
+        })
+    }
+
+    fn counter_value(&self, value: f64) -> StoredValue {
+        if value.fract() == 0.0 && value >= i64::MIN as f64 && value <= i64::MAX as f64 {
+            StoredValue::Integer(value as i64)
+        } else {
+            StoredValue::Float(value)
+        }
+    }
+}

litellm-rust/crates/cache-disk/src/python/value.rs

@@ -0,0 +1,173 @@
+use litellm_cache::Error;
+use py_literal::Value;
+use serde_json::{Map, Number};
+
+pub(crate) fn from_pickle(bytes: &[u8]) -> Result<Value, Error> {
+    let value = serde_pickle::value_from_slice(bytes, Default::default())
+        .map_err(|_| Error::InvalidEntry)?;
+    from_pickle_value(value)
+}
+
+fn from_pickle_value(value: serde_pickle::Value) -> Result<Value, Error> {
+    match value {
+        serde_pickle::Value::None => Ok(Value::None),
+        serde_pickle::Value::Bool(value) => Ok(Value::Boolean(value)),
+        serde_pickle::Value::I64(value) => integer(value.to_string()),
+        serde_pickle::Value::Int(value) => integer(value.to_string()),
+        serde_pickle::Value::F64(value) => Ok(Value::Float(value)),
+        serde_pickle::Value::String(value) => Ok(Value::String(value)),
+        serde_pickle::Value::Bytes(value) => Ok(Value::Bytes(value)),
+        serde_pickle::Value::List(values) => values
+            .into_iter()
+            .map(from_pickle_value)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::List),
+        serde_pickle::Value::Tuple(values) => values
+            .into_iter()
+            .map(from_pickle_value)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::Tuple),
+        serde_pickle::Value::Set(values) => values
+            .into_iter()
+            .map(from_pickle_hashable)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::Set),
+        serde_pickle::Value::FrozenSet(values) => values
+            .into_iter()
+            .map(from_pickle_hashable)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::Set),
+        serde_pickle::Value::Dict(values) => values
+            .into_iter()
+            .map(|(key, value)| Ok((from_pickle_hashable(key)?, from_pickle_value(value)?)))
+            .collect::<Result<Vec<_>, Error>>()
+            .map(Value::Dict),
+    }
+}
+
+fn from_pickle_hashable(value: serde_pickle::HashableValue) -> Result<Value, Error> {
+    Ok(match value {
+        serde_pickle::HashableValue::None => Value::None,
+        serde_pickle::HashableValue::Bool(value) => Value::Boolean(value),
+        serde_pickle::HashableValue::I64(value) => integer(value.to_string())?,
+        serde_pickle::HashableValue::Int(value) => integer(value.to_string())?,
+        serde_pickle::HashableValue::F64(value) => Value::Float(value),
+        serde_pickle::HashableValue::Bytes(value) => Value::Bytes(value),
+        serde_pickle::HashableValue::String(value) => Value::String(value),
+        serde_pickle::HashableValue::Tuple(values) => Value::Tuple(
+            values
+                .into_iter()
+                .map(from_pickle_hashable)
+                .collect::<Result<Vec<_>, _>>()?,
+        ),
+        serde_pickle::HashableValue::FrozenSet(values) => Value::Set(
+            values
+                .into_iter()
+                .map(from_pickle_hashable)
+                .collect::<Result<Vec<_>, _>>()?,
+        ),
+    })
+}
+
+fn integer(value: String) -> Result<Value, Error> {
+    value.parse().map_err(|_| Error::InvalidEntry)
+}
+
+pub(crate) fn from_json(value: serde_json::Value) -> Value {
+    match value {
+        serde_json::Value::Null => Value::None,
+        serde_json::Value::Bool(value) => Value::Boolean(value),
+        serde_json::Value::Number(value) => {
+            if value.is_i64() || value.is_u64() {
+                integer(value.to_string())
+                    .unwrap_or(Value::Float(value.as_f64().unwrap_or(f64::NAN)))
+            } else {
+                Value::Float(value.as_f64().unwrap_or(f64::NAN))
+            }
+        }
+        serde_json::Value::String(value) => Value::String(value),
+        serde_json::Value::Array(values) => {
+            Value::List(values.into_iter().map(from_json).collect())
+        }
+        serde_json::Value::Object(values) => Value::Dict(
+            values
+                .into_iter()
+                .map(|(key, value)| (Value::String(key), from_json(value)))
+                .collect(),
+        ),
+    }
+}
+
+pub(crate) fn from_json_text(value: &str) -> Result<Value, Error> {
+    serde_json::from_str(value)
+        .map(from_json)
+        .map_err(|_| Error::InvalidEntry)
+}
+
+pub(crate) fn is_truthy(value: &Value) -> bool {
+    match value {
+        Value::None => false,
+        Value::Boolean(value) => *value,
+        Value::Integer(value) => value.to_string() != "0",
+        Value::Float(value) => *value != 0.0,
+        Value::Complex(value) => value.re != 0.0 || value.im != 0.0,
+        Value::String(value) => !value.is_empty(),
+        Value::Bytes(value) => !value.is_empty(),
+        Value::Tuple(value) | Value::List(value) | Value::Set(value) => !value.is_empty(),
+        Value::Dict(value) => !value.is_empty(),
+    }
+}
+
+pub(crate) fn is_int(value: &Value) -> bool {
+    matches!(value, Value::Integer(_) | Value::Boolean(_))
+}
+
+pub(crate) fn to_f64(value: &Value) -> Option<f64> {
+    match value {
+        Value::Integer(value) => value.to_string().parse().ok(),
+        Value::Boolean(value) => Some(if *value { 1.0 } else { 0.0 }),
+        _ => None,
+    }
+}
+
+pub(crate) fn to_json(value: &Value) -> Result<Vec<u8>, Error> {
+    serde_json::to_vec(&to_json_value(value)?).map_err(|_| Error::InvalidEntry)
+}
+
+fn to_json_value(value: &Value) -> Result<serde_json::Value, Error> {
+    Ok(match value {
+        Value::None => serde_json::Value::Null,
+        Value::Boolean(value) => serde_json::Value::Bool(*value),
+        Value::Integer(value) => serde_json::Value::Number(
+            value
+                .to_string()
+                .parse::<Number>()
+                .map_err(|_| Error::InvalidEntry)?,
+        ),
+        Value::Float(value) => {
+            serde_json::Value::Number(Number::from_f64(*value).ok_or(Error::InvalidEntry)?)
+        }
+        Value::Complex(_) | Value::Bytes(_) => return Err(Error::InvalidEntry),
+        Value::String(value) => serde_json::Value::String(value.clone()),
+        Value::Tuple(values) | Value::List(values) | Value::Set(values) => {
+            serde_json::Value::Array(
+                values
+                    .iter()
+                    .map(to_json_value)
+                    .collect::<Result<Vec<_>, _>>()?,
+            )
+        }
+        Value::Dict(values) => {
+            let values = values
+                .iter()
+                .map(|(key, value)| {
+                    let Value::String(key) = key else {
+                        return Err(Error::InvalidEntry);
+                    };
+                    Ok((key.clone(), to_json_value(value)?))
+                })
+                .collect::<Result<Map<String, serde_json::Value>, _>>()?;
+            serde_json::Value::Object(values)
+        }
+    })
+}

litellm-rust/crates/cache-disk/src/sqlite.rs

@@ -0,0 +1,817 @@
+use std::{
+    collections::HashMap,
+    fs::{self, OpenOptions},
+    io::Write,
+    path::{Path, PathBuf},
+    sync::Mutex,
+};
+
+use litellm_cache::Error;
+use rand::RngCore;
+use rusqlite::{Connection, OptionalExtension, params, types::Value};
+
+use crate::{DiskStore, StoredValue};
+
+const MODE_RAW: i64 = 1;
+const MODE_BINARY: i64 = 2;
+const MODE_TEXT: i64 = 3;
+const MODE_PICKLE: i64 = 4;
+
+const DEFAULT_DISK_MIN_FILE_SIZE: i64 = 2_i64.pow(15);
+const DEFAULT_SIZE_LIMIT: i64 = 2_i64.pow(30);
+const DEFAULT_CULL_LIMIT: i64 = 10;
+
+pub struct DiskcacheSqliteStore {
+    directory: PathBuf,
+    connection: Mutex<Connection>,
+    min_file_size: usize,
+    eviction_policy: String,
+    size_limit: i64,
+    cull_limit: i64,
+    statistics: bool,
+}
+
+struct StoredColumns {
+    size: i64,
+    mode: i64,
+    filename: Option<String>,
+    value: Option<Value>,
+}
+
+struct Row {
+    rowid: i64,
+    mode: i64,
+    filename: Option<String>,
+    value: Value,
+}
+
+impl DiskcacheSqliteStore {
+    pub fn open(directory: impl AsRef<Path>) -> Result<Self, Error> {
+        let directory = directory.as_ref().to_path_buf();
+        fs::create_dir_all(&directory).map_err(|_| Error::Unavailable)?;
+        let directory = std::path::absolute(&directory).map_err(|_| Error::Unavailable)?;
+        let database = directory.join("cache.db");
+        let connection = Connection::open(database).map_err(|_| Error::Unavailable)?;
+        connection
+            .busy_timeout(std::time::Duration::from_secs(60))
+            .map_err(|_| Error::Unavailable)?;
+
+        let mut settings = read_settings(&connection)?;
+        for (key, value) in default_settings() {
+            settings.entry(key).or_insert(value);
+        }
+        for (key, value) in settings
+            .iter()
+            .filter(|(key, _)| key.starts_with("sqlite_"))
+        {
+            apply_pragma(&connection, key, value)?;
+        }
+
+        connection
+            .execute_batch(
+                "CREATE TABLE IF NOT EXISTS Settings (
+                    key TEXT NOT NULL UNIQUE,
+                    value
+                )",
+            )
+            .map_err(|_| Error::Unavailable)?;
+        for (key, value) in &settings {
+            if !matches!(key.as_str(), "count" | "size" | "hits" | "misses") {
+                connection
+                    .execute(
+                        "INSERT OR REPLACE INTO Settings VALUES (?, ?)",
+                        params![key, value],
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+        }
+        for (key, value) in [
+            ("count", Value::Integer(0)),
+            ("size", Value::Integer(0)),
+            ("hits", Value::Integer(0)),
+            ("misses", Value::Integer(0)),
+        ] {
+            connection
+                .execute(
+                    "INSERT OR IGNORE INTO Settings VALUES (?, ?)",
+                    params![key, value],
+                )
+                .map_err(|_| Error::Unavailable)?;
+        }
+        connection
+            .execute_batch(
+                "CREATE TABLE IF NOT EXISTS Cache (
+                    rowid INTEGER PRIMARY KEY,
+                    key BLOB,
+                    raw INTEGER,
+                    store_time REAL,
+                    expire_time REAL,
+                    access_time REAL,
+                    access_count INTEGER DEFAULT 0,
+                    tag BLOB,
+                    size INTEGER DEFAULT 0,
+                    mode INTEGER DEFAULT 0,
+                    filename TEXT,
+                    value BLOB
+                );
+                CREATE UNIQUE INDEX IF NOT EXISTS Cache_key_raw ON Cache(key, raw);
+                CREATE INDEX IF NOT EXISTS Cache_expire_time ON Cache(expire_time);",
+            )
+            .map_err(|_| Error::Unavailable)?;
+
+        let eviction_policy = setting_string(&settings, "eviction_policy")
+            .unwrap_or_else(|| "least-recently-stored".to_string());
+        match eviction_policy.as_str() {
+            "none" => {}
+            "least-recently-stored" => {
+                connection
+                    .execute_batch(
+                        "CREATE INDEX IF NOT EXISTS Cache_store_time ON Cache(store_time)",
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            "least-recently-used" => {
+                connection
+                    .execute_batch(
+                        "CREATE INDEX IF NOT EXISTS Cache_access_time ON Cache(access_time)",
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            "least-frequently-used" => {
+                connection
+                    .execute_batch(
+                        "CREATE INDEX IF NOT EXISTS Cache_access_count ON Cache(access_count)",
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            _ => return Err(Error::Unavailable),
+        }
+        connection
+            .execute_batch(
+                "CREATE TRIGGER IF NOT EXISTS Settings_count_insert
+                    AFTER INSERT ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings SET value = value + 1
+                    WHERE key = \"count\"; END;
+                 CREATE TRIGGER IF NOT EXISTS Settings_count_delete
+                    AFTER DELETE ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings SET value = value - 1
+                    WHERE key = \"count\"; END;
+                 CREATE TRIGGER IF NOT EXISTS Settings_size_insert
+                    AFTER INSERT ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings SET value = value + NEW.size
+                    WHERE key = \"size\"; END;
+                 CREATE TRIGGER IF NOT EXISTS Settings_size_update
+                    AFTER UPDATE ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings
+                    SET value = value + NEW.size - OLD.size
+                    WHERE key = \"size\"; END;
+                 CREATE TRIGGER IF NOT EXISTS Settings_size_delete
+                    AFTER DELETE ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings SET value = value - OLD.size
+                    WHERE key = \"size\"; END;",
+            )
+            .map_err(|_| Error::Unavailable)?;
+
+        let min_file_size = setting_i64(&settings, "disk_min_file_size")
+            .unwrap_or(DEFAULT_DISK_MIN_FILE_SIZE)
+            .try_into()
+            .map_err(|_| Error::Unavailable)?;
+        let size_limit = setting_i64(&settings, "size_limit").unwrap_or(DEFAULT_SIZE_LIMIT);
+        let cull_limit = setting_i64(&settings, "cull_limit").unwrap_or(DEFAULT_CULL_LIMIT);
+        let statistics = setting_i64(&settings, "statistics").unwrap_or_default() != 0;
+
+        Ok(Self {
+            directory,
+            connection: Mutex::new(connection),
+            min_file_size,
+            eviction_policy,
+            size_limit,
+            cull_limit,
+            statistics,
+        })
+    }
+
+    fn set_locked(
+        &self,
+        connection: &Connection,
+        key: &str,
+        columns: StoredColumns,
+        expire_time: Option<f64>,
+        now: f64,
+    ) -> Result<Vec<String>, Error> {
+        let mut cleanup = Vec::new();
+        if let Some(old_filename) = connection
+            .query_row(
+                "SELECT filename FROM Cache WHERE key = ? AND raw = 1",
+                params![key],
+                |row| row.get::<_, Option<String>>(0),
+            )
+            .optional()
+            .map_err(|_| Error::Unavailable)?
+            .flatten()
+        {
+            cleanup.push(old_filename);
+        }
+        let (size, mode, filename, value) =
+            (columns.size, columns.mode, columns.filename, columns.value);
+        let rowid = connection
+            .query_row(
+                "SELECT rowid FROM Cache WHERE key = ? AND raw = 1",
+                params![key],
+                |row| row.get::<_, i64>(0),
+            )
+            .optional()
+            .map_err(|_| Error::Unavailable)?;
+        if let Some(rowid) = rowid {
+            connection
+                .execute(
+                    "UPDATE Cache SET store_time = ?, expire_time = ?, access_time = ?,
+                        access_count = 0, tag = NULL, size = ?, mode = ?, filename = ?, value = ?
+                        WHERE rowid = ?",
+                    params![now, expire_time, now, size, mode, filename, value, rowid],
+                )
+                .map_err(|_| Error::Unavailable)?;
+        } else {
+            connection
+                .execute(
+                    "INSERT INTO Cache(
+                        key, raw, store_time, expire_time, access_time, access_count,
+                        tag, size, mode, filename, value
+                    ) VALUES (?, 1, ?, ?, ?, 0, NULL, ?, ?, ?, ?)",
+                    params![key, now, expire_time, now, size, mode, filename, value],
+                )
+                .map_err(|_| Error::Unavailable)?;
+        }
+        cleanup.extend(self.cull(connection, now)?);
+        Ok(cleanup)
+    }
+
+    fn cull(&self, connection: &Connection, now: f64) -> Result<Vec<String>, Error> {
+        if self.cull_limit <= 0 {
+            return Ok(Vec::new());
+        }
+        let mut cleanup = Vec::new();
+        let expired = connection
+            .prepare(
+                "SELECT rowid, filename FROM Cache
+                 WHERE expire_time IS NOT NULL AND expire_time < ?
+                 ORDER BY expire_time LIMIT ?",
+            )
+            .map_err(|_| Error::Unavailable)?
+            .query_map(params![now, self.cull_limit], |row| {
+                Ok((row.get::<_, i64>(0)?, row.get::<_, Option<String>>(1)?))
+            })
+            .map_err(|_| Error::Unavailable)?
+            .collect::<Result<Vec<_>, _>>()
+            .map_err(|_| Error::Unavailable)?;
+        for (_, filename) in &expired {
+            if let Some(filename) = filename {
+                cleanup.push(filename.clone());
+            }
+        }
+        for (rowid, _) in &expired {
+            connection
+                .execute("DELETE FROM Cache WHERE rowid = ?", params![rowid])
+                .map_err(|_| Error::Unavailable)?;
+        }
+        let remaining = self.cull_limit - i64::try_from(expired.len()).unwrap_or(self.cull_limit);
+        if remaining <= 0 || self.volume(connection)? < self.size_limit {
+            return Ok(cleanup);
+        }
+        let order = match self.eviction_policy.as_str() {
+            "none" => return Ok(cleanup),
+            "least-recently-stored" => "store_time",
+            "least-recently-used" => "access_time",
+            "least-frequently-used" => "access_count",
+            _ => return Err(Error::Unavailable),
+        };
+        let rows = connection
+            .prepare(&format!(
+                "SELECT rowid, filename FROM Cache ORDER BY {order} LIMIT ?"
+            ))
+            .map_err(|_| Error::Unavailable)?
+            .query_map(params![remaining], |row| {
+                Ok((row.get::<_, i64>(0)?, row.get::<_, Option<String>>(1)?))
+            })
+            .map_err(|_| Error::Unavailable)?
+            .collect::<Result<Vec<_>, _>>()
+            .map_err(|_| Error::Unavailable)?;
+        for (_, filename) in &rows {
+            if let Some(filename) = filename {
+                cleanup.push(filename.clone());
+            }
+        }
+        for (rowid, _) in rows {
+            connection
+                .execute("DELETE FROM Cache WHERE rowid = ?", params![rowid])
+                .map_err(|_| Error::Unavailable)?;
+        }
+        Ok(cleanup)
+    }
+
+    fn volume(&self, connection: &Connection) -> Result<i64, Error> {
+        let page_count: i64 = connection
+            .query_row("PRAGMA page_count", [], |row| row.get(0))
+            .map_err(|_| Error::Unavailable)?;
+        let page_size: i64 = connection
+            .query_row("PRAGMA page_size", [], |row| row.get(0))
+            .map_err(|_| Error::Unavailable)?;
+        let size: i64 = connection
+            .query_row("SELECT value FROM Settings WHERE key = 'size'", [], |row| {
+                row.get(0)
+            })
+            .map_err(|_| Error::Unavailable)?;
+        Ok(page_count.saturating_mul(page_size).saturating_add(size))
+    }
+}
+
+impl DiskStore for DiskcacheSqliteStore {
+    fn directory(&self) -> &Path {
+        &self.directory
+    }
+
+    fn get(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let select = "SELECT rowid, expire_time, mode, filename, value FROM Cache
+            WHERE key = ? AND raw = 1 AND (expire_time IS NULL OR expire_time > ?)";
+        let row = connection
+            .query_row(select, params![key, now], row_from_query)
+            .optional()
+            .map_err(|_| Error::Unavailable)?;
+        if !self.statistics && !has_get_update(&self.eviction_policy) {
+            return row
+                .map(|row| fetch_row(&self.directory, row))
+                .transpose()
+                .map(|value| value.flatten());
+        }
+        transactional(&connection, |connection| {
+            let row = connection
+                .query_row(select, params![key, now], row_from_query)
+                .optional()
+                .map_err(|_| Error::Unavailable)?;
+            let Some(row) = row else {
+                if self.statistics {
+                    connection
+                        .execute(
+                            "UPDATE Settings SET value = value + 1 WHERE key = 'misses'",
+                            [],
+                        )
+                        .map_err(|_| Error::Unavailable)?;
+                }
+                return Ok(None);
+            };
+            let rowid = row.rowid;
+            let value = fetch_row(&self.directory, row);
+            let hit = value.as_ref().is_ok_and(Option::is_some);
+            if hit && self.statistics {
+                connection
+                    .execute(
+                        "UPDATE Settings SET value = value + 1 WHERE key = 'hits'",
+                        [],
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            } else if !hit && self.statistics {
+                connection
+                    .execute(
+                        "UPDATE Settings SET value = value + 1 WHERE key = 'misses'",
+                        [],
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            if has_get_update(&self.eviction_policy) && hit {
+                let update = match self.eviction_policy.as_str() {
+                    "least-recently-used" => "UPDATE Cache SET access_time = ? WHERE rowid = ?",
+                    "least-frequently-used" => {
+                        "UPDATE Cache SET access_count = access_count + 1 WHERE rowid = ?"
+                    }
+                    _ => return Err(Error::Unavailable),
+                };
+                if self.eviction_policy == "least-recently-used" {
+                    connection
+                        .execute(update, params![now, rowid])
+                        .map_err(|_| Error::Unavailable)?;
+                } else {
+                    connection
+                        .execute(update, params![rowid])
+                        .map_err(|_| Error::Unavailable)?;
+                }
+            }
+            value
+        })
+    }
+
+    fn set(
+        &self,
+        key: &str,
+        value: StoredValue,
+        expire_time: Option<f64>,
+        now: f64,
+    ) -> Result<(), Error> {
+        let columns = store_value(&self.directory, self.min_file_size, value)?;
+        let new_filename = columns.filename.clone();
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let result = transactional(&connection, |connection| {
+            self.set_locked(connection, key, columns, expire_time, now)
+        });
+        match result {
+            Ok(cleanup) => {
+                cleanup_files(&self.directory, cleanup);
+                Ok(())
+            }
+            Err(error) => {
+                if let Some(filename) = new_filename {
+                    remove_file(&self.directory, &filename);
+                }
+                Err(error)
+            }
+        }
+    }
+
+    fn pop(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let selected = transactional(&connection, |connection| {
+            let row = connection
+                .query_row(
+                    "SELECT rowid, expire_time, mode, filename, value FROM Cache
+                     WHERE key = ? AND raw = 1
+                     AND (expire_time IS NULL OR expire_time > ?)",
+                    params![key, now],
+                    row_from_query,
+                )
+                .optional()
+                .map_err(|_| Error::Unavailable)?;
+            let Some(row) = row else {
+                return Ok(None);
+            };
+            connection
+                .execute("DELETE FROM Cache WHERE rowid = ?", params![row.rowid])
+                .map_err(|_| Error::Unavailable)?;
+            Ok(Some(row))
+        })?;
+        let Some(row) = selected else {
+            return Ok(None);
+        };
+        let filename = row.filename.clone();
+        let result = fetch_row(&self.directory, row)?;
+        if let Some(filename) = filename {
+            remove_file(&self.directory, &filename);
+        }
+        Ok(result)
+    }
+
+    fn clear(&self) -> Result<(), Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let mut last_rowid = 0_i64;
+        loop {
+            let batch = transactional(&connection, |connection| {
+                let rows = connection
+                    .prepare(
+                        "SELECT rowid, filename FROM Cache
+                         WHERE rowid > ? ORDER BY rowid LIMIT 100",
+                    )
+                    .map_err(|_| Error::Unavailable)?
+                    .query_map(params![last_rowid], |row| {
+                        Ok((row.get::<_, i64>(0)?, row.get::<_, Option<String>>(1)?))
+                    })
+                    .map_err(|_| Error::Unavailable)?
+                    .collect::<Result<Vec<_>, _>>()
+                    .map_err(|_| Error::Unavailable)?;
+                if rows.is_empty() {
+                    return Ok(rows);
+                }
+                let ids = rows
+                    .iter()
+                    .map(|(rowid, _)| rowid.to_string())
+                    .collect::<Vec<_>>()
+                    .join(",");
+                connection
+                    .execute(&format!("DELETE FROM Cache WHERE rowid IN ({ids})"), [])
+                    .map_err(|_| Error::Unavailable)?;
+                Ok(rows)
+            })?;
+            if batch.is_empty() {
+                return Ok(());
+            }
+            last_rowid = batch.last().map(|(rowid, _)| *rowid).unwrap_or(last_rowid);
+            cleanup_files(
+                &self.directory,
+                batch
+                    .into_iter()
+                    .filter_map(|(_, filename)| filename)
+                    .collect(),
+            );
+        }
+    }
+
+    fn update(
+        &self,
+        key: &str,
+        now: f64,
+        apply: &mut dyn FnMut(Option<StoredValue>) -> Result<(StoredValue, Option<f64>), Error>,
+    ) -> Result<(), Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let mut created_filename = None;
+        let result = transactional(&connection, |connection| {
+            let current = connection
+                .query_row(
+                    "SELECT rowid, expire_time, mode, filename, value FROM Cache
+                     WHERE key = ? AND raw = 1
+                     AND (expire_time IS NULL OR expire_time > ?)",
+                    params![key, now],
+                    row_from_query,
+                )
+                .optional()
+                .map_err(|_| Error::Unavailable)?
+                .map(|row| fetch_row(&self.directory, row))
+                .transpose()?
+                .flatten();
+            let (value, expire_time) = apply(current)?;
+            let columns = store_value(&self.directory, self.min_file_size, value)?;
+            created_filename = columns.filename.clone();
+            let cleanup = self.set_locked(connection, key, columns, expire_time, now)?;
+            Ok(cleanup)
+        });
+        match result {
+            Ok(cleanup) => {
+                cleanup_files(&self.directory, cleanup);
+                Ok(())
+            }
+            Err(error) => {
+                if let Some(filename) = created_filename {
+                    remove_file(&self.directory, &filename);
+                }
+                Err(error)
+            }
+        }
+    }
+
+    fn probe(&self) -> Result<(), Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        connection
+            .query_row(
+                "SELECT value FROM Settings WHERE key = 'count'",
+                [],
+                |row| row.get::<_, i64>(0),
+            )
+            .map(|_| ())
+            .map_err(|_| Error::Unavailable)
+    }
+}
+
+fn default_settings() -> HashMap<String, Value> {
+    HashMap::from([
+        ("statistics".to_string(), Value::Integer(0)),
+        ("tag_index".to_string(), Value::Integer(0)),
+        (
+            "eviction_policy".to_string(),
+            Value::Text("least-recently-stored".to_string()),
+        ),
+        ("size_limit".to_string(), Value::Integer(DEFAULT_SIZE_LIMIT)),
+        ("cull_limit".to_string(), Value::Integer(DEFAULT_CULL_LIMIT)),
+        ("sqlite_auto_vacuum".to_string(), Value::Integer(1)),
+        ("sqlite_cache_size".to_string(), Value::Integer(8192)),
+        (
+            "sqlite_journal_mode".to_string(),
+            Value::Text("wal".to_string()),
+        ),
+        (
+            "sqlite_mmap_size".to_string(),
+            Value::Integer(2_i64.pow(26)),
+        ),
+        ("sqlite_synchronous".to_string(), Value::Integer(1)),
+        (
+            "disk_min_file_size".to_string(),
+            Value::Integer(DEFAULT_DISK_MIN_FILE_SIZE),
+        ),
+        ("disk_pickle_protocol".to_string(), Value::Integer(5)),
+    ])
+}
+
+fn read_settings(connection: &Connection) -> Result<HashMap<String, Value>, Error> {
+    let mut statement = match connection.prepare("SELECT key, value FROM Settings") {
+        Ok(statement) => statement,
+        Err(_) => return Ok(HashMap::new()),
+    };
+    statement
+        .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
+        .map_err(|_| Error::Unavailable)?
+        .collect::<Result<HashMap<_, _>, _>>()
+        .map_err(|_| Error::Unavailable)
+}
+
+fn apply_pragma(connection: &Connection, key: &str, value: &Value) -> Result<(), Error> {
+    let pragma = key.strip_prefix("sqlite_").ok_or(Error::Unavailable)?;
+    match value {
+        Value::Integer(value) => connection
+            .pragma_update(None, pragma, value)
+            .map_err(|_| Error::Unavailable),
+        Value::Text(value) => connection
+            .pragma_update(None, pragma, value)
+            .map_err(|_| Error::Unavailable),
+        _ => Err(Error::Unavailable),
+    }
+}
+
+fn setting_i64(settings: &HashMap<String, Value>, key: &str) -> Option<i64> {
+    match settings.get(key) {
+        Some(Value::Integer(value)) => Some(*value),
+        _ => None,
+    }
+}
+
+fn setting_string(settings: &HashMap<String, Value>, key: &str) -> Option<String> {
+    match settings.get(key) {
+        Some(Value::Text(value)) => Some(value.clone()),
+        _ => None,
+    }
+}
+
+fn has_get_update(policy: &str) -> bool {
+    matches!(policy, "least-recently-used" | "least-frequently-used")
+}
+
+fn row_from_query(row: &rusqlite::Row<'_>) -> rusqlite::Result<Row> {
+    Ok(Row {
+        rowid: row.get(0)?,
+        mode: row.get(2)?,
+        filename: row.get(3)?,
+        value: row.get(4)?,
+    })
+}
+
+fn fetch_row(directory: &Path, row: Row) -> Result<Option<StoredValue>, Error> {
+    match row.mode {
+        MODE_RAW => match row.value {
+            Value::Blob(value) => Ok(Some(StoredValue::Bytes(value))),
+            Value::Text(value) => Ok(Some(StoredValue::Text(value))),
+            Value::Integer(value) => Ok(Some(StoredValue::Integer(value))),
+            Value::Real(value) => Ok(Some(StoredValue::Float(value))),
+            Value::Null => Err(Error::InvalidEntry),
+        },
+        MODE_BINARY | MODE_PICKLE => {
+            let bytes = match row.value {
+                Value::Blob(value) => value,
+                Value::Null => {
+                    let Some(value) = read_file(directory, row.filename.as_deref())? else {
+                        return Ok(None);
+                    };
+                    value
+                }
+                _ => return Err(Error::InvalidEntry),
+            };
+            Ok(Some(if row.mode == MODE_BINARY {
+                StoredValue::Bytes(bytes)
+            } else {
+                StoredValue::Pickle(bytes)
+            }))
+        }
+        MODE_TEXT => {
+            let bytes = match row.value {
+                Value::Null => {
+                    let Some(value) = read_file(directory, row.filename.as_deref())? else {
+                        return Ok(None);
+                    };
+                    value
+                }
+                Value::Blob(value) => value,
+                Value::Text(value) => value.into_bytes(),
+                _ => return Err(Error::InvalidEntry),
+            };
+            Ok(Some(StoredValue::Text(
+                String::from_utf8(bytes).map_err(|_| Error::InvalidEntry)?,
+            )))
+        }
+        _ => Err(Error::InvalidEntry),
+    }
+}
+
+fn read_file(directory: &Path, filename: Option<&str>) -> Result<Option<Vec<u8>>, Error> {
+    let Some(filename) = filename else {
+        return Err(Error::InvalidEntry);
+    };
+    match fs::read(directory.join(filename)) {
+        Ok(value) => Ok(Some(value)),
+        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
+        Err(_) => Err(Error::Unavailable),
+    }
+}
+
+fn store_value(
+    directory: &Path,
+    min_file_size: usize,
+    value: StoredValue,
+) -> Result<StoredColumns, Error> {
+    match value {
+        StoredValue::Integer(value) => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_RAW,
+            filename: None,
+            value: Some(Value::Integer(value)),
+        }),
+        StoredValue::Float(value) => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_RAW,
+            filename: None,
+            value: Some(Value::Real(value)),
+        }),
+        StoredValue::Text(value) if value.chars().count() < min_file_size => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_RAW,
+            filename: None,
+            value: Some(Value::Text(value)),
+        }),
+        StoredValue::Text(value) => {
+            let bytes = value.into_bytes();
+            let filename = write_file(directory, &bytes)?;
+            Ok(StoredColumns {
+                size: i64::try_from(bytes.len()).map_err(|_| Error::Unavailable)?,
+                mode: MODE_TEXT,
+                filename: Some(filename),
+                value: None,
+            })
+        }
+        StoredValue::Bytes(value) if value.len() < min_file_size => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_RAW,
+            filename: None,
+            value: Some(Value::Blob(value)),
+        }),
+        StoredValue::Bytes(value) => {
+            let filename = write_file(directory, &value)?;
+            Ok(StoredColumns {
+                size: i64::try_from(value.len()).map_err(|_| Error::Unavailable)?,
+                mode: MODE_BINARY,
+                filename: Some(filename),
+                value: None,
+            })
+        }
+        StoredValue::Pickle(value) if value.len() < min_file_size => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_PICKLE,
+            filename: None,
+            value: Some(Value::Blob(value)),
+        }),
+        StoredValue::Pickle(value) => {
+            let filename = write_file(directory, &value)?;
+            Ok(StoredColumns {
+                size: i64::try_from(value.len()).map_err(|_| Error::Unavailable)?,
+                mode: MODE_PICKLE,
+                filename: Some(filename),
+                value: None,
+            })
+        }
+    }
+}
+
+fn write_file(directory: &Path, bytes: &[u8]) -> Result<String, Error> {
+    let mut random = [0_u8; 16];
+    rand::rngs::OsRng.fill_bytes(&mut random);
+    let hex = random
+        .iter()
+        .map(|byte| format!("{byte:02x}"))
+        .collect::<String>();
+    let filename = format!("{}/{}/{}.val", &hex[..2], &hex[2..4], &hex[4..]);
+    let path = directory.join(&filename);
+    if let Some(parent) = path.parent() {
+        fs::create_dir_all(parent).map_err(|_| Error::Unavailable)?;
+    }
+    let mut file = OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(path)
+        .map_err(|_| Error::Unavailable)?;
+    file.write_all(bytes).map_err(|_| Error::Unavailable)?;
+    Ok(filename)
+}
+
+fn cleanup_files(directory: &Path, filenames: Vec<String>) {
+    for filename in filenames {
+        remove_file(directory, &filename);
+    }
+}
+
+fn remove_file(directory: &Path, filename: &str) {
+    let path = directory.join(filename);
+    let _ = fs::remove_file(&path);
+}
+
+fn transactional<T>(
+    connection: &Connection,
+    operation: impl FnOnce(&Connection) -> Result<T, Error>,
+) -> Result<T, Error> {
+    connection
+        .execute_batch("BEGIN IMMEDIATE")
+        .map_err(|_| Error::Unavailable)?;
+    match operation(connection) {
+        Ok(value) => {
+            connection
+                .execute_batch("COMMIT")
+                .map_err(|_| Error::Unavailable)?;
+            Ok(value)
+        }
+        Err(error) => {
+            let _ = connection.execute_batch("ROLLBACK");
+            Err(error)
+        }
+    }
+}

litellm-rust/crates/cache-disk/src/store.rs

@@ -0,0 +1,33 @@
+use std::path::Path;
+
+use litellm_cache::Error;
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum StoredValue {
+    Bytes(Vec<u8>),
+    Text(String),
+    Integer(i64),
+    Float(f64),
+    Pickle(Vec<u8>),
+}
+
+pub trait DiskStore: Send + Sync + 'static {
+    fn directory(&self) -> &Path;
+    fn get(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error>;
+    fn set(
+        &self,
+        key: &str,
+        value: StoredValue,
+        expire_time: Option<f64>,
+        now: f64,
+    ) -> Result<(), Error>;
+    fn pop(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error>;
+    fn clear(&self) -> Result<(), Error>;
+    fn update(
+        &self,
+        key: &str,
+        now: f64,
+        apply: &mut dyn FnMut(Option<StoredValue>) -> Result<(StoredValue, Option<f64>), Error>,
+    ) -> Result<(), Error>;
+    fn probe(&self) -> Result<(), Error>;
+}

litellm-rust/crates/cache-disk/tests/cache.rs

@@ -0,0 +1,431 @@
+use std::{
+    fs,
+    path::{Path, PathBuf},
+    sync::Arc,
+    thread,
+    time::Duration,
+};
+
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheCodec, CounterCache, DeleteCache, ExactCacheContext,
+    FlushCache, JsonCodec,
+};
+use litellm_cache_disk::{DiskCache, DiskStore, DiskcacheSqliteStore, StoredValue, ValueAdapter};
+use rstest::{fixture, rstest};
+use rusqlite::Connection;
+use serde_json::{Value, json};
+use tempfile::TempDir;
+
+struct Sandbox {
+    directory: TempDir,
+}
+
+#[fixture]
+fn sandbox() -> Sandbox {
+    Sandbox {
+        directory: tempfile::tempdir().unwrap(),
+    }
+}
+
+impl Sandbox {
+    fn store(&self) -> DiskcacheSqliteStore {
+        DiskcacheSqliteStore::open(self.directory.path()).unwrap()
+    }
+
+    fn cache<V>(&self) -> DiskCache<JsonCodec<V>>
+    where
+        JsonCodec<V>: CacheCodec,
+    {
+        DiskCache::open(self.directory.path(), JsonCodec::new()).unwrap()
+    }
+
+    fn db(&self) -> Connection {
+        Connection::open(self.directory.path().join("cache.db")).unwrap()
+    }
+
+    fn value_files(&self) -> Vec<PathBuf> {
+        fn visit(directory: &Path, files: &mut Vec<PathBuf>) {
+            for entry in fs::read_dir(directory).unwrap() {
+                let path = entry.unwrap().path();
+                if path.is_dir() {
+                    visit(&path, files);
+                } else if path.extension().is_some_and(|extension| extension == "val") {
+                    files.push(path);
+                }
+            }
+        }
+
+        let mut files = Vec::new();
+        visit(self.directory.path(), &mut files);
+        files
+    }
+}
+
+#[rstest]
+fn relative_store_directory_is_absolutized(sandbox: Sandbox) {
+    let relative = PathBuf::from(format!(
+        ".litellm-cache-disk-{}",
+        sandbox
+            .directory
+            .path()
+            .file_name()
+            .unwrap()
+            .to_string_lossy()
+    ));
+    let store = DiskcacheSqliteStore::open(&relative).unwrap();
+    assert!(store.directory().is_absolute());
+    assert!(store.directory().ends_with(&relative));
+    let directory = store.directory().to_path_buf();
+    drop(store);
+    fs::remove_dir_all(directory).unwrap();
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+struct TextAdapter;
+
+impl ValueAdapter for TextAdapter {
+    fn read(&self, value: StoredValue) -> Result<Option<Vec<u8>>, litellm_cache::Error> {
+        match value {
+            StoredValue::Text(value) => Ok(Some(value.into_bytes())),
+            _ => Ok(None),
+        }
+    }
+
+    fn write(&self, payload: Vec<u8>) -> StoredValue {
+        StoredValue::Text(String::from_utf8(payload).unwrap())
+    }
+
+    fn counter_seed(&self, _: Option<StoredValue>) -> Result<f64, litellm_cache::Error> {
+        Ok(0.0)
+    }
+
+    fn counter_value(&self, value: f64) -> StoredValue {
+        if value.fract() == 0.0 {
+            StoredValue::Integer(value as i64)
+        } else {
+            StoredValue::Float(value)
+        }
+    }
+}
+
+#[rstest]
+fn roundtrip_persists_and_reopens(sandbox: Sandbox) {
+    let context = ExactCacheContext::default();
+    let opened = sandbox.cache::<Value>();
+    opened
+        .set_cache("key", json!({"answer": 42}), &context)
+        .unwrap();
+    assert_eq!(
+        opened.get_cache("key", &context).unwrap(),
+        Some(json!({"answer": 42}))
+    );
+    drop(opened);
+    let reopened = sandbox.cache::<Value>();
+    assert_eq!(
+        reopened.get_cache("key", &context).unwrap(),
+        Some(json!({"answer": 42}))
+    );
+}
+
+#[rstest]
+fn ttl_and_expired_culling_match_cache_contract(sandbox: Sandbox) {
+    let store = sandbox.store();
+    store
+        .set(
+            "expired",
+            StoredValue::Bytes(b"old".to_vec()),
+            Some(10.0),
+            0.0,
+        )
+        .unwrap();
+    assert_eq!(store.get("expired", 10.0).unwrap(), None);
+    store
+        .set("new", StoredValue::Bytes(b"new".to_vec()), None, 11.0)
+        .unwrap();
+    assert_eq!(
+        sandbox
+            .db()
+            .query_row("SELECT COUNT(*) FROM Cache", [], |row| row.get::<_, i64>(0))
+            .unwrap(),
+        1
+    );
+    assert_eq!(
+        sandbox
+            .db()
+            .query_row(
+                "SELECT value FROM Settings WHERE key = 'count'",
+                [],
+                |row| row.get::<_, i64>(0)
+            )
+            .unwrap(),
+        1
+    );
+}
+
+#[rstest]
+fn batch_preserves_order_and_classifies_misses_and_invalid_values(sandbox: Sandbox) {
+    let store = sandbox.store();
+    store
+        .set(
+            "hit",
+            StoredValue::Bytes(br#"{"ok":true}"#.to_vec()),
+            None,
+            0.0,
+        )
+        .unwrap();
+    store
+        .set(
+            "invalid",
+            StoredValue::Pickle(vec![0x80, 0x05, 0x2e]),
+            None,
+            0.0,
+        )
+        .unwrap();
+    let entries = sandbox
+        .cache::<Value>()
+        .batch_get_cache(
+            &["hit".into(), "missing".into(), "invalid".into()],
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    assert_eq!(
+        entries,
+        vec![
+            BatchEntry::Hit(json!({"ok": true})),
+            BatchEntry::Miss,
+            BatchEntry::Invalid
+        ]
+    );
+}
+
+#[rstest]
+#[case(StoredValue::Bytes(Vec::new()))]
+#[case(StoredValue::Text(String::new()))]
+#[case(StoredValue::Integer(0))]
+#[case(StoredValue::Float(0.0))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x89, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x00, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x94, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x5d, 0x94, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x7d, 0x94, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x29, 0x2e]))]
+fn falsy_values_are_misses(sandbox: Sandbox, #[case] value: StoredValue) {
+    sandbox.store().set("key", value, None, 0.0).unwrap();
+    assert_eq!(
+        sandbox
+            .cache::<Value>()
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap(),
+        None
+    );
+}
+
+#[rstest]
+#[case(Some(StoredValue::Integer(2)), 1.5, 3.5, "real")]
+#[case(Some(StoredValue::Integer(2)), 1.0, 3.0, "integer")]
+#[case(Some(StoredValue::Float(3.5)), 1.0, 1.0, "integer")]
+#[case(Some(StoredValue::Text("not a number".into())), 2.0, 2.0, "integer")]
+#[case(Some(StoredValue::Text("5".into())), 2.0, 7.0, "integer")]
+#[case(Some(StoredValue::Text("3.5".into())), 2.0, 2.0, "integer")]
+#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e])), 1.0, 2.0, "integer")]
+#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x02, 0x2e])), 1.0, 3.0, "integer")]
+#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e])), 1.0, 1.0, "integer")]
+#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e])), 4.0, 4.0, "integer")]
+fn counters_follow_python_initialization(
+    sandbox: Sandbox,
+    #[case] initial: Option<StoredValue>,
+    #[case] amount: f64,
+    #[case] expected: f64,
+    #[case] sqlite_type: &str,
+) {
+    if let Some(initial) = initial {
+        sandbox.store().set("counter", initial, None, 0.0).unwrap();
+    }
+    let cache = sandbox.cache::<f64>();
+    assert_eq!(
+        cache
+            .increment_cache("counter", amount, ExactCacheContext::default())
+            .unwrap(),
+        expected
+    );
+    assert_eq!(
+        sandbox
+            .db()
+            .query_row(
+                "SELECT typeof(value) FROM Cache WHERE key = 'counter'",
+                [],
+                |row| row.get::<_, String>(0)
+            )
+            .unwrap(),
+        sqlite_type
+    );
+}
+
+#[rstest]
+fn counters_are_atomic_across_concurrent_callers(sandbox: Sandbox) {
+    let cache = Arc::new(sandbox.cache::<f64>());
+    let workers = (0..8)
+        .map(|_| {
+            let cache = Arc::clone(&cache);
+            thread::spawn(move || {
+                for _ in 0..25 {
+                    cache
+                        .increment_cache("counter", 1.0, ExactCacheContext::default())
+                        .unwrap();
+                }
+            })
+        })
+        .collect::<Vec<_>>();
+    for worker in workers {
+        worker.join().unwrap();
+    }
+    assert_eq!(
+        cache
+            .increment_cache("counter", 0.0, ExactCacheContext::default())
+            .unwrap(),
+        200.0
+    );
+}
+
+#[rstest]
+fn fractional_then_integer_increment_follows_python_behavior(sandbox: Sandbox) {
+    let cache = sandbox.cache::<f64>();
+    assert_eq!(
+        cache
+            .increment_cache("counter", 3.5, ExactCacheContext::default())
+            .unwrap(),
+        3.5
+    );
+    assert_eq!(
+        cache
+            .increment_cache("counter", 1.0, ExactCacheContext::default())
+            .unwrap(),
+        1.0
+    );
+}
+
+#[rstest]
+fn increment_ttl_replacement_clears_expiry_without_ttl(sandbox: Sandbox) {
+    let cache = sandbox.cache::<f64>();
+    cache
+        .increment_cache(
+            "counter",
+            1.0,
+            ExactCacheContext {
+                ttl: Some(Duration::from_secs(60)),
+            },
+        )
+        .unwrap();
+    assert!(
+        sandbox
+            .db()
+            .query_row(
+                "SELECT expire_time IS NOT NULL FROM Cache WHERE key = 'counter'",
+                [],
+                |row| row.get::<_, bool>(0)
+            )
+            .unwrap()
+    );
+    cache
+        .increment_cache("counter", 1.0, ExactCacheContext::default())
+        .unwrap();
+    assert!(
+        !sandbox
+            .db()
+            .query_row(
+                "SELECT expire_time IS NOT NULL FROM Cache WHERE key = 'counter'",
+                [],
+                |row| row.get::<_, bool>(0)
+            )
+            .unwrap()
+    );
+}
+
+#[rstest]
+fn custom_adapter_controls_storage_and_reads(sandbox: Sandbox) {
+    let cache = DiskCache::with_adapter(sandbox.store(), TextAdapter, JsonCodec::<Value>::new());
+    cache
+        .set_cache("key", json!({"answer": 42}), &ExactCacheContext::default())
+        .unwrap();
+    assert!(matches!(
+        sandbox.store().get("key", 0.0).unwrap(),
+        Some(StoredValue::Text(_))
+    ));
+    assert_eq!(
+        cache
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap(),
+        Some(json!({"answer": 42}))
+    );
+}
+
+#[rstest]
+fn delete_flush_and_spilled_file_replacement_clean_up_storage(sandbox: Sandbox) {
+    let large = vec![b'x'; 32 * 1024];
+    sandbox
+        .store()
+        .set("large", StoredValue::Bytes(large.clone()), None, 0.0)
+        .unwrap();
+    assert_eq!(sandbox.value_files().len(), 1);
+    sandbox
+        .store()
+        .set(
+            "large",
+            StoredValue::Bytes(vec![b'y'; 32 * 1024]),
+            None,
+            0.0,
+        )
+        .unwrap();
+    assert_eq!(sandbox.value_files().len(), 1);
+    sandbox.store().pop("large", 0.0).unwrap();
+    assert!(sandbox.value_files().is_empty());
+    sandbox
+        .store()
+        .set("a", StoredValue::Bytes(large.clone()), None, 0.0)
+        .unwrap();
+    sandbox
+        .store()
+        .set("b", StoredValue::Bytes(large), None, 0.0)
+        .unwrap();
+    sandbox.store().clear().unwrap();
+    assert!(sandbox.value_files().is_empty());
+}
+
+#[rstest]
+#[tokio::test]
+async fn async_operations_connection_and_delete_match_sync_operations(sandbox: Sandbox) {
+    let cache = sandbox.cache::<Value>();
+    let context = ExactCacheContext {
+        ttl: Some(Duration::from_secs(60)),
+    };
+    cache
+        .async_set_cache("a", json!(1), context.clone())
+        .await
+        .unwrap();
+    cache
+        .async_set_cache_pipeline(
+            vec![("b".into(), json!(2)), ("c".into(), json!(3))],
+            context.clone(),
+        )
+        .await
+        .unwrap();
+    assert_eq!(
+        cache.async_get_cache("a", &context).await.unwrap(),
+        Some(json!(1))
+    );
+    assert_eq!(
+        cache
+            .async_batch_get_cache(vec!["c".into(), "missing".into()], context.clone())
+            .await
+            .unwrap(),
+        vec![BatchEntry::Hit(json!(3)), BatchEntry::Miss]
+    );
+    cache.async_delete_cache("a").await.unwrap();
+    cache.async_flush_cache().await.unwrap();
+    assert_eq!(
+        cache.test_connection().await.unwrap().status,
+        litellm_cache::CacheConnectionStatus::Success
+    );
+}

litellm-rust/crates/cache-disk/tests/python_compat.rs

@@ -0,0 +1,113 @@
+use litellm_cache::Error;
+use litellm_cache_disk::{PythonDiskCacheAdapter, StoredValue, ValueAdapter};
+use rstest::rstest;
+
+enum ReadExpectation {
+    Bytes(&'static [u8]),
+    Miss,
+    Invalid,
+}
+
+#[rstest]
+#[case::pickled_dictionary_with_string_keys(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e]),
+    ReadExpectation::Bytes(br#"{"a":1}"#)
+)]
+#[case::pickled_list_of_integers(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x94, 0x28, 0x4b, 0x01, 0x4b, 0x02, 0x65, 0x2e]),
+    ReadExpectation::Bytes(br#"[1,2]"#)
+)]
+#[case::pickled_tuple_of_integers(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x01, 0x4b, 0x02, 0x86, 0x94, 0x2e]),
+    ReadExpectation::Bytes(br#"[1,2]"#)
+)]
+#[case::pickled_set_of_integers(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x94, 0x28, 0x4b, 0x01, 0x4b, 0x02, 0x90, 0x2e]),
+    ReadExpectation::Bytes(br#"[1,2]"#)
+)]
+#[case::pickled_response_envelope(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x28, 0x8c, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x94, 0x47, 0x3f, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x94, 0x8c, 0x08, 0x7b, 0x22, 0x61, 0x22, 0x3a, 0x20, 0x31, 0x7d, 0x94, 0x75, 0x2e]),
+    ReadExpectation::Bytes(br#"{"response":"{\"a\": 1}","timestamp":1.5}"#)
+)]
+#[case::non_json_text(
+    StoredValue::Text("not json".into()),
+    ReadExpectation::Bytes(b"not json")
+)]
+#[case::json_text(
+    StoredValue::Text("{\"a\": 1}".into()),
+    ReadExpectation::Bytes(br#"{"a": 1}"#)
+)]
+#[case::non_utf8_bytes(
+    StoredValue::Bytes(vec![0xff, 0xfe]),
+    ReadExpectation::Bytes(&[0xff, 0xfe])
+)]
+#[case::integer_seven(StoredValue::Integer(7), ReadExpectation::Bytes(b"7"))]
+#[case::float_one_point_five(StoredValue::Float(1.5), ReadExpectation::Bytes(b"1.5"))]
+#[case::pickled_true(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e]),
+    ReadExpectation::Bytes(b"true")
+)]
+#[case::pickled_negative_integer(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0xfd, 0xff, 0xff, 0xff, 0x2e]),
+    ReadExpectation::Bytes(b"-3")
+)]
+#[case::pickled_bytes(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x05, 0x62, 0x79, 0x74, 0x65, 0x73, 0x94, 0x2e]),
+    ReadExpectation::Invalid
+)]
+#[case::pickled_dictionary_with_integer_key(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x4b, 0x01, 0x8c, 0x01, 0x61, 0x94, 0x73, 0x2e]),
+    ReadExpectation::Invalid
+)]
+#[case::pickled_complex(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x2e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x08, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x73, 0x94, 0x8c, 0x07, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x94, 0x93, 0x94, 0x47, 0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0x94, 0x52, 0x94, 0x2e]),
+    ReadExpectation::Invalid
+)]
+#[case::truncated_pickle(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x2e]),
+    ReadExpectation::Invalid
+)]
+#[case::empty_bytes(StoredValue::Bytes(Vec::new()), ReadExpectation::Miss)]
+#[case::empty_text(StoredValue::Text(String::new()), ReadExpectation::Miss)]
+#[case::zero_integer(StoredValue::Integer(0), ReadExpectation::Miss)]
+#[case::zero_float(StoredValue::Float(0.0), ReadExpectation::Miss)]
+#[case::pickled_none(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_false(StoredValue::Pickle(vec![0x80, 0x05, 0x89, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_zero(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x00, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_zero_float(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_empty_string(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x94, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_empty_list(StoredValue::Pickle(vec![0x80, 0x05, 0x5d, 0x94, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_empty_dictionary(StoredValue::Pickle(vec![0x80, 0x05, 0x7d, 0x94, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_empty_tuple(StoredValue::Pickle(vec![0x80, 0x05, 0x29, 0x2e]), ReadExpectation::Miss)]
+fn python_read_cases(#[case] row: StoredValue, #[case] expected: ReadExpectation) {
+    let result = PythonDiskCacheAdapter.read(row);
+    match expected {
+        ReadExpectation::Bytes(expected) => assert_eq!(result.unwrap().unwrap(), expected),
+        ReadExpectation::Miss => assert_eq!(result.unwrap(), None),
+        ReadExpectation::Invalid => assert!(matches!(result, Err(Error::InvalidEntry))),
+    }
+}
+
+#[rstest]
+#[case::integer_two(Some(StoredValue::Integer(2)), 2.0)]
+#[case::float_three_point_five(Some(StoredValue::Float(3.5)), 0.0)]
+#[case::text_not_a_number(Some(StoredValue::Text("not a number".into())), 0.0)]
+#[case::text_five(Some(StoredValue::Text("5".into())), 5.0)]
+#[case::text_three_point_five(Some(StoredValue::Text("3.5".into())), 0.0)]
+#[case::pickled_true(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e])), 1.0)]
+#[case::pickled_two(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x02, 0x2e])), 2.0)]
+#[case::pickled_dictionary(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e])), 0.0)]
+#[case::missing(None, 0.0)]
+#[case::pickled_none(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e])), 0.0)]
+fn python_counter_seed_cases(#[case] row: Option<StoredValue>, #[case] expected: f64) {
+    assert_eq!(PythonDiskCacheAdapter.counter_seed(row).unwrap(), expected);
+}
+
+#[rstest]
+#[case::integer_three(3.0, StoredValue::Integer(3))]
+#[case::fractional_three_point_five(3.5, StoredValue::Float(3.5))]
+#[case::negative_zero(-0.0, StoredValue::Integer(0))]
+#[case::large_float(1e300, StoredValue::Float(1e300))]
+fn python_counter_value_cases(#[case] value: f64, #[case] expected: StoredValue) {
+    assert_eq!(PythonDiskCacheAdapter.counter_value(value), expected);
+}

litellm-rust/crates/cache-gcs/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "litellm-cache-gcs"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+futures-util.workspace = true
+litellm-auth-gcp.workspace = true
+litellm-auth-types.workspace = true
+litellm-cache.workspace = true
+percent-encoding.workspace = true
+reqwest.workspace = true
+tokio.workspace = true
+
+[dev-dependencies]
+serde_json.workspace = true
+tokio.workspace = true
+wiremock = "0.6.5"

litellm-rust/crates/cache-gcs/src/cache.rs

@@ -0,0 +1,260 @@
+use std::{future::Future, sync::Arc, time::Duration};
+
+use futures_util::future::try_join_all;
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, Error, ExactCacheContext,
+    FlushCache,
+};
+use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode};
+use reqwest::Client;
+
+use crate::{GcpTokenSource, TokenSource};
+
+pub const DEFAULT_ENDPOINT: &str = "https://storage.googleapis.com";
+
+const OBJECT_NAME_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC
+    .remove(b'-')
+    .remove(b'_')
+    .remove(b'.')
+    .remove(b'~');
+
+pub fn key_prefix(gcs_path: Option<&str>) -> String {
+    match gcs_path {
+        Some(path) if !path.is_empty() => format!("{}/", path.trim_end_matches('/')),
+        _ => String::new(),
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct GcsConfig {
+    pub bucket_name: String,
+    pub gcs_path: Option<String>,
+    pub path_service_account: Option<String>,
+    pub endpoint: String,
+}
+
+impl GcsConfig {
+    pub fn new(bucket_name: impl Into<String>) -> Self {
+        Self {
+            bucket_name: bucket_name.into(),
+            gcs_path: None,
+            path_service_account: None,
+            endpoint: DEFAULT_ENDPOINT.to_string(),
+        }
+    }
+}
+
+pub struct GcsCache<S: CacheCodec> {
+    config: GcsConfig,
+    key_prefix: String,
+    client: Client,
+    token: Arc<dyn TokenSource>,
+    codec: S,
+}
+
+impl<S: CacheCodec> GcsCache<S> {
+    pub fn new(config: GcsConfig, codec: S) -> Result<Self, Error> {
+        let token = Arc::new(GcpTokenSource::new(config.path_service_account.clone()));
+        Self::with_token_source(config, codec, token)
+    }
+
+    pub fn with_token_source(
+        config: GcsConfig,
+        codec: S,
+        token: Arc<dyn TokenSource>,
+    ) -> Result<Self, Error> {
+        let client = Client::builder().build().map_err(|_| Error::Unavailable)?;
+        let key_prefix = key_prefix(config.gcs_path.as_deref());
+        Ok(Self {
+            config,
+            key_prefix,
+            client,
+            token,
+            codec,
+        })
+    }
+
+    pub fn bucket_name(&self) -> &str {
+        &self.config.bucket_name
+    }
+
+    pub fn key_prefix(&self) -> &str {
+        &self.key_prefix
+    }
+
+    pub fn path_service_account(&self) -> Option<&str> {
+        self.config.path_service_account.as_deref()
+    }
+
+    pub fn object_name(&self, key: &str) -> String {
+        format!("{}{}", self.key_prefix, key)
+    }
+
+    fn encoded_object_name(&self, key: &str) -> String {
+        percent_encode(self.object_name(key).as_bytes(), OBJECT_NAME_ENCODE_SET).to_string()
+    }
+
+    fn endpoint(&self, path: &str) -> String {
+        format!("{}{}", self.config.endpoint.trim_end_matches('/'), path)
+    }
+
+    async fn async_set(&self, key: &str, value: S::Value) -> Result<(), Error> {
+        let token = self.token.bearer_token().await?;
+        let payload = self.codec.encode(&value)?;
+        let url = self.endpoint(&format!(
+            "/upload/storage/v1/b/{}/o?uploadType=media&name={}",
+            self.config.bucket_name,
+            self.encoded_object_name(key)
+        ));
+        let response = self
+            .client
+            .post(url)
+            .bearer_auth(token)
+            .header(reqwest::header::CONTENT_TYPE, "application/json")
+            .body(payload)
+            .send()
+            .await
+            .map_err(|_| Error::Unavailable)?;
+        if !response.status().is_success() {
+            return Err(Error::Unavailable);
+        }
+        Ok(())
+    }
+
+    async fn async_get(&self, key: &str) -> Result<Option<S::Value>, Error> {
+        let token = self.token.bearer_token().await?;
+        let url = self.endpoint(&format!(
+            "/storage/v1/b/{}/o/{}?alt=media",
+            self.config.bucket_name,
+            self.encoded_object_name(key)
+        ));
+        let response = self
+            .client
+            .get(url)
+            .bearer_auth(token)
+            .header(reqwest::header::CONTENT_TYPE, "application/json")
+            .send()
+            .await
+            .map_err(|_| Error::Unavailable)?;
+        if response.status() == reqwest::StatusCode::NOT_FOUND {
+            return Ok(None);
+        }
+        if !response.status().is_success() {
+            return Err(Error::Unavailable);
+        }
+        let body = response.bytes().await.map_err(|_| Error::Unavailable)?;
+        self.codec
+            .decode(&body)
+            .map(Some)
+            .map_err(|_| Error::InvalidEntry)
+    }
+
+    fn run_sync<T, F>(future: F) -> Result<T, Error>
+    where
+        F: Future<Output = Result<T, Error>> + Send,
+        T: Send,
+    {
+        let run = || {
+            tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .map_err(|_| Error::Unavailable)
+                .and_then(|runtime| runtime.block_on(future))
+        };
+        if let Ok(handle) = tokio::runtime::Handle::try_current() {
+            if handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread {
+                return tokio::task::block_in_place(run);
+            }
+            return std::thread::scope(|scope| {
+                scope
+                    .spawn(run)
+                    .join()
+                    .map_err(|_| Error::Unavailable)
+                    .and_then(|result| result)
+            });
+        }
+        run()
+    }
+}
+
+impl<S: CacheCodec> BaseCache for GcsCache<S> {
+    type Value = S::Value;
+    type Context = ExactCacheContext;
+
+    fn get_ttl(&self, _: &Self::Context) -> Option<Duration> {
+        None
+    }
+
+    fn set_cache(&self, key: &str, value: Self::Value, _: &Self::Context) -> Result<(), Error> {
+        Self::run_sync(self.async_set(key, value))
+    }
+
+    fn get_cache(&self, key: &str, _: &Self::Context) -> Result<Option<Self::Value>, Error> {
+        Self::run_sync(self.async_get(key))
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        _: Self::Context,
+    ) -> Result<(), Error> {
+        self.async_set(key, value).await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        _: &Self::Context,
+    ) -> Result<Option<Self::Value>, Error> {
+        self.async_get(key).await
+    }
+
+    async fn async_set_cache_pipeline(
+        &self,
+        entries: Vec<(String, Self::Value)>,
+        context: Self::Context,
+    ) -> Result<(), Error> {
+        try_join_all(entries.into_iter().map(|(key, value)| {
+            let context = context.clone();
+            async move { self.async_set_cache(&key, value, context).await }
+        }))
+        .await
+        .map(|_| ())
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        Err(Error::UnsupportedOperation)
+    }
+}
+
+impl<S: CacheCodec> BatchCache for GcsCache<S> {
+    async fn async_batch_get_cache(
+        &self,
+        keys: Vec<String>,
+        context: Self::Context,
+    ) -> Result<Vec<BatchEntry<Self::Value>>, Error> {
+        try_join_all(keys.into_iter().map(|key| {
+            let context = context.clone();
+            async move {
+                match self.async_get_cache(&key, &context).await {
+                    Ok(Some(value)) => Ok(BatchEntry::Hit(value)),
+                    Ok(None) => Ok(BatchEntry::Miss),
+                    Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid),
+                    Err(error) => Err(error),
+                }
+            }
+        }))
+        .await
+    }
+}
+
+impl<S: CacheCodec> FlushCache for GcsCache<S> {
+    fn flush_cache(&self) -> Result<(), Error> {
+        Ok(())
+    }
+}

litellm-rust/crates/cache-gcs/src/lib.rs

@@ -0,0 +1,5 @@
+mod cache;
+mod token;
+
+pub use cache::{DEFAULT_ENDPOINT, GcsCache, GcsConfig, key_prefix};
+pub use token::{GcpTokenSource, StaticTokenSource, TokenSource};

litellm-rust/crates/cache-gcs/src/token.rs

@@ -0,0 +1,44 @@
+use std::{future::Future, pin::Pin};
+
+use litellm_auth_gcp::{VertexAuth, VertexConfig};
+use litellm_auth_types::{InputSource, SecretValue, Sourced};
+use litellm_cache::Error;
+
+pub trait TokenSource: Send + Sync + 'static {
+    fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>>;
+}
+
+pub struct GcpTokenSource {
+    auth: VertexAuth,
+    config: VertexConfig,
+}
+
+impl GcpTokenSource {
+    pub fn new(path_service_account: Option<String>) -> Self {
+        let credentials = path_service_account
+            .map(|path| Sourced::new(SecretValue::new(path), InputSource::Deployment));
+        Self {
+            auth: VertexAuth::default(),
+            config: VertexConfig::new(credentials, None, None),
+        }
+    }
+}
+
+impl TokenSource for GcpTokenSource {
+    fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>> {
+        Box::pin(async move {
+            self.auth
+                .access_token(&self.config, &|name| std::env::var(name).ok())
+                .await
+                .map_err(|_| Error::Unavailable)
+        })
+    }
+}
+
+pub struct StaticTokenSource(pub String);
+
+impl TokenSource for StaticTokenSource {
+    fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>> {
+        Box::pin(async move { Ok(self.0.clone()) })
+    }
+}

litellm-rust/crates/cache-gcs/tests/cache.rs

@@ -0,0 +1,324 @@
+use std::{sync::Arc, time::Duration};
+
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheContext, Error, ExactCacheContext, FlushCache,
+    JsonCodec,
+};
+use litellm_cache_gcs::{GcsCache, GcsConfig, StaticTokenSource, TokenSource, key_prefix};
+use serde_json::json;
+use wiremock::{
+    Mock, MockServer, ResponseTemplate,
+    matchers::{body_bytes, header, method, path, query_param},
+};
+
+fn config(server: &MockServer, gcs_path: Option<&str>) -> GcsConfig {
+    GcsConfig {
+        bucket_name: "bucket".into(),
+        gcs_path: gcs_path.map(str::to_string),
+        path_service_account: None,
+        endpoint: server.uri(),
+    }
+}
+
+fn cache(server: &MockServer, gcs_path: Option<&str>) -> GcsCache<JsonCodec<serde_json::Value>> {
+    GcsCache::with_token_source(
+        config(server, gcs_path),
+        JsonCodec::new(),
+        Arc::new(StaticTokenSource("tok".into())),
+    )
+    .unwrap()
+}
+
+#[tokio::test]
+async fn set_writes_encoded_object_and_headers() {
+    let server = MockServer::start().await;
+    Mock::given(method("POST"))
+        .and(path("/upload/storage/v1/b/bucket/o"))
+        .and(query_param("uploadType", "media"))
+        .and(header("authorization", "Bearer tok"))
+        .and(header("content-type", "application/json"))
+        .and(body_bytes(br#"{"value":"entry"}"#))
+        .respond_with(ResponseTemplate::new(200))
+        .expect(1)
+        .mount(&server)
+        .await;
+    cache(&server, Some("cache/"))
+        .set_cache(
+            "team:a b/c",
+            json!({"value": "entry"}),
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    let requests = server.received_requests().await.unwrap();
+    assert_eq!(requests.len(), 1);
+    assert_eq!(
+        requests[0].url.query(),
+        Some("uploadType=media&name=cache%2Fteam%3Aa%20b%2Fc")
+    );
+}
+
+#[tokio::test]
+async fn get_maps_statuses_and_decode_failures() {
+    let server = MockServer::start().await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/hit"))
+        .and(query_param("alt", "media"))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/missing"))
+        .respond_with(ResponseTemplate::new(404))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/server-error"))
+        .respond_with(ResponseTemplate::new(500))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/invalid"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("not json"))
+        .mount(&server)
+        .await;
+
+    let cache = cache(&server, None);
+    assert_eq!(
+        cache
+            .get_cache("hit", &ExactCacheContext::default())
+            .unwrap(),
+        Some(json!({"value": "entry"}))
+    );
+    assert_eq!(
+        cache
+            .get_cache("missing", &ExactCacheContext::default())
+            .unwrap(),
+        None
+    );
+    assert_eq!(
+        cache
+            .get_cache("server-error", &ExactCacheContext::default())
+            .unwrap_err(),
+        Error::Unavailable
+    );
+    assert_eq!(
+        cache
+            .get_cache("invalid", &ExactCacheContext::default())
+            .unwrap_err(),
+        Error::InvalidEntry
+    );
+}
+
+#[test]
+fn key_prefix_normalizes_paths() {
+    assert_eq!(key_prefix(None), "");
+    assert_eq!(key_prefix(Some("a/b/")), "a/b/");
+    assert_eq!(key_prefix(Some("a/b")), "a/b/");
+    assert_eq!(key_prefix(Some("")), "");
+}
+
+#[tokio::test]
+async fn object_names_use_python_quote_encoding() {
+    let server = MockServer::start().await;
+    Mock::given(method("POST"))
+        .and(path("/upload/storage/v1/b/bucket/o"))
+        .and(query_param("uploadType", "media"))
+        .respond_with(ResponseTemplate::new(200))
+        .expect(2)
+        .mount(&server)
+        .await;
+    let cache = cache(&server, Some("p/"));
+    cache
+        .set_cache(
+            "a~b-c_d.e/f g%h",
+            json!({"value": "punctuation"}),
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    cache
+        .set_cache(
+            "ключ",
+            json!({"value": "utf8"}),
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    let requests = server.received_requests().await.unwrap();
+    let queries: Vec<_> = requests
+        .iter()
+        .filter_map(|request| request.url.query())
+        .collect();
+    assert!(queries.contains(&"uploadType=media&name=p%2Fa~b-c_d.e%2Ff%20g%25h"));
+    assert!(queries.contains(&"uploadType=media&name=p%2F%D0%BA%D0%BB%D1%8E%D1%87"));
+}
+
+#[tokio::test]
+async fn ignores_ttl_and_writes_pipeline_concurrently() {
+    let server = MockServer::start().await;
+    for key in ["one", "two", "three"] {
+        Mock::given(method("POST"))
+            .and(path("/upload/storage/v1/b/bucket/o"))
+            .and(query_param("uploadType", "media"))
+            .and(query_param("name", key))
+            .respond_with(ResponseTemplate::new(200))
+            .expect(1)
+            .mount(&server)
+            .await;
+    }
+    let cache = cache(&server, None);
+    assert_eq!(cache.get_ttl(&ExactCacheContext::default()), None);
+    assert_eq!(
+        cache.get_ttl(&ExactCacheContext::default().with_ttl(Some(Duration::from_secs(5)))),
+        None
+    );
+    cache
+        .async_set_cache_pipeline(
+            vec![
+                ("one".into(), json!({"key": "one"})),
+                ("two".into(), json!({"key": "two"})),
+                ("three".into(), json!({"key": "three"})),
+            ],
+            ExactCacheContext::default().with_ttl(Some(Duration::from_secs(5))),
+        )
+        .await
+        .unwrap();
+}
+
+#[tokio::test]
+async fn async_batch_get_preserves_hits_misses_and_invalid_entries() {
+    let server = MockServer::start().await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/hit"))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/missing"))
+        .respond_with(ResponseTemplate::new(404))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/invalid"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("not json"))
+        .mount(&server)
+        .await;
+
+    assert_eq!(
+        cache(&server, None)
+            .async_batch_get_cache(
+                vec!["hit".into(), "missing".into(), "invalid".into()],
+                ExactCacheContext::default(),
+            )
+            .await
+            .unwrap(),
+        vec![
+            BatchEntry::Hit(json!({"value": "entry"})),
+            BatchEntry::Miss,
+            BatchEntry::Invalid,
+        ]
+    );
+}
+
+#[tokio::test]
+async fn lifecycle_operations_are_noops_and_connection_test_is_unsupported() {
+    let server = MockServer::start().await;
+    let cache = cache(&server, None);
+    assert_eq!(cache.flush_cache(), Ok(()));
+    assert_eq!(cache.disconnect().await, Ok(()));
+    assert_eq!(
+        cache.test_connection().await,
+        Err(Error::UnsupportedOperation)
+    );
+}
+
+#[test]
+fn sync_operations_work_without_an_active_runtime() {
+    let runtime = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+    let server = runtime.block_on(MockServer::start());
+    runtime.block_on(
+        Mock::given(method("POST"))
+            .and(path("/upload/storage/v1/b/bucket/o"))
+            .respond_with(ResponseTemplate::new(200))
+            .mount(&server),
+    );
+    runtime.block_on(
+        Mock::given(method("GET"))
+            .and(path("/storage/v1/b/bucket/o/key"))
+            .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})))
+            .mount(&server),
+    );
+    let cache = cache(&server, None);
+    cache
+        .set_cache(
+            "key",
+            json!({"value": "entry"}),
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    assert_eq!(
+        cache
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap(),
+        Some(json!({"value": "entry"}))
+    );
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn sync_operations_work_inside_a_multi_thread_runtime() {
+    let server = MockServer::start().await;
+    Mock::given(method("POST"))
+        .and(path("/upload/storage/v1/b/bucket/o"))
+        .respond_with(ResponseTemplate::new(200))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/key"))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})))
+        .mount(&server)
+        .await;
+    let cache = cache(&server, None);
+    cache
+        .set_cache(
+            "key",
+            json!({"value": "entry"}),
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    assert_eq!(
+        cache
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap(),
+        Some(json!({"value": "entry"}))
+    );
+}
+
+struct FailingTokenSource;
+
+impl TokenSource for FailingTokenSource {
+    fn bearer_token(
+        &self,
+    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<String, Error>> + Send + '_>>
+    {
+        Box::pin(async { Err(Error::Unavailable) })
+    }
+}
+
+#[tokio::test]
+async fn token_source_failure_skips_http() {
+    let server = MockServer::start().await;
+    let cache = GcsCache::with_token_source(
+        config(&server, None),
+        JsonCodec::<serde_json::Value>::new(),
+        Arc::new(FailingTokenSource),
+    )
+    .unwrap();
+    assert_eq!(
+        cache
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap_err(),
+        Error::Unavailable
+    );
+    assert_eq!(server.received_requests().await.unwrap().len(), 0);
+}

litellm-rust/crates/cache-memory/Cargo.toml

@@ -7,8 +7,8 @@ repository.workspace = true
 
 [dependencies]
 litellm-cache.workspace = true
-serde_json.workspace = true
 
 [dev-dependencies]
+serde_json.workspace = true
 rstest.workspace = true
 tokio.workspace = true

litellm-rust/crates/cache-memory/src/cache.rs

@@ -1,18 +1,20 @@
-use std::cmp::Reverse;
-use std::collections::{BinaryHeap, HashMap};
-use std::sync::{Arc, Mutex};
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use std::{
+    cmp::Reverse,
+    collections::{BinaryHeap, HashMap, HashSet},
+    hash::Hash,
+    sync::{Arc, Mutex},
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
 
 use litellm_cache::{
-    BaseCache, CacheConnectionResult, CacheConnectionStatus, CacheEntry, CacheFuture, CacheKwargs,
-    Error,
+    BaseCache, BatchCache, CacheConnectionResult, CacheConnectionStatus, ClaimCache, CounterCache,
+    DeleteCache, Error, ExactCacheContext, FlushCache, IncrementOperation, SetCache, TtlCache,
 };
 
 const DEFAULT_MAX_SIZE_IN_MEMORY: usize = 200;
 const DEFAULT_TTL: Duration = Duration::from_secs(600);
 
 type ValueMeasure<V> = Arc<dyn Fn(&V) -> Result<usize, Error> + Send + Sync>;
-type ValueValidator<V> = Arc<dyn Fn(&V) -> Result<(), Error> + Send + Sync>;
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum CacheWrite {
@@ -33,7 +35,6 @@ pub struct InMemoryCache<V: Clone> {
     default_ttl: Duration,
     max_entry_bytes: Option<usize>,
     measure_value: Option<ValueMeasure<V>>,
-    validate_value: Option<ValueValidator<V>>,
     now: Arc<dyn Fn() -> Duration + Send + Sync>,
 }
 
@@ -77,7 +78,6 @@ impl<V: Clone> InMemoryCache<V> {
             default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
             max_entry_bytes,
             measure_value,
-            validate_value: None,
             now: Arc::new(now),
         }
     }
@@ -91,9 +91,6 @@ impl<V: Clone> InMemoryCache<V> {
         if self.max_size_in_memory == 0 {
             return Ok(CacheWrite::Disabled);
         }
-        if let Some(validate) = &self.validate_value {
-            validate(&value)?;
-        }
         if let (Some(limit), Some(measure)) = (self.max_entry_bytes, &self.measure_value)
             && measure(&value)? > limit
         {
@@ -101,15 +98,13 @@ impl<V: Clone> InMemoryCache<V> {
         }
         let now = (self.now)();
         let mut state = self.state.lock().map_err(|_| Error::Unavailable)?;
-        Self::evict(&mut state, self.max_size_in_memory, now);
         let key = key.into();
-        state.values.insert(key.clone(), value);
+        Self::evict(&mut state, self.max_size_in_memory, now, &key);
         let expiration = state.expirations.get(&key).copied();
         if expiration.is_none_or(|expiration| expiration < now) {
-            let expiration = now + ttl.unwrap_or(self.default_ttl);
-            state.expirations.insert(key.clone(), expiration);
-            state.expiration_heap.push(Reverse((expiration, key)));
+            Self::set_expiration(&mut state, &key, now + ttl.unwrap_or(self.default_ttl));
         }
+        state.values.insert(key, value);
         Ok(CacheWrite::Stored)
     }
 
@@ -126,6 +121,14 @@ impl<V: Clone> InMemoryCache<V> {
         Ok(state.values.get(key).cloned())
     }
 
+    pub fn max_size_in_memory(&self) -> usize {
+        self.max_size_in_memory
+    }
+
+    pub fn max_entry_bytes(&self) -> Option<usize> {
+        self.max_entry_bytes
+    }
+
     pub fn expires_at(&self, key: &str) -> Result<Option<Duration>, Error> {
         Ok(self
             .state
@@ -136,6 +139,25 @@ impl<V: Clone> InMemoryCache<V> {
             .copied())
     }
 
+    pub async fn async_get_ttl(&self, key: &str) -> Result<Option<Duration>, Error> {
+        self.expires_at(key)
+    }
+
+    pub async fn async_get_oldest_n_keys(&self, count: usize) -> Result<Vec<String>, Error> {
+        let state = self.state.lock().map_err(|_| Error::Unavailable)?;
+        let mut expirations = state
+            .expirations
+            .iter()
+            .map(|(key, expiration)| (key.clone(), *expiration))
+            .collect::<Vec<_>>();
+        expirations.sort_unstable_by_key(|(_, expiration)| *expiration);
+        Ok(expirations
+            .into_iter()
+            .take(count)
+            .map(|(key, _)| key)
+            .collect())
+    }
+
     pub fn delete_cache(&self, key: &str) -> Result<(), Error> {
         let mut state = self.state.lock().map_err(|_| Error::Unavailable)?;
         Self::remove(&mut state, key);
@@ -150,7 +172,7 @@ impl<V: Clone> InMemoryCache<V> {
         Ok(())
     }
 
-    fn evict(state: &mut CacheState<V>, capacity: usize, now: Duration) {
+    fn evict(state: &mut CacheState<V>, capacity: usize, now: Duration, key: &str) {
         while let Some(Reverse((expiration, key))) = state.expiration_heap.peek().cloned() {
             if state.expirations.get(&key).copied() != Some(expiration) {
                 state.expiration_heap.pop();
@@ -161,6 +183,9 @@ impl<V: Clone> InMemoryCache<V> {
                 break;
             }
         }
+        if state.values.contains_key(key) {
+            return;
+        }
         while state.values.len() >= capacity {
             let Some(Reverse((expiration, key))) = state.expiration_heap.pop() else {
                 break;
@@ -171,84 +196,205 @@ impl<V: Clone> InMemoryCache<V> {
         }
     }
 
+    fn set_expiration(state: &mut CacheState<V>, key: &str, expiration: Duration) {
+        if state.expirations.get(key).copied() != Some(expiration) {
+            state.expirations.insert(key.into(), expiration);
+            state
+                .expiration_heap
+                .push(Reverse((expiration, key.into())));
+        }
+    }
+
     fn remove(state: &mut CacheState<V>, key: &str) {
         state.values.remove(key);
         state.expirations.remove(key);
     }
 }
 
-impl InMemoryCache<CacheEntry> {
-    pub fn response_cache(capacity: usize, ttl: Duration, max_entry_bytes: usize) -> Self {
-        Self::response_cache_with_clock(capacity, ttl, max_entry_bytes, || {
-            SystemTime::now()
-                .duration_since(UNIX_EPOCH)
-                .unwrap_or_default()
-        })
-    }
-
-    pub fn response_cache_with_clock(
-        capacity: usize,
-        ttl: Duration,
-        max_entry_bytes: usize,
-        now: impl Fn() -> Duration + Send + Sync + 'static,
-    ) -> Self {
-        let mut cache = Self::with_clock_and_size_measurement(
-            Some(capacity),
-            Some(ttl),
-            Some(max_entry_bytes),
-            Some(Arc::new(|entry: &CacheEntry| {
-                serde_json::to_vec(entry)
-                    .map(|bytes| bytes.len())
-                    .map_err(|_| Error::InvalidEntry)
-            })),
-            now,
+impl<V> ClaimCache for InMemoryCache<V>
+where
+    V: Clone + PartialEq + Send + Sync + 'static,
+{
+    fn claim_cache(
+        &self,
+        key: &str,
+        candidate: V,
+        eligible: &[V],
+        context: ExactCacheContext,
+    ) -> Result<V, Error> {
+        if self.max_size_in_memory == 0 {
+            return Ok(candidate);
+        }
+        let now = (self.now)();
+        let mut state = self.state.lock().map_err(|_| Error::Unavailable)?;
+        Self::evict(&mut state, self.max_size_in_memory, now, key);
+        let existing = state
+            .values
+            .get(key)
+            .filter(|existing| eligible.is_empty() || eligible.contains(existing))
+            .cloned();
+        if let Some(existing) = &existing
+            && eligible.is_empty()
+            && *existing != candidate
+        {
+            return Ok(existing.clone());
+        }
+        let winner = existing.unwrap_or(candidate);
+        Self::set_expiration(
+            &mut state,
+            key,
+            now + self.get_ttl(&context).unwrap_or(self.default_ttl),
         );
-        cache.validate_value = Some(Arc::new(|entry: &CacheEntry| {
-            entry
-                .timestamp
-                .is_finite()
-                .then_some(())
-                .ok_or(Error::InvalidEntry)
-        }));
-        cache
+        state.values.insert(key.into(), winner.clone());
+        Ok(winner)
     }
 }
 
-impl BaseCache for InMemoryCache<CacheEntry> {
-    type Value = CacheEntry;
+impl CounterCache for InMemoryCache<f64> {
+    fn increment_cache(
+        &self,
+        key: &str,
+        amount: f64,
+        context: ExactCacheContext,
+    ) -> Result<f64, Error> {
+        if self.max_size_in_memory == 0 {
+            return Ok(amount);
+        }
+        let now = (self.now)();
+        let mut state = self.state.lock().map_err(|_| Error::Unavailable)?;
+        Self::evict(&mut state, self.max_size_in_memory, now, key);
+        let value = state.values.get(key).copied().unwrap_or_default() + amount;
+        if !state.expirations.contains_key(key) {
+            Self::set_expiration(
+                &mut state,
+                key,
+                now + self.get_ttl(&context).unwrap_or(self.default_ttl),
+            );
+        }
+        state.values.insert(key.into(), value);
+        Ok(value)
+    }
+}
 
-    fn default_ttl(&self) -> Duration {
-        self.default_ttl
+impl InMemoryCache<f64> {
+    pub async fn async_increment_pipeline(
+        &self,
+        operations: Vec<IncrementOperation>,
+    ) -> Result<Vec<f64>, Error> {
+        operations
+            .into_iter()
+            .map(|operation| {
+                self.increment_cache(
+                    &operation.key,
+                    operation.amount,
+                    ExactCacheContext { ttl: operation.ttl },
+                )
+            })
+            .collect()
+    }
+}
+
+impl<V: Clone + Send + Sync + 'static> BaseCache for InMemoryCache<V> {
+    type Value = V;
+    type Context = ExactCacheContext;
+
+    fn get_ttl(&self, context: &Self::Context) -> Option<Duration> {
+        context.ttl.or(Some(self.default_ttl))
     }
 
-    fn set_cache(&self, key: &str, value: Self::Value, kwargs: CacheKwargs) -> Result<(), Error> {
-        let ttl = self.get_ttl(&kwargs);
+    fn set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: &ExactCacheContext,
+    ) -> Result<(), Error> {
+        let ttl = self.get_ttl(context).unwrap_or(self.default_ttl);
         self.set_cache(key, value, Some(ttl)).map(|_| ())
     }
 
-    fn get_cache(&self, key: &str, _: &CacheKwargs) -> Result<Option<Self::Value>, Error> {
+    fn get_cache(&self, key: &str, _: &ExactCacheContext) -> Result<Option<Self::Value>, Error> {
         self.get_cache(key)
     }
 
-    fn delete_cache(&self, key: &str) -> Result<(), Error> {
-        self.delete_cache(key)
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
     }
 
-    fn flush_cache(&self) -> Result<(), Error> {
-        self.flush_cache()
-    }
-
-    fn disconnect(&self) -> CacheFuture<'_, ()> {
-        Box::pin(async { Ok(()) })
-    }
-
-    fn test_connection(&self) -> CacheFuture<'_, CacheConnectionResult> {
-        Box::pin(async {
-            Ok(CacheConnectionResult {
-                status: CacheConnectionStatus::Success,
-                message: "In-memory cache connection test successful".into(),
-                error: None,
-            })
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        Ok(CacheConnectionResult {
+            status: CacheConnectionStatus::Success,
+            message: "In-memory cache connection test successful".into(),
+            error: None,
         })
     }
 }
+
+impl<V: Clone + Send + Sync + 'static> BatchCache for InMemoryCache<V> {}
+
+impl<V: Clone + Send + Sync + 'static> DeleteCache for InMemoryCache<V> {
+    fn delete_cache(&self, key: &str) -> Result<(), Error> {
+        InMemoryCache::delete_cache(self, key)
+    }
+}
+
+impl<V: Clone + Send + Sync + 'static> FlushCache for InMemoryCache<V> {
+    fn flush_cache(&self) -> Result<(), Error> {
+        InMemoryCache::flush_cache(self)
+    }
+}
+
+impl<V: Clone + Send + Sync + 'static> TtlCache for InMemoryCache<V> {
+    async fn async_get_ttl(&self, key: &str) -> Result<Option<Duration>, Error> {
+        InMemoryCache::async_get_ttl(self, key).await
+    }
+}
+
+impl<T> SetCache for InMemoryCache<HashSet<T>>
+where
+    T: Clone + Eq + Hash + Send + Sync + 'static,
+{
+    type SetValue = T;
+    type SetResult = Vec<T>;
+
+    async fn async_set_cache_sadd(
+        &self,
+        key: &str,
+        values: Vec<Self::SetValue>,
+        ttl: Option<Duration>,
+    ) -> Result<Self::SetResult, Error> {
+        if self.max_size_in_memory == 0 {
+            return Ok(values);
+        }
+        let now = (self.now)();
+        let mut state = self.state.lock().map_err(|_| Error::Unavailable)?;
+        Self::evict(&mut state, self.max_size_in_memory, now, key);
+        let mut stored = state.values.get(key).cloned().unwrap_or_default();
+        stored.extend(values.iter().cloned());
+        if let (Some(limit), Some(measure)) = (self.max_entry_bytes, &self.measure_value)
+            && measure(&stored)? > limit
+        {
+            return Ok(values);
+        }
+        if !state.expirations.contains_key(key) {
+            Self::set_expiration(&mut state, key, now + ttl.unwrap_or(self.default_ttl));
+        }
+        state.values.insert(key.into(), stored);
+        Ok(values)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn repeated_increments_keep_one_heap_entry_per_expiration() {
+        let cache = InMemoryCache::<f64>::new(Some(4), None);
+        for _ in 0..100 {
+            cache
+                .increment_cache("counter", 1.0, ExactCacheContext::default())
+                .unwrap();
+        }
+        assert_eq!(cache.state.lock().unwrap().expiration_heap.len(), 1);
+    }
+}

litellm-rust/crates/cache-memory/tests/cache.rs

@@ -1,8 +1,16 @@
-use std::sync::Arc;
-use std::sync::atomic::{AtomicU64, Ordering};
-use std::time::Duration;
+use std::{
+    collections::HashSet,
+    sync::{
+        Arc,
+        atomic::{AtomicU64, Ordering},
+    },
+    time::Duration,
+};
 
-use litellm_cache::{BaseCache, CacheConnectionStatus, CacheEntry, Error};
+use litellm_cache::{
+    BaseCache, CacheBackend, CacheConnectionStatus, ClaimCache, CounterCache, DeleteCache, Error,
+    ExactCacheContext, IncrementOperation, SetCache, get_cache, set_cache,
+};
 use litellm_cache_memory::{CacheWrite, InMemoryCache};
 use rstest::{fixture, rstest};
 
@@ -84,66 +92,49 @@ fn capacity_evicts_earliest_and_ignores_stale_heap_entries(clock: Arc<AtomicU64>
 }
 
 #[test]
-fn disabled_size_limited_and_synchronized_response_writes_are_observable() {
-    let disabled = InMemoryCache::<CacheEntry>::response_cache(0, Duration::from_secs(60), 80);
+fn disabled_size_limited_and_validated_writes_are_observable() {
+    let cache = |capacity| {
+        InMemoryCache::with_clock_and_size_measurement(
+            Some(capacity),
+            Some(Duration::from_secs(60)),
+            Some(4),
+            Some(Arc::new(|value: &String| {
+                if value.is_empty() {
+                    return Err(Error::InvalidEntry);
+                }
+                Ok(value.len())
+            })),
+            || Duration::from_secs(100),
+        )
+    };
+    let disabled = cache(0);
     assert_eq!(
-        disabled
-            .set_cache(
-                "a",
-                CacheEntry {
-                    timestamp: 1.0,
-                    response: serde_json::json!("x")
-                },
-                None
-            )
-            .unwrap(),
+        disabled.set_cache("a", "x".into(), None).unwrap(),
         CacheWrite::Disabled
     );
-    let cache = InMemoryCache::<CacheEntry>::response_cache(2, Duration::from_secs(60), 80);
+    let cache = cache(2);
     assert_eq!(
-        cache
-            .set_cache(
-                "large",
-                CacheEntry {
-                    timestamp: 1.0,
-                    response: serde_json::json!("x".repeat(100))
-                },
-                None
-            )
-            .unwrap(),
+        cache.set_cache("large", "oversized".into(), None).unwrap(),
         CacheWrite::TooLarge
     );
-    cache
-        .set_cache(
-            "small",
-            CacheEntry {
-                timestamp: 1.0,
-                response: serde_json::json!("ok"),
-            },
-            None,
-        )
-        .unwrap();
-    assert!(cache.get_cache("small").unwrap().is_some());
+    assert_eq!(cache.get_cache("large").unwrap(), None);
     assert_eq!(
-        cache
-            .set_cache(
-                "invalid",
-                CacheEntry {
-                    timestamp: f64::NAN,
-                    response: serde_json::json!("bad"),
-                },
-                None,
-            )
-            .unwrap_err(),
-        Error::InvalidEntry
+        cache.set_cache("small", "ok".into(), None).unwrap(),
+        CacheWrite::Stored
     );
+    assert_eq!(cache.get_cache("small").unwrap(), Some("ok".into()));
+    assert_eq!(
+        cache.set_cache("invalid", String::new(), None),
+        Err(Error::InvalidEntry)
+    );
+    assert_eq!(cache.get_cache("invalid").unwrap(), None);
     cache.delete_cache("small").unwrap();
-    cache.flush_cache().unwrap();
+    assert_eq!(cache.get_cache("small").unwrap(), None);
 }
 
 #[tokio::test]
 async fn connection_test_matches_python_result_contract() {
-    let cache = InMemoryCache::<CacheEntry>::default();
+    let cache = InMemoryCache::<String>::default();
     let result = BaseCache::test_connection(&cache).await.unwrap();
     assert_eq!(result.status, CacheConnectionStatus::Success);
     assert_eq!(result.message, "In-memory cache connection test successful");
@@ -156,3 +147,222 @@ async fn connection_test_matches_python_result_contract() {
         })
     );
 }
+
+#[tokio::test]
+async fn generic_consumers_share_typed_values_and_honor_expiration() {
+    let clock = clock();
+    let cache: CacheBackend<InMemoryCache<String>> = Arc::new(cache(clock.clone(), 4));
+    let reader = Arc::clone(&cache);
+    let context = ExactCacheContext {
+        ttl: Some(Duration::from_secs(5)),
+    };
+    set_cache(cache.as_ref(), "sync", "first".into(), &context).unwrap();
+    assert_eq!(
+        get_cache(reader.as_ref(), "sync", &context).unwrap(),
+        Some("first".into())
+    );
+    cache
+        .batch_cache_write("async", "second".into(), context.clone())
+        .await
+        .unwrap();
+    cache
+        .async_set_cache_pipeline(vec![("batch".into(), "third".into())], context.clone())
+        .await
+        .unwrap();
+    drop(cache);
+    for (key, value) in [("sync", "first"), ("async", "second"), ("batch", "third")] {
+        assert_eq!(
+            reader.async_get_cache(key, &context).await.unwrap(),
+            Some(value.into())
+        );
+    }
+    reader.async_delete_cache("async").await.unwrap();
+    assert_eq!(
+        reader.async_get_cache("async", &context).await.unwrap(),
+        None
+    );
+    clock.store(106, Ordering::SeqCst);
+    assert_eq!(get_cache(reader.as_ref(), "sync", &context).unwrap(), None);
+    assert_eq!(
+        reader.async_get_cache("batch", &context).await.unwrap(),
+        None
+    );
+}
+
+#[test]
+fn claims_are_atomic_and_refresh_eligible_winners() {
+    let clock = clock();
+    let cache = InMemoryCache::with_clock(Some(4), Some(Duration::from_secs(60)), {
+        let clock = clock.clone();
+        move || Duration::from_secs(clock.load(Ordering::SeqCst))
+    });
+    let context = ExactCacheContext {
+        ttl: Some(Duration::from_secs(10)),
+    };
+    assert_eq!(
+        cache
+            .claim_cache("affinity", "first".to_string(), &[], context.clone())
+            .unwrap(),
+        "first"
+    );
+    clock.store(103, Ordering::SeqCst);
+    assert_eq!(
+        cache
+            .claim_cache("affinity", "second".to_string(), &[], context.clone())
+            .unwrap(),
+        "first"
+    );
+    assert_eq!(
+        cache.expires_at("affinity").unwrap(),
+        Some(Duration::from_secs(110))
+    );
+    clock.store(105, Ordering::SeqCst);
+    assert_eq!(
+        cache
+            .claim_cache(
+                "affinity",
+                "second".to_string(),
+                &["first".to_string(), "second".to_string()],
+                context,
+            )
+            .unwrap(),
+        "first"
+    );
+    assert_eq!(
+        cache.expires_at("affinity").unwrap(),
+        Some(Duration::from_secs(115))
+    );
+}
+
+#[test]
+fn counters_increment_under_one_lock() {
+    let cache = InMemoryCache::<f64>::default();
+    assert_eq!(
+        CounterCache::increment_cache(&cache, "counter", 1.5, ExactCacheContext::default())
+            .unwrap(),
+        1.5
+    );
+    assert_eq!(
+        CounterCache::increment_cache(&cache, "counter", 2.0, ExactCacheContext::default())
+            .unwrap(),
+        3.5
+    );
+}
+
+#[rstest]
+fn rewriting_an_existing_key_at_capacity_keeps_other_entries(clock: Arc<AtomicU64>) {
+    let cache = cache(clock, 2);
+    cache
+        .set_cache("hot", "1".into(), Some(Duration::from_secs(10)))
+        .unwrap();
+    cache
+        .set_cache("cold", "2".into(), Some(Duration::from_secs(20)))
+        .unwrap();
+
+    cache.set_cache("cold", "3".into(), None).unwrap();
+    assert_eq!(cache.get_cache("hot").unwrap(), Some("1".into()));
+    assert_eq!(cache.get_cache("cold").unwrap(), Some("3".into()));
+
+    cache
+        .claim_cache("cold", "4".into(), &[], ExactCacheContext::default())
+        .unwrap();
+    assert_eq!(cache.get_cache("hot").unwrap(), Some("1".into()));
+
+    cache.set_cache("new", "5".into(), None).unwrap();
+    assert_eq!(cache.get_cache("hot").unwrap(), None);
+    assert_eq!(cache.get_cache("cold").unwrap(), Some("3".into()));
+    assert_eq!(cache.get_cache("new").unwrap(), Some("5".into()));
+}
+
+#[test]
+fn incrementing_an_existing_counter_at_capacity_keeps_every_counter() {
+    let cache = InMemoryCache::<f64>::new(Some(2), None);
+    for key in ["a", "b", "a", "b"] {
+        cache
+            .increment_cache(key, 1.0, ExactCacheContext::default())
+            .unwrap();
+    }
+    assert_eq!(cache.get_cache("a").unwrap(), Some(2.0));
+    assert_eq!(cache.get_cache("b").unwrap(), Some(2.0));
+}
+
+#[test]
+fn disabled_cache_does_not_retain_claims_or_counters() {
+    let claims = InMemoryCache::<String>::new(Some(0), None);
+    assert_eq!(
+        claims
+            .claim_cache("key", "first".into(), &[], ExactCacheContext::default())
+            .unwrap(),
+        "first"
+    );
+    assert_eq!(claims.get_cache("key").unwrap(), None);
+
+    let counters = InMemoryCache::<f64>::new(Some(0), None);
+    assert_eq!(
+        counters
+            .increment_cache("key", 2.0, ExactCacheContext::default())
+            .unwrap(),
+        2.0
+    );
+    assert_eq!(counters.get_cache("key").unwrap(), None);
+}
+
+#[tokio::test]
+async fn ttl_and_oldest_key_operations_use_the_stored_expirations() {
+    let clock = Arc::new(AtomicU64::new(100));
+    let cache = cache(clock, 3);
+    cache
+        .set_cache("later", "2".into(), Some(Duration::from_secs(20)))
+        .unwrap();
+    cache
+        .set_cache("first", "1".into(), Some(Duration::from_secs(10)))
+        .unwrap();
+
+    assert_eq!(
+        cache.async_get_ttl("first").await.unwrap(),
+        Some(Duration::from_secs(110))
+    );
+    assert_eq!(cache.async_get_oldest_n_keys(1).await.unwrap(), ["first"]);
+    assert_eq!(cache.async_get_ttl("missing").await.unwrap(), None);
+}
+
+#[tokio::test]
+async fn increment_pipeline_preserves_operation_order() {
+    let cache = InMemoryCache::<f64>::new(Some(3), None);
+    assert_eq!(
+        cache
+            .async_increment_pipeline(vec![
+                IncrementOperation {
+                    key: "a".into(),
+                    amount: 1.0,
+                    ttl: Some(Duration::from_secs(10)),
+                },
+                IncrementOperation {
+                    key: "a".into(),
+                    amount: 2.0,
+                    ttl: Some(Duration::from_secs(20)),
+                },
+            ])
+            .await
+            .unwrap(),
+        [1.0, 3.0]
+    );
+    assert_eq!(cache.get_cache("a").unwrap(), Some(3.0));
+}
+
+#[tokio::test]
+async fn set_capability_preserves_python_result_and_deduplicates_storage() {
+    let cache = InMemoryCache::<HashSet<String>>::new(None, None);
+    let inserted = vec!["a".into(), "a".into(), "b".into()];
+    assert_eq!(
+        cache
+            .async_set_cache_sadd("members", inserted.clone(), None)
+            .await
+            .unwrap(),
+        inserted
+    );
+    assert_eq!(
+        cache.get_cache("members").unwrap(),
+        Some(HashSet::from(["a".into(), "b".into()]))
+    );
+}

litellm-rust/crates/cache-qdrant-semantic/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "litellm-cache-qdrant-semantic"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+futures-util.workspace = true
+litellm-cache.workspace = true
+qdrant-client = { workspace = true, features = ["serde"] }
+reqwest.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+tokio.workspace = true
+uuid.workspace = true
+
+[dev-dependencies]
+litellm-cache-response.workspace = true
+rstest.workspace = true
+tonic = "0.14"
+tonic-prost = "0.14"
+tokio-stream = "0.1"

litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs

@@ -0,0 +1,75 @@
+use std::time::Duration;
+
+use litellm_cache::Error;
+use reqwest::Client;
+use serde_json::Value;
+
+use crate::Embedder;
+
+pub struct OpenAiEmbedder {
+    client: Client,
+    api_base: String,
+    api_key: String,
+    model: String,
+    timeout: Option<Duration>,
+}
+
+pub struct OpenAiEmbedderConfig {
+    pub api_base: String,
+    pub api_key: String,
+    pub model: String,
+    pub timeout: Option<Duration>,
+}
+
+impl OpenAiEmbedder {
+    pub fn new(client: Client, config: OpenAiEmbedderConfig) -> Self {
+        Self {
+            client,
+            api_base: config.api_base.trim_end_matches('/').to_owned(),
+            api_key: config.api_key,
+            model: config.model,
+            timeout: config.timeout,
+        }
+    }
+}
+
+impl Embedder for OpenAiEmbedder {
+    fn model(&self) -> &str {
+        &self.model
+    }
+
+    async fn embed(&self, input: &str) -> Result<Vec<f32>, Error> {
+        let request = self
+            .client
+            .post(format!("{}/embeddings", self.api_base))
+            .bearer_auth(&self.api_key)
+            .json(&serde_json::json!({
+                "model": self.model,
+                "input": input,
+                "encoding_format": "float",
+            }));
+        let response = if let Some(timeout) = self.timeout {
+            request.timeout(timeout)
+        } else {
+            request
+        }
+        .send()
+        .await
+        .map_err(|_| Error::Unavailable)?
+        .error_for_status()
+        .map_err(|_| Error::Unavailable)?;
+        let body: Value = response.json().await.map_err(|_| Error::Unavailable)?;
+        body.get("data")
+            .and_then(Value::as_array)
+            .and_then(|data| data.first())
+            .and_then(|item| item.get("embedding"))
+            .and_then(Value::as_array)
+            .and_then(|embedding| {
+                embedding
+                    .iter()
+                    .map(|value| value.as_f64().map(|value| value as f32))
+                    .collect::<Option<Vec<_>>>()
+            })
+            .ok_or(Error::Unavailable)
+    }
+}

litellm-rust/crates/cache-qdrant-semantic/src/lib.rs

@@ -0,0 +1,7 @@
+mod embedder;
+mod prompt;
+mod semantic;
+
+pub use embedder::{OpenAiEmbedder, OpenAiEmbedderConfig};
+pub use prompt::prompt_from_messages;
+pub use semantic::{Embedder, QdrantSemanticCache, QdrantSemanticConfig, Quantization};

litellm-rust/crates/cache-qdrant-semantic/src/prompt.rs

@@ -0,0 +1,59 @@
+use serde_json::Value;
+
+fn search_results_text(search_results: Option<&Value>) -> String {
+    let Some(Value::Array(results)) = search_results else {
+        return String::new();
+    };
+    results
+        .iter()
+        .filter_map(Value::as_object)
+        .flat_map(|result| {
+            let source = result
+                .get("source")
+                .and_then(Value::as_str)
+                .map(str::to_owned);
+            let title = result
+                .get("title")
+                .and_then(Value::as_str)
+                .map(str::to_owned);
+            let content = result
+                .get("content")
+                .and_then(Value::as_array)
+                .into_iter()
+                .flatten()
+                .filter_map(Value::as_object)
+                .filter_map(|block| block.get("text").and_then(Value::as_str).map(str::to_owned));
+            let citations = result
+                .get("citations")
+                .filter(|value| !value.is_null())
+                .map(|value| serde_json::to_string(value).unwrap_or_default());
+            source
+                .into_iter()
+                .chain(title)
+                .chain(content)
+                .chain(citations)
+        })
+        .collect()
+}
+
+pub fn prompt_from_messages(messages: &[Value]) -> String {
+    messages
+        .iter()
+        .filter_map(Value::as_object)
+        .map(|message| {
+            let content = match message.get("content") {
+                Some(Value::String(content)) => content.clone(),
+                Some(Value::Array(parts)) => parts
+                    .iter()
+                    .filter_map(Value::as_object)
+                    .filter_map(|part| part.get("text").and_then(Value::as_str))
+                    .collect(),
+                _ => String::new(),
+            };
+            format!(
+                "{content}{}",
+                search_results_text(message.get("search_results"))
+            )
+        })
+        .collect()
+}

litellm-rust/crates/cache-qdrant-semantic/src/semantic.rs

@@ -0,0 +1,262 @@
+use std::future::Future;
+
+use futures_util::future::try_join_all;
+use litellm_cache::{BaseCache, CacheCodec, CacheConnectionResult, Error, SemanticCacheContext};
+use qdrant_client::{
+    Payload, Qdrant,
+    qdrant::{
+        BinaryQuantizationBuilder, CompressionRatio, Condition, CreateCollectionBuilder,
+        CreateFieldIndexCollectionBuilder, Distance, FieldType, Filter, PointStruct,
+        ProductQuantizationBuilder, QuantizationSearchParamsBuilder, ScalarQuantizationBuilder,
+        SearchParamsBuilder, SearchPointsBuilder, UpsertPointsBuilder, VectorParamsBuilder,
+    },
+};
+use serde_json::{Map, Value, json};
+use uuid::Uuid;
+
+use crate::prompt_from_messages;
+
+pub trait Embedder: Send + Sync + 'static {
+    fn model(&self) -> &str;
+    fn embed(&self, input: &str) -> impl Future<Output = Result<Vec<f32>, Error>> + Send;
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum Quantization {
+    Binary,
+    Scalar,
+    Product,
+}
+
+pub struct QdrantSemanticConfig {
+    pub collection_name: String,
+    pub similarity_threshold: f64,
+    pub vector_size: u64,
+    pub quantization: Quantization,
+}
+
+pub struct QdrantSemanticCache<E: Embedder, C: CacheCodec> {
+    client: Qdrant,
+    embedder: E,
+    codec: C,
+    config: QdrantSemanticConfig,
+    runtime: tokio::runtime::Handle,
+}
+
+impl<E: Embedder, C: CacheCodec> QdrantSemanticCache<E, C> {
+    pub async fn connect(
+        client: Qdrant,
+        embedder: E,
+        codec: C,
+        config: QdrantSemanticConfig,
+        runtime: tokio::runtime::Handle,
+    ) -> Result<Self, Error> {
+        let exists = client
+            .collection_exists(config.collection_name.clone())
+            .await
+            .map_err(|_| Error::Unavailable)?;
+        if !exists {
+            client
+                .create_collection(
+                    CreateCollectionBuilder::new(config.collection_name.clone())
+                        .vectors_config(VectorParamsBuilder::new(
+                            config.vector_size,
+                            Distance::Cosine,
+                        ))
+                        .quantization_config(quantization(&config.quantization)),
+                )
+                .await
+                .map_err(|_| Error::Unavailable)?;
+        }
+        let _ = client
+            .create_field_index(CreateFieldIndexCollectionBuilder::new(
+                config.collection_name.clone(),
+                "litellm_cache_key".to_owned(),
+                FieldType::Keyword,
+            ))
+            .await;
+        Ok(Self {
+            client,
+            embedder,
+            codec,
+            config,
+            runtime,
+        })
+    }
+
+    pub fn collection_name(&self) -> &str {
+        &self.config.collection_name
+    }
+
+    pub fn similarity_threshold(&self) -> f64 {
+        self.config.similarity_threshold
+    }
+
+    pub fn vector_size(&self) -> u64 {
+        self.config.vector_size
+    }
+
+    pub fn embedder(&self) -> &E {
+        &self.embedder
+    }
+
+    fn prompt(context: &SemanticCacheContext) -> Result<String, Error> {
+        let Some(messages) = context.messages.as_ref().and_then(Value::as_array) else {
+            return Err(Error::MissingPrompt);
+        };
+        if messages.is_empty() {
+            return Err(Error::MissingPrompt);
+        }
+        Ok(prompt_from_messages(messages))
+    }
+
+    async fn set(
+        &self,
+        key: &str,
+        value: C::Value,
+        context: &SemanticCacheContext,
+    ) -> Result<(), Error> {
+        let prompt = Self::prompt(context)?;
+        let vector = self.embedder.embed(&prompt).await?;
+        let response =
+            String::from_utf8(self.codec.encode(&value)?).map_err(|_| Error::InvalidEntry)?;
+        let payload = Payload::try_from(json!({
+            "litellm_cache_key": key,
+            "text": prompt,
+            "response": response,
+        }))
+        .map_err(|_| Error::InvalidEntry)?;
+        self.client
+            .upsert_points(
+                UpsertPointsBuilder::new(
+                    self.collection_name(),
+                    vec![PointStruct::new(
+                        Uuid::new_v4().to_string(),
+                        vector,
+                        payload,
+                    )],
+                )
+                .wait(true),
+            )
+            .await
+            .map_err(|_| Error::Unavailable)?;
+        Ok(())
+    }
+
+    async fn get(
+        &self,
+        key: &str,
+        context: &SemanticCacheContext,
+    ) -> Result<Option<C::Value>, Error> {
+        let prompt = Self::prompt(context)?;
+        let vector = self.embedder.embed(&prompt).await?;
+        let result = self
+            .client
+            .search_points(
+                SearchPointsBuilder::new(self.collection_name(), vector, 1)
+                    .with_payload(true)
+                    .filter(Filter::must([Condition::matches(
+                        "litellm_cache_key",
+                        key.to_owned(),
+                    )]))
+                    .params(
+                        SearchParamsBuilder::default().quantization(
+                            QuantizationSearchParamsBuilder::default()
+                                .ignore(false)
+                                .rescore(true)
+                                .oversampling(3.0),
+                        ),
+                    ),
+            )
+            .await
+            .map_err(|_| Error::Unavailable)?;
+        let Some(point) = result.result.into_iter().next() else {
+            return Ok(None);
+        };
+        let payload: Map<String, Value> = Payload::from(point.payload).into();
+        if payload.get("litellm_cache_key").and_then(Value::as_str) != Some(key) {
+            return Ok(None);
+        }
+        if f64::from(point.score) < self.config.similarity_threshold {
+            return Ok(None);
+        }
+        let response = payload
+            .get("response")
+            .and_then(Value::as_str)
+            .ok_or(Error::InvalidEntry)?;
+        self.codec.decode(response.as_bytes()).map(Some)
+    }
+}
+
+fn quantization(value: &Quantization) -> qdrant_client::qdrant::quantization_config::Quantization {
+    match value {
+        Quantization::Binary => BinaryQuantizationBuilder::new(false).into(),
+        Quantization::Scalar => ScalarQuantizationBuilder::default()
+            .quantile(0.99)
+            .always_ram(false)
+            .into(),
+        Quantization::Product => ProductQuantizationBuilder::new(CompressionRatio::X16.into())
+            .always_ram(false)
+            .into(),
+    }
+}
+
+impl<E: Embedder, C: CacheCodec> BaseCache for QdrantSemanticCache<E, C> {
+    type Value = C::Value;
+    type Context = SemanticCacheContext;
+
+    fn get_ttl(&self, _: &Self::Context) -> Option<std::time::Duration> {
+        None
+    }
+
+    fn set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: &Self::Context,
+    ) -> Result<(), Error> {
+        self.runtime.block_on(self.set(key, value, context))
+    }
+
+    fn get_cache(&self, key: &str, context: &Self::Context) -> Result<Option<Self::Value>, Error> {
+        self.runtime.block_on(self.get(key, context))
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: Self::Context,
+    ) -> Result<(), Error> {
+        self.set(key, value, &context).await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        context: &Self::Context,
+    ) -> Result<Option<Self::Value>, Error> {
+        self.get(key, context).await
+    }
+
+    async fn async_set_cache_pipeline(
+        &self,
+        entries: Vec<(String, Self::Value)>,
+        context: Self::Context,
+    ) -> Result<(), Error> {
+        try_join_all(entries.into_iter().map(|(key, value)| {
+            let context = context.clone();
+            async move { self.async_set_cache(&key, value, context).await }
+        }))
+        .await
+        .map(|_| ())
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        Err(Error::UnsupportedOperation)
+    }
+}

litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs

@@ -0,0 +1,166 @@
+use std::{
+    sync::{Arc, Mutex},
+    time::Duration,
+};
+
+use litellm_cache::Error;
+use litellm_cache_qdrant_semantic::{Embedder, OpenAiEmbedder, OpenAiEmbedderConfig};
+use serde_json::Value;
+use tokio::{
+    io::{AsyncReadExt, AsyncWriteExt},
+    net::TcpListener,
+};
+
+struct TestHttpServer {
+    address: std::net::SocketAddr,
+    request: Arc<Mutex<Option<Vec<u8>>>>,
+    task: tokio::task::JoinHandle<()>,
+}
+
+impl TestHttpServer {
+    async fn response(status: &str, body: &str) -> Self {
+        Self::response_after(status, body, Duration::ZERO).await
+    }
+
+    async fn response_after(status: &str, body: &str, delay: Duration) -> Self {
+        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let address = listener.local_addr().unwrap();
+        let request = Arc::new(Mutex::new(None));
+        let captured = request.clone();
+        let status = status.to_owned();
+        let body = body.to_owned();
+        let task = tokio::spawn(async move {
+            let (mut stream, _) = listener.accept().await.unwrap();
+            let request_bytes = read_request(&mut stream).await;
+            *captured.lock().unwrap() = Some(request_bytes);
+            tokio::time::sleep(delay).await;
+            let response = format!(
+                "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+                body.len()
+            );
+            stream.write_all(response.as_bytes()).await.unwrap();
+        });
+        Self {
+            address,
+            request,
+            task,
+        }
+    }
+
+    fn base_url(&self) -> String {
+        format!("http://{}", self.address)
+    }
+}
+
+impl Drop for TestHttpServer {
+    fn drop(&mut self) {
+        self.task.abort();
+    }
+}
+
+async fn read_request(stream: &mut tokio::net::TcpStream) -> Vec<u8> {
+    let mut bytes = Vec::new();
+    let header_end = loop {
+        let mut chunk = [0_u8; 1024];
+        let count = stream.read(&mut chunk).await.unwrap();
+        assert_ne!(count, 0);
+        bytes.extend_from_slice(&chunk[..count]);
+        if let Some(end) = bytes.windows(4).position(|window| window == b"\r\n\r\n") {
+            break end + 4;
+        }
+    };
+    let headers = String::from_utf8_lossy(&bytes[..header_end]);
+    let content_length = headers
+        .lines()
+        .find_map(|line| {
+            line.split_once(':')
+                .filter(|(name, _)| name.eq_ignore_ascii_case("content-length"))
+                .map(|(_, value)| value.trim())
+        })
+        .unwrap()
+        .parse::<usize>()
+        .unwrap();
+    while bytes.len() < header_end + content_length {
+        let mut chunk = [0_u8; 1024];
+        let count = stream.read(&mut chunk).await.unwrap();
+        assert_ne!(count, 0);
+        bytes.extend_from_slice(&chunk[..count]);
+    }
+    bytes
+}
+
+fn config(base: String, timeout: Option<Duration>) -> OpenAiEmbedderConfig {
+    OpenAiEmbedderConfig {
+        api_base: base,
+        api_key: "test-key".to_owned(),
+        model: "test-model".to_owned(),
+        timeout,
+    }
+}
+
+#[tokio::test]
+async fn posts_embeddings_request_and_parses_vector() {
+    let server = TestHttpServer::response("200 OK", r#"{"data":[{"embedding":[0.1,0.2]}]}"#).await;
+    let embedder = OpenAiEmbedder::new(
+        reqwest::Client::new(),
+        config(
+            format!("{}/", server.base_url()),
+            Some(Duration::from_secs(1)),
+        ),
+    );
+    assert_eq!(embedder.embed("hello").await.unwrap(), vec![0.1, 0.2]);
+    let request = server.request.lock().unwrap().clone().unwrap();
+    let request_text = String::from_utf8(request).unwrap();
+    assert!(request_text.starts_with("POST /embeddings HTTP/1.1\r\n"));
+    assert!(request_text.contains("\r\nauthorization: Bearer test-key\r\n"));
+    let body = request_text.split("\r\n\r\n").nth(1).unwrap();
+    let body: Value = serde_json::from_str(body).unwrap();
+    assert_eq!(body["model"], "test-model");
+    assert_eq!(body["input"], "hello");
+    assert_eq!(body["encoding_format"], "float");
+}
+
+#[tokio::test]
+async fn status_and_timeout_errors_are_unavailable() {
+    let server = TestHttpServer::response("500 Internal Server Error", "{}").await;
+    let embedder = OpenAiEmbedder::new(reqwest::Client::new(), config(server.base_url(), None));
+    assert_eq!(embedder.embed("hello").await, Err(Error::Unavailable));
+
+    let server = TestHttpServer::response_after(
+        "200 OK",
+        r#"{"data":[{"embedding":[0.1,0.2]}]}"#,
+        Duration::from_millis(500),
+    )
+    .await;
+    let embedder = OpenAiEmbedder::new(
+        reqwest::Client::new(),
+        config(server.base_url(), Some(Duration::from_millis(200))),
+    );
+    assert_eq!(embedder.embed("hello").await, Err(Error::Unavailable));
+
+    let server = TestHttpServer::response_after(
+        "200 OK",
+        r#"{"data":[{"embedding":[0.1,0.2]}]}"#,
+        Duration::from_millis(100),
+    )
+    .await;
+    let embedder = OpenAiEmbedder::new(
+        reqwest::Client::new(),
+        config(server.base_url(), Some(Duration::from_secs(1))),
+    );
+    assert_eq!(embedder.embed("hello").await.unwrap(), vec![0.1, 0.2]);
+}
+
+#[tokio::test]
+async fn uses_the_injected_client() {
+    let server = TestHttpServer::response("200 OK", r#"{"data":[{"embedding":[0.1,0.2]}]}"#).await;
+    let client = reqwest::Client::builder()
+        .user_agent("litellm-embedder-test")
+        .build()
+        .unwrap();
+    let embedder = OpenAiEmbedder::new(client, config(server.base_url(), None));
+    assert_eq!(embedder.embed("hello").await.unwrap(), vec![0.1, 0.2]);
+    let request = server.request.lock().unwrap().clone().unwrap();
+    let request_text = String::from_utf8(request).unwrap();
+    assert!(request_text.contains("\r\nuser-agent: litellm-embedder-test\r\n"));
+}

litellm-rust/crates/cache-qdrant-semantic/tests/prompt.rs

@@ -0,0 +1,38 @@
+use litellm_cache_qdrant_semantic::prompt_from_messages;
+use serde_json::json;
+
+#[test]
+fn prompt_matches_python_message_content_rules() {
+    let messages = vec![
+        json!({"role": "user", "content": "hello"}),
+        json!({
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "world"},
+                {"type": "image_url", "image_url": {"url": "ignored"}},
+                {"type": "text", "text": "!"},
+            ],
+        }),
+    ];
+
+    assert_eq!(prompt_from_messages(&messages), "helloworld!");
+}
+
+#[test]
+fn prompt_includes_search_result_text_and_compact_citations() {
+    let messages = vec![json!({
+        "role": "tool",
+        "content": null,
+        "search_results": [{
+            "source": "source",
+            "title": "title",
+            "content": [{"text": "body"}],
+            "citations": {"page": 1, "section": "intro"},
+        }],
+    })];
+
+    assert_eq!(
+        prompt_from_messages(&messages),
+        r#"sourcetitlebody{"page":1,"section":"intro"}"#
+    );
+}

litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs

@@ -0,0 +1,422 @@
+#[path = "support/mod.rs"]
+mod support;
+
+use std::{collections::HashMap, sync::Arc, time::Duration};
+
+use litellm_cache::{BaseCache, CacheCodec, CacheContext, Error, SemanticCacheContext};
+use litellm_cache_qdrant_semantic::{
+    Embedder, QdrantSemanticCache, QdrantSemanticConfig, Quantization,
+};
+use litellm_cache_response::{
+    CacheEntry, CacheKeyInput, ResponseCache, ResponseCacheCodec, ResponseCacheRequest,
+};
+use qdrant_client::Payload;
+use qdrant_client::{
+    Qdrant,
+    qdrant::{self, CompressionRatio, Distance, PointId, QuantizationType, Value, VectorParams},
+};
+use serde_json::{Value as JsonValue, json};
+
+use support::{FakeQdrant, FakeState, StoredPoint};
+
+#[derive(Clone)]
+struct FixedEmbedder {
+    vectors: Arc<HashMap<String, Vec<f32>>>,
+}
+
+impl FixedEmbedder {
+    fn new(vectors: impl IntoIterator<Item = (&'static str, Vec<f32>)>) -> Self {
+        Self {
+            vectors: Arc::new(
+                vectors
+                    .into_iter()
+                    .map(|(prompt, vector)| (prompt.to_owned(), vector))
+                    .collect(),
+            ),
+        }
+    }
+}
+
+impl Embedder for FixedEmbedder {
+    fn model(&self) -> &str {
+        "fixed"
+    }
+
+    async fn embed(&self, input: &str) -> Result<Vec<f32>, Error> {
+        self.vectors.get(input).cloned().ok_or(Error::Unavailable)
+    }
+}
+
+fn config(quantization: Quantization) -> QdrantSemanticConfig {
+    QdrantSemanticConfig {
+        collection_name: "semantic".to_owned(),
+        similarity_threshold: 0.9,
+        vector_size: 2,
+        quantization,
+    }
+}
+
+fn context(prompt: &str) -> SemanticCacheContext {
+    SemanticCacheContext {
+        messages: Some(json!([{"role": "user", "content": prompt}])),
+        ..Default::default()
+    }
+}
+
+fn value(response: JsonValue) -> CacheEntry {
+    CacheEntry {
+        timestamp: Some(1.0),
+        response,
+    }
+}
+
+async fn connect(
+    server: &FakeQdrant,
+    vectors: impl IntoIterator<Item = (&'static str, Vec<f32>)>,
+) -> QdrantSemanticCache<FixedEmbedder, ResponseCacheCodec> {
+    let client = Qdrant::from_url(&server.url()).build().unwrap();
+    QdrantSemanticCache::connect(
+        client,
+        FixedEmbedder::new(vectors),
+        ResponseCacheCodec,
+        config(Quantization::Binary),
+        tokio::runtime::Handle::current(),
+    )
+    .await
+    .unwrap()
+}
+
+#[tokio::test(flavor = "multi_thread")]
+#[expect(
+    deprecated,
+    reason = "the test verifies Qdrant's legacy always_ram quantization contract"
+)]
+async fn connect_sets_collection_quantization_and_index() {
+    for (quantization, expected) in [
+        (Quantization::Binary, 0),
+        (Quantization::Scalar, 1),
+        (Quantization::Product, 2),
+    ] {
+        let server = FakeQdrant::start(FakeState::default()).await;
+        let client = Qdrant::from_url(&server.url()).build().unwrap();
+        QdrantSemanticCache::connect(
+            client,
+            FixedEmbedder::new([]),
+            ResponseCacheCodec,
+            config(quantization),
+            tokio::runtime::Handle::current(),
+        )
+        .await
+        .unwrap();
+        let state = server.state.lock().unwrap();
+        let request = &state.created_collections[0];
+        let Some(qdrant::vectors_config::Config::Params(VectorParams { size, distance, .. })) =
+            request
+                .vectors_config
+                .as_ref()
+                .and_then(|config| config.config.clone())
+        else {
+            panic!("missing vector params");
+        };
+        assert_eq!(size, 2);
+        assert_eq!(distance, Distance::Cosine as i32);
+        let quantization_config = request
+            .quantization_config
+            .as_ref()
+            .unwrap()
+            .quantization
+            .unwrap();
+        match (expected, quantization_config) {
+            (0, qdrant::quantization_config::Quantization::Binary(binary)) => {
+                assert_eq!(binary.always_ram, Some(false));
+            }
+            (1, qdrant::quantization_config::Quantization::Scalar(scalar)) => {
+                assert_eq!(scalar.r#type, QuantizationType::Int8 as i32);
+                assert_eq!(scalar.quantile, Some(0.99));
+                assert_eq!(scalar.always_ram, Some(false));
+            }
+            (2, qdrant::quantization_config::Quantization::Product(product)) => {
+                assert_eq!(product.compression, CompressionRatio::X16 as i32);
+                assert_eq!(product.always_ram, Some(false));
+            }
+            _ => panic!("unexpected quantization"),
+        }
+        assert!(state.index_creations >= 1);
+        assert_eq!(state.field_indexes[0].collection_name, "semantic");
+        assert_eq!(state.field_indexes[0].field_name, "litellm_cache_key");
+        assert_eq!(
+            state.field_indexes[0].field_type,
+            Some(qdrant::FieldType::Keyword as i32)
+        );
+        server.stop();
+    }
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn existing_collection_skips_create_and_index_failure_is_non_fatal() {
+    let server = FakeQdrant::start(FakeState {
+        collections: ["semantic".to_owned()].into_iter().collect(),
+        fail_field_index: true,
+        ..Default::default()
+    })
+    .await;
+    let _cache = connect(&server, [("hello", vec![1.0, 0.0])]).await;
+    let state = server.state.lock().unwrap();
+    assert!(state.created_collections.is_empty());
+    assert!(state.index_creations >= 1);
+    server.stop();
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn async_and_sync_set_get_store_exact_payload() {
+    let server = FakeQdrant::start(FakeState::default()).await;
+    let cache = Arc::new(connect(&server, [("hello", vec![1.0, 0.0])]).await);
+    let ctx = context("hello");
+    let entry = value(json!({"answer": 42}));
+    cache
+        .async_set_cache("key", entry.clone(), ctx.clone())
+        .await
+        .unwrap();
+    assert_eq!(
+        cache.async_get_cache("key", &ctx).await.unwrap().as_ref(),
+        Some(&entry)
+    );
+    {
+        let state = server.state.lock().unwrap();
+        let payload = &state.points[0].payload;
+        let mut payload_keys = payload.keys().cloned().collect::<Vec<_>>();
+        payload_keys.sort();
+        assert_eq!(payload_keys, ["litellm_cache_key", "response", "text"]);
+        assert_eq!(payload["litellm_cache_key"], Value::from("key"));
+        assert_eq!(
+            payload["response"],
+            Value::from(String::from_utf8(ResponseCacheCodec.encode(&entry).unwrap()).unwrap())
+        );
+    }
+    let sync_entry = entry.clone();
+    let sync_cache = cache.clone();
+    let sync_ctx = ctx.clone();
+    tokio::task::spawn_blocking(move || {
+        sync_cache
+            .set_cache("sync", sync_entry.clone(), &sync_ctx)
+            .unwrap();
+        assert_eq!(
+            sync_cache.get_cache("sync", &sync_ctx).unwrap(),
+            Some(sync_entry)
+        );
+    })
+    .await
+    .unwrap();
+    server.stop();
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn misses_and_payload_validation_are_safe() {
+    let server = FakeQdrant::start(FakeState::default()).await;
+    let cache = connect(
+        &server,
+        [("hello", vec![1.0, 0.0]), ("near", vec![0.7, 0.71414286])],
+    )
+    .await;
+    let entry = value(json!({"answer": 1}));
+    cache
+        .async_set_cache("key", entry, context("hello"))
+        .await
+        .unwrap();
+    assert_eq!(
+        cache
+            .async_get_cache("other", &context("hello"))
+            .await
+            .unwrap(),
+        None
+    );
+    assert_eq!(
+        cache
+            .async_get_cache("key", &context("near"))
+            .await
+            .unwrap(),
+        None
+    );
+    server.insert_point(StoredPoint {
+        id: Some(PointId::from(99_u64)),
+        vector: vec![1.0, 0.0],
+        payload: Payload::try_from(json!({
+            "litellm_cache_key": 99,
+            "response": "{}",
+        }))
+        .unwrap()
+        .into(),
+    });
+    assert_eq!(
+        cache
+            .async_get_cache("99", &context("hello"))
+            .await
+            .unwrap(),
+        None
+    );
+    server.stop();
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn decoding_errors_missing_prompt_pipeline_and_ttl_behave_as_required() {
+    let server = FakeQdrant::start(FakeState::default()).await;
+    let cache = connect(&server, [("one", vec![1.0, 0.0]), ("two", vec![0.0, 1.0])]).await;
+    let empty = SemanticCacheContext::default();
+    assert_eq!(
+        cache
+            .async_set_cache("key", value(json!({})), empty.clone())
+            .await,
+        Err(Error::MissingPrompt)
+    );
+    assert_eq!(
+        cache.async_get_cache("key", &empty).await,
+        Err(Error::MissingPrompt)
+    );
+    assert_eq!(
+        cache.async_get_cache("key", &context("unknown")).await,
+        Err(Error::Unavailable)
+    );
+    cache
+        .async_set_cache(
+            "ttl",
+            value(json!({"ttl": true})),
+            context("one").with_ttl(Some(Duration::from_secs(1))),
+        )
+        .await
+        .unwrap();
+    tokio::time::sleep(Duration::from_millis(1_100)).await;
+    assert!(
+        cache
+            .async_get_cache(
+                "ttl",
+                &context("one").with_ttl(Some(Duration::from_secs(1))),
+            )
+            .await
+            .unwrap()
+            .is_some()
+    );
+    cache
+        .async_set_cache_pipeline(
+            vec![
+                ("one".to_owned(), value(json!({"n": 1}))),
+                ("two".to_owned(), value(json!({"n": 2}))),
+            ],
+            context("one"),
+        )
+        .await
+        .unwrap();
+    assert!(
+        cache
+            .async_get_cache("one", &context("one"))
+            .await
+            .unwrap()
+            .is_some()
+    );
+    assert!(
+        cache
+            .async_get_cache("two", &context("one"))
+            .await
+            .unwrap()
+            .is_some()
+    );
+    assert_eq!(
+        server.state.lock().unwrap().upsert_waits,
+        vec![Some(true), Some(true), Some(true)]
+    );
+    assert_eq!(cache.get_ttl(&context("one")), None);
+    assert_eq!(
+        cache.test_connection().await,
+        Err(Error::UnsupportedOperation)
+    );
+    server.stop();
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn response_payloads_decode_and_invalid_entries_fail() {
+    let server = FakeQdrant::start(FakeState::default()).await;
+    let cache = connect(&server, [("hello", vec![1.0, 0.0])]).await;
+    for (key, response) in [
+        ("python", json!("{'timestamp': 1.0, 'response': {'a': 1}}")),
+        ("garbage", json!("not json")),
+        ("missing", json!("unused")),
+    ] {
+        let mut payload = serde_json::Map::new();
+        payload.insert("litellm_cache_key".to_owned(), json!(key));
+        if key != "missing" {
+            payload.insert("response".to_owned(), response);
+        }
+        server.insert_point(StoredPoint {
+            id: Some(PointId::from(key.len() as u64)),
+            vector: vec![1.0, 0.0],
+            payload: Payload::try_from(JsonValue::Object(payload))
+                .unwrap()
+                .into(),
+        });
+    }
+    assert_eq!(
+        cache
+            .async_get_cache("python", &context("hello"))
+            .await
+            .unwrap(),
+        Some(value(json!({"a": 1})))
+    );
+    assert_eq!(
+        cache.async_get_cache("garbage", &context("hello")).await,
+        Err(Error::InvalidEntry)
+    );
+    assert_eq!(
+        cache.async_get_cache("missing", &context("hello")).await,
+        Err(Error::InvalidEntry)
+    );
+    server.stop();
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn response_cache_facade_turns_invalid_entry_into_miss() {
+    let server = FakeQdrant::start(FakeState::default()).await;
+    let cache = Arc::new(connect(&server, [("hello", vec![1.0, 0.0])]).await);
+    let request = ResponseCacheRequest::<SemanticCacheContext>::new(CacheKeyInput {
+        preset: Some("key".to_owned()),
+        ..Default::default()
+    })
+    .with_context(context("hello"));
+    let response = json!({"answer": 42});
+    let facade = ResponseCache::new(cache.clone());
+    facade
+        .async_store(&request, response.clone(), Duration::from_secs(1))
+        .await
+        .unwrap();
+    assert_eq!(
+        facade
+            .async_lookup(&request, Duration::from_secs(1))
+            .await
+            .unwrap(),
+        Some(response)
+    );
+    {
+        let mut state = server.state.lock().unwrap();
+        state.points[0]
+            .payload
+            .insert("response".to_owned(), Value::from("not json"));
+    }
+    assert_eq!(
+        facade
+            .async_lookup(&request, Duration::from_secs(1))
+            .await
+            .unwrap(),
+        None
+    );
+    server.stop();
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn stopped_qdrant_server_maps_to_unavailable() {
+    let server = FakeQdrant::start(FakeState::default()).await;
+    let cache = connect(&server, [("hello", vec![1.0, 0.0])]).await;
+    server.stop();
+    tokio::time::sleep(Duration::from_millis(50)).await;
+    assert_eq!(
+        cache.async_get_cache("key", &context("hello")).await,
+        Err(Error::Unavailable)
+    );
+}

litellm-rust/crates/cache-qdrant-semantic/tests/support/mod.rs

@@ -0,0 +1,342 @@
+use std::{
+    collections::{HashMap, HashSet},
+    net::SocketAddr,
+    sync::{Arc, Mutex},
+};
+
+use qdrant_client::qdrant::collections_server::CollectionsServer;
+use qdrant_client::qdrant::{
+    self, CollectionExists, CollectionExistsRequest, CollectionExistsResponse,
+    CollectionOperationResponse, CreateCollection, CreateFieldIndexCollection, Filter, PointId,
+    PointsOperationResponse, ScoredPoint, SearchPoints, SearchResponse, Value, Vector, Vectors,
+    collections_server::Collections,
+    points_server::{Points, PointsServer},
+};
+use tokio::sync::oneshot;
+use tokio_stream::wrappers::TcpListenerStream;
+use tonic::{Request, Response, Status, transport::Server};
+
+#[derive(Clone, Debug)]
+pub struct StoredPoint {
+    pub id: Option<PointId>,
+    pub vector: Vec<f32>,
+    pub payload: HashMap<String, Value>,
+}
+
+#[derive(Default)]
+pub struct FakeState {
+    pub collections: HashSet<String>,
+    pub created_collections: Vec<CreateCollection>,
+    pub field_indexes: Vec<CreateFieldIndexCollection>,
+    pub points: Vec<StoredPoint>,
+    pub upsert_waits: Vec<Option<bool>>,
+    pub index_creations: usize,
+    pub fail_field_index: bool,
+}
+
+#[derive(Clone)]
+pub struct FakeQdrant {
+    pub state: Arc<Mutex<FakeState>>,
+    pub address: SocketAddr,
+    shutdown: Arc<Mutex<Option<oneshot::Sender<()>>>>,
+}
+
+impl FakeQdrant {
+    pub async fn start(state: FakeState) -> Self {
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let address = listener.local_addr().unwrap();
+        let state = Arc::new(Mutex::new(state));
+        let service = FakeService {
+            state: state.clone(),
+        };
+        let (shutdown_tx, shutdown_rx) = oneshot::channel();
+        tokio::spawn(async move {
+            Server::builder()
+                .add_service(CollectionsServer::new(service.clone()))
+                .add_service(PointsServer::new(service))
+                .serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
+                    let _ = shutdown_rx.await;
+                })
+                .await
+                .unwrap();
+        });
+        Self {
+            state,
+            address,
+            shutdown: Arc::new(Mutex::new(Some(shutdown_tx))),
+        }
+    }
+
+    pub fn url(&self) -> String {
+        format!("http://{}", self.address)
+    }
+
+    pub fn stop(&self) {
+        self.shutdown
+            .lock()
+            .unwrap()
+            .take()
+            .unwrap()
+            .send(())
+            .unwrap();
+    }
+
+    pub fn insert_point(&self, point: StoredPoint) {
+        self.state.lock().unwrap().points.push(point);
+    }
+}
+
+#[derive(Clone)]
+struct FakeService {
+    state: Arc<Mutex<FakeState>>,
+}
+
+macro_rules! unimplemented_collections {
+    ($($name:ident, $request:ty, $response:ty);* $(;)?) => {
+        $(
+            fn $name<'life0, 'async_trait>(
+                &'life0 self,
+                _: Request<$request>,
+            ) -> std::pin::Pin<
+                Box<
+                    dyn std::future::Future<
+                            Output = Result<Response<$response>, Status>,
+                        > + Send
+                        + 'async_trait,
+                >,
+            >
+            where
+                'life0: 'async_trait,
+                Self: 'async_trait,
+            {
+                Box::pin(async { Err(Status::unimplemented(stringify!($name))) })
+            }
+        )*
+    };
+}
+
+macro_rules! unimplemented_points {
+    ($($name:ident, $request:ty, $response:ty);* $(;)?) => {
+        $(
+            fn $name<'life0, 'async_trait>(
+                &'life0 self,
+                _: Request<$request>,
+            ) -> std::pin::Pin<
+                Box<
+                    dyn std::future::Future<
+                            Output = Result<Response<$response>, Status>,
+                        > + Send
+                        + 'async_trait,
+                >,
+            >
+            where
+                'life0: 'async_trait,
+                Self: 'async_trait,
+            {
+                Box::pin(async { Err(Status::unimplemented(stringify!($name))) })
+            }
+        )*
+    };
+}
+
+#[tonic::async_trait]
+impl Collections for FakeService {
+    async fn create(
+        &self,
+        request: Request<CreateCollection>,
+    ) -> Result<Response<CollectionOperationResponse>, Status> {
+        let request = request.into_inner();
+        let mut state = self.state.lock().unwrap();
+        state.collections.insert(request.collection_name.clone());
+        state.created_collections.push(request);
+        Ok(Response::new(CollectionOperationResponse {
+            result: true,
+            ..Default::default()
+        }))
+    }
+
+    async fn collection_exists(
+        &self,
+        request: Request<CollectionExistsRequest>,
+    ) -> Result<Response<CollectionExistsResponse>, Status> {
+        let exists = self
+            .state
+            .lock()
+            .unwrap()
+            .collections
+            .contains(&request.into_inner().collection_name);
+        Ok(Response::new(CollectionExistsResponse {
+            result: Some(CollectionExists { exists }),
+            ..Default::default()
+        }))
+    }
+
+    unimplemented_collections!(
+        get, qdrant::GetCollectionInfoRequest, qdrant::GetCollectionInfoResponse;
+        list, qdrant::ListCollectionsRequest, qdrant::ListCollectionsResponse;
+        update, qdrant::UpdateCollection, qdrant::CollectionOperationResponse;
+        delete, qdrant::DeleteCollection, qdrant::CollectionOperationResponse;
+        update_aliases, qdrant::ChangeAliases, qdrant::CollectionOperationResponse;
+        list_collection_aliases, qdrant::ListCollectionAliasesRequest, qdrant::ListAliasesResponse;
+        list_aliases, qdrant::ListAliasesRequest, qdrant::ListAliasesResponse;
+        collection_cluster_info, qdrant::CollectionClusterInfoRequest, qdrant::CollectionClusterInfoResponse;
+        update_collection_cluster_setup, qdrant::UpdateCollectionClusterSetupRequest, qdrant::UpdateCollectionClusterSetupResponse;
+        create_shard_key, qdrant::CreateShardKeyRequest, qdrant::CreateShardKeyResponse;
+        delete_shard_key, qdrant::DeleteShardKeyRequest, qdrant::DeleteShardKeyResponse;
+        list_shard_keys, qdrant::ListShardKeysRequest, qdrant::ListShardKeysResponse;
+    );
+}
+
+#[tonic::async_trait]
+impl Points for FakeService {
+    async fn create_field_index(
+        &self,
+        request: Request<CreateFieldIndexCollection>,
+    ) -> Result<Response<PointsOperationResponse>, Status> {
+        let mut state = self.state.lock().unwrap();
+        state.index_creations += 1;
+        state.field_indexes.push(request.into_inner());
+        if state.fail_field_index {
+            return Err(Status::internal("field index failure"));
+        }
+        Ok(Response::new(PointsOperationResponse::default()))
+    }
+
+    async fn upsert(
+        &self,
+        request: Request<qdrant::UpsertPoints>,
+    ) -> Result<Response<PointsOperationResponse>, Status> {
+        let request = request.into_inner();
+        let mut state = self.state.lock().unwrap();
+        state.upsert_waits.push(request.wait);
+        for point in request.points {
+            let stored = StoredPoint {
+                id: point.id.clone(),
+                vector: dense_vector(point.vectors)?,
+                payload: point.payload,
+            };
+            if let Some(existing) = state
+                .points
+                .iter_mut()
+                .find(|existing| existing.id == stored.id)
+            {
+                *existing = stored;
+            } else {
+                state.points.push(stored);
+            }
+        }
+        Ok(Response::new(PointsOperationResponse::default()))
+    }
+
+    async fn search(
+        &self,
+        request: Request<SearchPoints>,
+    ) -> Result<Response<SearchResponse>, Status> {
+        let request = request.into_inner();
+        let key_filter = keyword_filter(request.filter.as_ref());
+        let state = self.state.lock().unwrap();
+        let mut results = state
+            .points
+            .iter()
+            .filter(|point| {
+                key_filter.as_ref().is_none_or(|(field, expected)| {
+                    point
+                        .payload
+                        .get(field)
+                        .and_then(|value| {
+                            let value: serde_json::Value = value.clone().into();
+                            value
+                                .as_str()
+                                .map(str::to_owned)
+                                .or_else(|| value.as_i64().map(|value| value.to_string()))
+                        })
+                        .is_some_and(|value| value == *expected)
+                })
+            })
+            .map(|point| ScoredPoint {
+                id: point.id.clone(),
+                payload: point.payload.clone(),
+                score: cosine(&request.vector, &point.vector),
+                ..Default::default()
+            })
+            .collect::<Vec<_>>();
+        results.sort_by(|left, right| right.score.total_cmp(&left.score));
+        results.truncate(request.limit as usize);
+        Ok(Response::new(SearchResponse {
+            result: results,
+            ..Default::default()
+        }))
+    }
+
+    unimplemented_points!(
+        delete, qdrant::DeletePoints, qdrant::PointsOperationResponse;
+        get, qdrant::GetPoints, qdrant::GetResponse;
+        update_vectors, qdrant::UpdatePointVectors, qdrant::PointsOperationResponse;
+        delete_vectors, qdrant::DeletePointVectors, qdrant::PointsOperationResponse;
+        set_payload, qdrant::SetPayloadPoints, qdrant::PointsOperationResponse;
+        overwrite_payload, qdrant::SetPayloadPoints, qdrant::PointsOperationResponse;
+        delete_payload, qdrant::DeletePayloadPoints, qdrant::PointsOperationResponse;
+        clear_payload, qdrant::ClearPayloadPoints, qdrant::PointsOperationResponse;
+        delete_field_index, qdrant::DeleteFieldIndexCollection, qdrant::PointsOperationResponse;
+        create_vector_name, qdrant::CreateVectorNameRequest, qdrant::PointsOperationResponse;
+        delete_vector_name, qdrant::DeleteVectorNameRequest, qdrant::PointsOperationResponse;
+        search_batch, qdrant::SearchBatchPoints, qdrant::SearchBatchResponse;
+        search_groups, qdrant::SearchPointGroups, qdrant::SearchGroupsResponse;
+        scroll, qdrant::ScrollPoints, qdrant::ScrollResponse;
+        recommend, qdrant::RecommendPoints, qdrant::RecommendResponse;
+        recommend_batch, qdrant::RecommendBatchPoints, qdrant::RecommendBatchResponse;
+        recommend_groups, qdrant::RecommendPointGroups, qdrant::RecommendGroupsResponse;
+        discover, qdrant::DiscoverPoints, qdrant::DiscoverResponse;
+        discover_batch, qdrant::DiscoverBatchPoints, qdrant::DiscoverBatchResponse;
+        count, qdrant::CountPoints, qdrant::CountResponse;
+        update_batch, qdrant::UpdateBatchPoints, qdrant::UpdateBatchResponse;
+        query, qdrant::QueryPoints, qdrant::QueryResponse;
+        query_batch, qdrant::QueryBatchPoints, qdrant::QueryBatchResponse;
+        query_groups, qdrant::QueryPointGroups, qdrant::QueryGroupsResponse;
+        facet, qdrant::FacetCounts, qdrant::FacetResponse;
+        search_matrix_pairs, qdrant::SearchMatrixPoints, qdrant::SearchMatrixPairsResponse;
+        search_matrix_offsets, qdrant::SearchMatrixPoints, qdrant::SearchMatrixOffsetsResponse;
+    );
+}
+
+fn dense_vector(vectors: Option<Vectors>) -> Result<Vec<f32>, Status> {
+    let Some(Vectors {
+        vectors_options:
+            Some(qdrant::vectors::VectorsOptions::Vector(Vector {
+                vector: Some(qdrant::vector::Vector::Dense(qdrant::DenseVector { data })),
+                ..
+            })),
+    }) = vectors
+    else {
+        return Err(Status::invalid_argument("expected dense vector"));
+    };
+    Ok(data)
+}
+
+fn keyword_filter(filter: Option<&Filter>) -> Option<(String, String)> {
+    filter?
+        .must
+        .iter()
+        .find_map(|condition| match condition.condition_one_of.as_ref()? {
+            qdrant::condition::ConditionOneOf::Field(field) => {
+                let qdrant::r#match::MatchValue::Keyword(value) =
+                    field.r#match.as_ref()?.match_value.as_ref()?
+                else {
+                    return None;
+                };
+                Some((field.key.clone(), value.clone()))
+            }
+            _ => None,
+        })
+}
+
+fn cosine(left: &[f32], right: &[f32]) -> f32 {
+    let dot = left
+        .iter()
+        .zip(right)
+        .map(|(left, right)| left * right)
+        .sum::<f32>();
+    let left_norm = left.iter().map(|value| value * value).sum::<f32>().sqrt();
+    let right_norm = right.iter().map(|value| value * value).sum::<f32>().sqrt();
+    dot / (left_norm * right_norm)
+}

litellm-rust/crates/cache-redis-semantic/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "litellm-cache-redis-semantic"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-cache.workspace = true
+litellm-cache-redis.workspace = true
+litellm-cache-response.workspace = true
+redis = { version = "1.7.0", features = ["tls-rustls"] }
+r2d2 = "0.8.10"
+serde_json.workspace = true
+sha2.workspace = true
+tokio.workspace = true
+
+[dev-dependencies]
+redis-test = "1.0.4"
+serde_json.workspace = true
+tokio.workspace = true

litellm-rust/crates/cache-redis-semantic/src/cache.rs

@@ -0,0 +1,618 @@
+use std::{
+    future::Future,
+    sync::{Arc, OnceLock},
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+
+use litellm_cache::{
+    BaseCache, CacheCodec, CacheConnectionResult, CacheConnectionStatus, Error,
+    SemanticCacheContext,
+};
+use litellm_cache_redis::{
+    RedisTopology,
+    connection::{ConnectionRef, Connections},
+};
+use litellm_cache_response::{CacheEntry, ResponseCacheCodec};
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+
+use crate::prompt::prompt_from_context;
+
+const CACHE_KEY_FIELD: &str = "litellm_cache_key";
+const VECTOR_FIELD: &str = "prompt_vector";
+
+pub trait Embedder: Send + Sync + 'static {
+    fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, Error>;
+
+    fn async_embed(
+        &self,
+        prompt: &str,
+        metadata: Option<&Value>,
+    ) -> impl Future<Output = Result<Vec<f32>, Error>> + Send;
+}
+
+#[derive(Clone, Debug)]
+pub struct RedisSemanticConfig {
+    pub index_name: String,
+    pub similarity_threshold: f32,
+}
+
+struct Inner {
+    index_name: String,
+    distance_threshold: f64,
+    resolved_index: OnceLock<String>,
+    codec: ResponseCacheCodec,
+    clock: fn() -> f64,
+}
+
+impl Inner {
+    fn new(config: RedisSemanticConfig) -> Self {
+        Self {
+            index_name: config.index_name,
+            distance_threshold: 1.0 - f64::from(config.similarity_threshold),
+            resolved_index: OnceLock::new(),
+            codec: ResponseCacheCodec,
+            clock: timestamp,
+        }
+    }
+
+    fn ensure_index(
+        &self,
+        connection: &mut ConnectionRef<'_>,
+        dims: usize,
+    ) -> Result<String, Error> {
+        if let Some(name) = self.resolved_index.get() {
+            return Ok(name.clone());
+        }
+        let name = match index_compatible(connection, &self.index_name, dims)? {
+            Some(true) => self.index_name.clone(),
+            Some(false) => self.isolated_index(connection, dims)?,
+            None => match create_index(connection, &self.index_name, dims) {
+                Ok(()) => self.index_name.clone(),
+                Err(_) => match index_compatible(connection, &self.index_name, dims)? {
+                    Some(true) => self.index_name.clone(),
+                    Some(false) => self.isolated_index(connection, dims)?,
+                    None => return Err(Error::Unavailable),
+                },
+            },
+        };
+        let _ = self.resolved_index.set(name.clone());
+        Ok(name)
+    }
+
+    fn isolated_index(
+        &self,
+        connection: &mut ConnectionRef<'_>,
+        dims: usize,
+    ) -> Result<String, Error> {
+        let name = format!("{}_isolated", self.index_name);
+        match index_compatible(connection, &name, dims)? {
+            Some(true) => Ok(name),
+            Some(false) => {
+                redis::cmd("FT.DROPINDEX")
+                    .arg(&name)
+                    .query::<()>(connection)
+                    .map_err(|_| Error::Unavailable)?;
+                create_index(connection, &name, dims)?;
+                Ok(name)
+            }
+            None => {
+                create_index(connection, &name, dims)?;
+                Ok(name)
+            }
+        }
+    }
+
+    fn store(
+        &self,
+        connection: &mut ConnectionRef<'_>,
+        tag: &str,
+        value: &CacheEntry,
+        prompt: &str,
+        vector: &[f32],
+        ttl: Option<Duration>,
+    ) -> Result<(), Error> {
+        let index = self.ensure_index(connection, vector.len())?;
+        let entry_id = entry_id(prompt, tag);
+        let hash_key = format!("{index}:{entry_id}");
+        let response = self.codec.encode(value)?;
+        redis::cmd("HSET")
+            .arg(&hash_key)
+            .arg("entry_id")
+            .arg(&entry_id)
+            .arg("prompt")
+            .arg(prompt)
+            .arg("response")
+            .arg(response)
+            .arg(VECTOR_FIELD)
+            .arg(vector_buffer(vector))
+            .arg("inserted_at")
+            .arg(format!("{}", (self.clock)()))
+            .arg("updated_at")
+            .arg(format!("{}", (self.clock)()))
+            .arg(CACHE_KEY_FIELD)
+            .arg(tag)
+            .query::<()>(connection)
+            .map_err(|_| Error::Unavailable)?;
+        if let Some(ttl) = ttl {
+            redis::cmd("EXPIRE")
+                .arg(&hash_key)
+                .arg(ttl_seconds(ttl))
+                .query::<()>(connection)
+                .map_err(|_| Error::Unavailable)?;
+        }
+        Ok(())
+    }
+
+    fn lookup(
+        &self,
+        connection: &mut ConnectionRef<'_>,
+        tag: &str,
+        vector: &[f32],
+    ) -> Result<Option<CacheEntry>, Error> {
+        let index = self.ensure_index(connection, vector.len())?;
+        let query = format!(
+            "(@{CACHE_KEY_FIELD}:{{{}}})=>[KNN 1 @{VECTOR_FIELD} $vector AS vector_distance]",
+            escape_tag(tag)
+        );
+        let result = redis::cmd("FT.SEARCH")
+            .arg(&index)
+            .arg(query)
+            .arg("RETURN")
+            .arg(8)
+            .arg("entry_id")
+            .arg("prompt")
+            .arg("response")
+            .arg("inserted_at")
+            .arg("updated_at")
+            .arg("metadata")
+            .arg(CACHE_KEY_FIELD)
+            .arg("vector_distance")
+            .arg("SORTBY")
+            .arg("vector_distance")
+            .arg("ASC")
+            .arg("DIALECT")
+            .arg(2)
+            .arg("LIMIT")
+            .arg(0)
+            .arg(1)
+            .arg("PARAMS")
+            .arg(2)
+            .arg("vector")
+            .arg(vector_buffer(vector))
+            .query::<redis::Value>(connection)
+            .map_err(|_| Error::Unavailable)?;
+        let Some(fields) = first_document(&result) else {
+            return Ok(None);
+        };
+        if string_field(fields, CACHE_KEY_FIELD).as_deref() != Some(tag) {
+            return Ok(None);
+        }
+        if number_field(fields, "vector_distance")
+            .is_none_or(|distance| distance > self.distance_threshold)
+        {
+            return Ok(None);
+        }
+        let Some(response) = bytes_field(fields, "response") else {
+            return Ok(None);
+        };
+        self.codec.decode(&response).map(Some)
+    }
+}
+
+pub struct RedisSemanticCache<E: Embedder, C = redis::Connection> {
+    connections: Arc<Connections<C>>,
+    embedder: E,
+    inner: Arc<Inner>,
+}
+
+impl<E: Embedder> RedisSemanticCache<E> {
+    pub fn new(url: &str, embedder: E, config: RedisSemanticConfig) -> Result<Self, Error> {
+        Ok(Self {
+            connections: Arc::new(Connections::open(url, &RedisTopology::Standalone)?),
+            embedder,
+            inner: Arc::new(Inner::new(config)),
+        })
+    }
+}
+
+impl<E: Embedder, C: redis::ConnectionLike + Send + 'static> RedisSemanticCache<E, C> {
+    pub fn with_connection(connection: C, embedder: E, config: RedisSemanticConfig) -> Self {
+        Self {
+            connections: Arc::new(Connections::fixed(connection)),
+            embedder,
+            inner: Arc::new(Inner::new(config)),
+        }
+    }
+
+    pub fn with_clock(self, clock: fn() -> f64) -> Self {
+        Self {
+            inner: Arc::new(Inner {
+                index_name: self.inner.index_name.clone(),
+                distance_threshold: self.inner.distance_threshold,
+                resolved_index: OnceLock::new(),
+                codec: self.inner.codec,
+                clock,
+            }),
+            ..self
+        }
+    }
+
+    pub fn embedder(&self) -> &E {
+        &self.embedder
+    }
+
+    pub fn index_name(&self) -> &str {
+        &self.inner.index_name
+    }
+
+    pub fn similarity_threshold(&self) -> f32 {
+        (1.0 - self.inner.distance_threshold) as f32
+    }
+
+    fn tag<'a>(key: &'a str, context: &'a SemanticCacheContext) -> &'a str {
+        context.scope.as_deref().unwrap_or(key)
+    }
+}
+
+impl<E: Embedder, C: redis::ConnectionLike + Send + 'static> BaseCache
+    for RedisSemanticCache<E, C>
+{
+    type Value = CacheEntry;
+    type Context = SemanticCacheContext;
+
+    fn get_ttl(&self, context: &Self::Context) -> Option<Duration> {
+        context.ttl
+    }
+
+    fn set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: &Self::Context,
+    ) -> Result<(), Error> {
+        let Some(prompt) = prompt_from_context(context) else {
+            return Ok(());
+        };
+        let vector = self.embedder.embed(&prompt, context.metadata.as_ref())?;
+        let tag = Self::tag(key, context).to_string();
+        self.connections.execute(|connection| {
+            self.inner
+                .store(connection, &tag, &value, &prompt, &vector, context.ttl)
+        })
+    }
+
+    fn get_cache(&self, key: &str, context: &Self::Context) -> Result<Option<Self::Value>, Error> {
+        let Some(prompt) = prompt_from_context(context) else {
+            return Ok(None);
+        };
+        let vector = self.embedder.embed(&prompt, context.metadata.as_ref())?;
+        let tag = Self::tag(key, context).to_string();
+        self.connections
+            .execute(|connection| self.inner.lookup(connection, &tag, &vector))
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: Self::Context,
+    ) -> Result<(), Error> {
+        let Some(prompt) = prompt_from_context(&context) else {
+            return Ok(());
+        };
+        let vector = self
+            .embedder
+            .async_embed(&prompt, context.metadata.as_ref())
+            .await?;
+        let tag = Self::tag(key, &context).to_string();
+        let inner = Arc::clone(&self.inner);
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            inner.store(connection, &tag, &value, &prompt, &vector, context.ttl)
+        })
+        .await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        context: &Self::Context,
+    ) -> Result<Option<Self::Value>, Error> {
+        let Some(prompt) = prompt_from_context(context) else {
+            return Ok(None);
+        };
+        let vector = self
+            .embedder
+            .async_embed(&prompt, context.metadata.as_ref())
+            .await?;
+        let tag = Self::tag(key, context).to_string();
+        let inner = Arc::clone(&self.inner);
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            inner.lookup(connection, &tag, &vector)
+        })
+        .await
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        match Connections::run_blocking(Arc::clone(&self.connections), |connection| {
+            Ok(match redis::cmd("PING").query::<String>(connection) {
+                Ok(_) => CacheConnectionResult {
+                    status: CacheConnectionStatus::Success,
+                    message: "Redis cache connection test successful".into(),
+                    error: None,
+                },
+                Err(error) => CacheConnectionResult {
+                    status: CacheConnectionStatus::Failed,
+                    message: format!("Redis connection failed: {error}"),
+                    error: Some(error.to_string()),
+                },
+            })
+        })
+        .await
+        {
+            Ok(result) => Ok(result),
+            Err(error) => Ok(CacheConnectionResult {
+                status: CacheConnectionStatus::Failed,
+                message: format!("Redis connection failed: {error}"),
+                error: Some(error.to_string()),
+            }),
+        }
+    }
+}
+
+fn timestamp() -> f64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|duration| duration.as_secs_f64())
+        .unwrap_or_default()
+}
+
+fn entry_id(prompt: &str, tag: &str) -> String {
+    let mut digest = Sha256::new();
+    digest.update(prompt.as_bytes());
+    digest.update(CACHE_KEY_FIELD.as_bytes());
+    digest.update(tag.as_bytes());
+    format!("{:x}", digest.finalize())
+}
+
+fn vector_buffer(vector: &[f32]) -> Vec<u8> {
+    vector
+        .iter()
+        .flat_map(|component| component.to_le_bytes())
+        .collect()
+}
+
+fn escape_tag(value: &str) -> String {
+    value
+        .chars()
+        .flat_map(|ch| {
+            if matches!(
+                ch,
+                ',' | '.'
+                    | '<'
+                    | '>'
+                    | '{'
+                    | '}'
+                    | '['
+                    | ']'
+                    | '\\'
+                    | '"'
+                    | '\''
+                    | ':'
+                    | ';'
+                    | '!'
+                    | '@'
+                    | '#'
+                    | '$'
+                    | '%'
+                    | '^'
+                    | '&'
+                    | '*'
+                    | '('
+                    | ')'
+                    | '-'
+                    | '+'
+                    | '='
+                    | '~'
+                    | '|'
+                    | '/'
+                    | ' '
+                    | '?'
+            ) {
+                vec!['\\', ch]
+            } else {
+                vec![ch]
+            }
+        })
+        .collect()
+}
+
+fn create_index(connection: &mut ConnectionRef<'_>, name: &str, dims: usize) -> Result<(), Error> {
+    redis::cmd("FT.CREATE")
+        .arg(name)
+        .arg("ON")
+        .arg("HASH")
+        .arg("PREFIX")
+        .arg(1)
+        .arg(name)
+        .arg("SCORE")
+        .arg(1.0)
+        .arg("SCHEMA")
+        .arg("prompt")
+        .arg("TEXT")
+        .arg("WEIGHT")
+        .arg(1)
+        .arg("response")
+        .arg("TEXT")
+        .arg("WEIGHT")
+        .arg(1)
+        .arg("inserted_at")
+        .arg("NUMERIC")
+        .arg("updated_at")
+        .arg("NUMERIC")
+        .arg(VECTOR_FIELD)
+        .arg("VECTOR")
+        .arg("FLAT")
+        .arg(6)
+        .arg("TYPE")
+        .arg("FLOAT32")
+        .arg("DIM")
+        .arg(dims)
+        .arg("DISTANCE_METRIC")
+        .arg("COSINE")
+        .arg(CACHE_KEY_FIELD)
+        .arg("TAG")
+        .arg("SEPARATOR")
+        .arg(",")
+        .query::<()>(connection)
+        .map_err(|_| Error::Unavailable)
+}
+
+fn index_compatible(
+    connection: &mut ConnectionRef<'_>,
+    name: &str,
+    dims: usize,
+) -> Result<Option<bool>, Error> {
+    let info = match redis::cmd("FT.INFO")
+        .arg(name)
+        .query::<redis::Value>(connection)
+    {
+        Ok(info) => info,
+        Err(error) if unknown_index(&error) => return Ok(None),
+        Err(_) => return Err(Error::Unavailable),
+    };
+    Ok(Some(schema_compatible(&info, dims)))
+}
+
+fn unknown_index(error: &redis::RedisError) -> bool {
+    let message = error.to_string().to_lowercase();
+    message.contains("unknown") && message.contains("index")
+}
+
+fn schema_compatible(info: &redis::Value, dims: usize) -> bool {
+    let redis::Value::Array(entries) = info else {
+        return false;
+    };
+    let attributes = entries
+        .as_chunks::<2>()
+        .0
+        .iter()
+        .find(|pair| string_value(&pair[0]).as_deref() == Some("attributes"))
+        .map(|pair| &pair[1]);
+    let Some(redis::Value::Array(attributes)) = attributes else {
+        return false;
+    };
+    let fields = attributes
+        .iter()
+        .map(|attribute| {
+            let redis::Value::Array(attribute) = attribute else {
+                return (None, None, None, None, None);
+            };
+            let mut name = None;
+            let mut field_type = None;
+            let mut dim = None;
+            let mut data_type = None;
+            let mut distance_metric = None;
+            for pair in attribute.as_chunks::<2>().0 {
+                match string_value(&pair[0]).as_deref() {
+                    Some("identifier") => name = string_value(&pair[1]),
+                    Some("type") => field_type = string_value(&pair[1]),
+                    Some("dim") => dim = number_value(&pair[1]),
+                    Some("data_type") => data_type = string_value(&pair[1]),
+                    Some("distance_metric") => distance_metric = string_value(&pair[1]),
+                    _ => {}
+                }
+            }
+            (name, field_type, dim, data_type, distance_metric)
+        })
+        .collect::<Vec<_>>();
+    let has_field = |name: &str, field_type: &str| {
+        fields
+            .iter()
+            .any(|(n, t, ..)| n.as_deref() == Some(name) && t.as_deref() == Some(field_type))
+    };
+    has_field("prompt", "TEXT")
+        && has_field("response", "TEXT")
+        && has_field("inserted_at", "NUMERIC")
+        && has_field("updated_at", "NUMERIC")
+        && has_field(CACHE_KEY_FIELD, "TAG")
+        && fields.iter().any(|(n, t, d, data, metric)| {
+            n.as_deref() == Some(VECTOR_FIELD)
+                && t.as_deref() == Some("VECTOR")
+                && *d == Some(dims as f64)
+                && data
+                    .as_deref()
+                    .is_some_and(|data| data.eq_ignore_ascii_case("float32"))
+                && metric
+                    .as_deref()
+                    .is_some_and(|metric| metric.eq_ignore_ascii_case("cosine"))
+        })
+}
+
+fn string_value(value: &redis::Value) -> Option<String> {
+    match value {
+        redis::Value::BulkString(bytes) => String::from_utf8(bytes.clone()).ok(),
+        redis::Value::SimpleString(text) => Some(text.clone()),
+        redis::Value::VerbatimString { text, .. } => Some(text.clone()),
+        _ => None,
+    }
+}
+
+fn number_value(value: &redis::Value) -> Option<f64> {
+    match value {
+        redis::Value::Int(number) => Some(*number as f64),
+        redis::Value::Double(number) => Some(*number),
+        _ => string_value(value).and_then(|text| text.parse().ok()),
+    }
+}
+
+fn first_document(result: &redis::Value) -> Option<&[redis::Value]> {
+    let redis::Value::Array(items) = result else {
+        return None;
+    };
+    let [count, _document_id, fields, ..] = items.as_slice() else {
+        return None;
+    };
+    if !matches!(count, redis::Value::Int(count) if *count > 0) {
+        return None;
+    }
+    match fields {
+        redis::Value::Array(fields) => Some(fields.as_slice()),
+        _ => None,
+    }
+}
+
+fn field_value<'a>(fields: &'a [redis::Value], name: &str) -> Option<&'a redis::Value> {
+    fields
+        .as_chunks::<2>()
+        .0
+        .iter()
+        .find(|pair| string_value(&pair[0]).as_deref() == Some(name))
+        .map(|pair| &pair[1])
+}
+
+fn string_field(fields: &[redis::Value], name: &str) -> Option<String> {
+    field_value(fields, name).and_then(string_value)
+}
+
+fn number_field(fields: &[redis::Value], name: &str) -> Option<f64> {
+    field_value(fields, name).and_then(number_value)
+}
+
+fn bytes_field(fields: &[redis::Value], name: &str) -> Option<Vec<u8>> {
+    match field_value(fields, name)? {
+        redis::Value::BulkString(bytes) => Some(bytes.clone()),
+        redis::Value::SimpleString(text) => Some(text.clone().into_bytes()),
+        _ => None,
+    }
+}
+
+fn ttl_seconds(ttl: Duration) -> u64 {
+    ttl.as_secs()
+        .saturating_add(u64::from(ttl.subsec_nanos() > 0))
+        .max(1)
+}

litellm-rust/crates/cache-redis-semantic/src/lib.rs

@@ -0,0 +1,5 @@
+mod cache;
+mod prompt;
+
+pub use cache::{Embedder, RedisSemanticCache, RedisSemanticConfig};
+pub use prompt::prompt_from_context;

litellm-rust/crates/cache-redis-semantic/src/prompt.rs

@@ -0,0 +1,97 @@
+use litellm_cache::SemanticCacheContext;
+use serde_json::Value;
+
+pub fn prompt_from_context(context: &SemanticCacheContext) -> Option<String> {
+    if let Some(messages) = context.messages.as_ref().and_then(Value::as_array)
+        && !messages.is_empty()
+    {
+        return Some(messages_text(messages));
+    }
+    let input = context.input.as_ref()?;
+    let mut parts = Vec::new();
+    collect_input_text(input, &mut parts);
+    let prompt = parts.join("\n").trim().to_string();
+    (!prompt.is_empty()).then_some(prompt)
+}
+
+fn messages_text(messages: &[Value]) -> String {
+    let mut text = String::new();
+    for message in messages {
+        let Some(message) = message.as_object() else {
+            continue;
+        };
+        match message.get("content") {
+            Some(Value::String(content)) => text.push_str(content),
+            Some(Value::Array(parts)) => {
+                for part in parts {
+                    if let Some(text_content) = part.get("text").and_then(Value::as_str) {
+                        text.push_str(text_content);
+                    }
+                }
+            }
+            _ => {}
+        }
+        text.push_str(&search_results_text(message.get("search_results")));
+    }
+    text
+}
+
+fn search_results_text(search_results: Option<&Value>) -> String {
+    let Some(Value::Array(results)) = search_results else {
+        return String::new();
+    };
+    let mut text = String::new();
+    for result in results {
+        let Some(result) = result.as_object() else {
+            continue;
+        };
+        for key in ["source", "title"] {
+            if let Some(value) = result.get(key).and_then(Value::as_str) {
+                text.push_str(value);
+            }
+        }
+        if let Some(Value::Array(content)) = result.get("content") {
+            for block in content {
+                if let Some(value) = block.get("text").and_then(Value::as_str) {
+                    text.push_str(value);
+                }
+            }
+        }
+        if let Some(citations) = result.get("citations") {
+            text.push_str(&citations.to_string());
+        }
+    }
+    text
+}
+
+fn collect_input_text(value: &Value, parts: &mut Vec<String>) {
+    match value {
+        Value::String(text) => {
+            let trimmed = text.trim();
+            if !trimmed.is_empty() {
+                parts.push(trimmed.to_string());
+            }
+        }
+        Value::Array(items) => {
+            for item in items {
+                collect_input_text(item, parts);
+            }
+        }
+        Value::Object(map) => {
+            if let Some(content) = map.get("content").filter(|content| !content.is_null()) {
+                collect_input_text(content, parts);
+                return;
+            }
+            for key in ["text", "output", "input_text", "output_text"] {
+                if let Some(Value::String(text)) = map.get(key) {
+                    let trimmed = text.trim();
+                    if !trimmed.is_empty() {
+                        parts.push(trimmed.to_string());
+                        return;
+                    }
+                }
+            }
+        }
+        _ => {}
+    }
+}

litellm-rust/crates/cache-redis/Cargo.toml

@@ -7,9 +7,10 @@ repository.workspace = true
 
 [dependencies]
 litellm-cache.workspace = true
-redis = "1.7.0"
-serde_json.workspace = true
+redis = { version = "1.7.0", features = ["cluster", "tls-rustls"] }
+r2d2 = "0.8.10"
 tokio.workspace = true
 
 [dev-dependencies]
 redis-test = "1.0.4"
+serde_json.workspace = true

litellm-rust/crates/cache-redis/src/cache.rs

@@ -1,58 +1,220 @@
-use std::sync::{Arc, Mutex, MutexGuard};
-use std::time::Duration;
+use std::{
+    sync::{Arc, Mutex},
+    time::Duration,
+};
 
 use litellm_cache::{
-    BaseCache, CacheConnectionResult, CacheConnectionStatus, CacheEntry, CacheFuture, CacheKwargs,
-    Error,
+    BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, CacheConnectionStatus,
+    ClaimCache, CounterCache, DeleteCache, Error, ExactCacheContext, FlushCache,
 };
 use redis::Commands;
 
+use crate::topology::RedisTopology;
+
+mod connection;
+mod operations;
+
+pub use connection::ConnectionRef;
+use connection::{ClusterConnectionManager, ConnectionManager};
+
+pub use operations::{
+    RedisArg, RedisLpopOperation, RedisLpopResult, RedisRpushOperation, RedisScript,
+};
+
 const DEFAULT_TTL: Duration = Duration::from_secs(600);
-const KEY_PREFIX: &str = "litellm-cache:";
+const REDIS_TIMEOUT: Duration = Duration::from_secs(5);
+const REDIS_POOL_SIZE: u32 = 16;
 
-pub struct RedisCache<C = redis::Connection> {
-    connection: Arc<Mutex<C>>,
-    default_ttl: Duration,
+const INCREMENT_SCRIPT: &str = concat!(
+    "local value = redis.call('INCRBYFLOAT', KEYS[1], ARGV[1]); ",
+    "if redis.call('TTL', KEYS[1]) == -1 then ",
+    "redis.call('EXPIRE', KEYS[1], ARGV[2]); end; return value"
+);
+
+const CLAIM_SCRIPT: &str = concat!(
+    "local current = redis.call('GET', KEYS[1]); ",
+    "if ARGV[1] == '' then if current ~= false and current ~= '' then return 0; end; ",
+    "elseif current ~= ARGV[1] then return 0; end; ",
+    "if ARGV[3] ~= '' then redis.call('SET', KEYS[1], ARGV[3], 'EX', ARGV[2]); ",
+    "elseif ARGV[4] == '1' then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; return 1"
+);
+const CLAIM_ATTEMPTS: usize = 8;
+
+#[allow(private_interfaces)]
+pub enum Connections<C> {
+    Pool(r2d2::Pool<ConnectionManager>),
+    Cluster(r2d2::Pool<ClusterConnectionManager>),
+    Fixed(Mutex<C>),
 }
 
-impl RedisCache<redis::Connection> {
-    pub fn new(url: &str, default_ttl: Option<Duration>) -> Result<Self, Error> {
-        let client = redis::Client::open(url).map_err(|_| Error::Unavailable)?;
-        let connection = client.get_connection().map_err(|_| Error::Unavailable)?;
-        Ok(Self::with_connection(connection, default_ttl))
-    }
-}
-
-impl<C> RedisCache<C>
+impl<C> Connections<C>
 where
     C: redis::ConnectionLike + Send + 'static,
 {
-    fn with_connection(connection: C, default_ttl: Option<Duration>) -> Self {
-        Self {
-            connection: Arc::new(Mutex::new(connection)),
-            default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
+    pub fn execute<T>(
+        &self,
+        operation: impl FnOnce(&mut ConnectionRef<'_>) -> Result<T, Error>,
+    ) -> Result<T, Error> {
+        match self {
+            Self::Pool(pool) => {
+                let mut pooled = pool.get().map_err(|_| Error::Unavailable)?;
+                let result = operation(&mut ConnectionRef::Node(&mut pooled.connection));
+                pooled.failed = matches!(result, Err(Error::Unavailable));
+                result
+            }
+            Self::Cluster(pool) => {
+                let mut pooled = pool.get().map_err(|_| Error::Unavailable)?;
+                let result = operation(&mut ConnectionRef::Cluster(&mut pooled.connection));
+                pooled.failed = matches!(result, Err(Error::Unavailable));
+                result
+            }
+            Self::Fixed(connection) => {
+                let mut connection = connection.lock().map_err(|_| Error::Unavailable)?;
+                operation(&mut ConnectionRef::Node(&mut *connection))
+            }
         }
     }
 
-    fn connection(&self) -> Result<MutexGuard<'_, C>, Error> {
-        self.connection.lock().map_err(|_| Error::Unavailable)
+    pub async fn run_blocking<T, F>(connections: Arc<Self>, operation: F) -> Result<T, Error>
+    where
+        T: Send + 'static,
+        F: FnOnce(&mut ConnectionRef<'_>) -> Result<T, Error> + Send + 'static,
+    {
+        tokio::task::spawn_blocking(move || connections.execute(operation))
+            .await
+            .map_err(|_| Error::Unavailable)?
     }
 
-    fn namespaced_key(key: &str) -> String {
-        format!("{KEY_PREFIX}{key}")
+    pub fn fixed(connection: C) -> Self {
+        Self::Fixed(Mutex::new(connection))
     }
 
-    fn namespaced_pattern() -> &'static str {
-        const PATTERN: &str = "litellm-cache:*";
-        PATTERN
+    pub fn open(url: &str, topology: &RedisTopology) -> Result<Self, Error> {
+        match topology {
+            RedisTopology::Standalone => Ok(Self::Pool(pool(ConnectionManager::open(url)?)?)),
+            RedisTopology::Cluster { startup_nodes } => Ok(Self::Cluster(pool(
+                ClusterConnectionManager::open(url, startup_nodes)?,
+            )?)),
+        }
+    }
+}
+
+pub struct RedisCache<S, C = redis::Connection> {
+    connections: Arc<Connections<C>>,
+    default_ttl: Duration,
+    codec: S,
+    namespace: Option<String>,
+    topology: RedisTopology,
+}
+
+impl<S: CacheCodec> RedisCache<S> {
+    pub fn new(url: &str, default_ttl: Option<Duration>, codec: S) -> Result<Self, Error> {
+        Self::connect(url, &RedisTopology::Standalone, default_ttl, codec)
     }
 
-    fn encode(value: &CacheEntry) -> Result<Vec<u8>, Error> {
-        serde_json::to_vec(value).map_err(|_| Error::InvalidEntry)
+    pub fn connect(
+        url: &str,
+        topology: &RedisTopology,
+        default_ttl: Option<Duration>,
+        codec: S,
+    ) -> Result<Self, Error> {
+        let connections = Connections::open(url, topology)?;
+        Ok(Self {
+            connections: Arc::new(connections),
+            default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
+            codec,
+            namespace: None,
+            topology: topology.clone(),
+        })
+    }
+}
+
+fn pool<M: r2d2::ManageConnection>(manager: M) -> Result<r2d2::Pool<M>, Error> {
+    r2d2::Pool::builder()
+        .max_size(REDIS_POOL_SIZE)
+        .min_idle(Some(0))
+        .connection_timeout(REDIS_TIMEOUT)
+        .test_on_check_out(false)
+        .build(manager)
+        .map_err(|_| Error::Unavailable)
+}
+
+impl<S, C> RedisCache<S, C>
+where
+    S: CacheCodec,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    pub fn with_connection(connection: C, default_ttl: Option<Duration>, codec: S) -> Self {
+        Self {
+            connections: Arc::new(Connections::fixed(connection)),
+            default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
+            codec,
+            namespace: None,
+            topology: RedisTopology::Standalone,
+        }
     }
 
-    fn decode(value: Vec<u8>) -> Result<CacheEntry, Error> {
-        serde_json::from_slice(&value).map_err(|_| Error::InvalidEntry)
+    pub fn with_namespace(self, namespace: Option<String>) -> Self {
+        Self {
+            namespace: namespace.filter(|value| !value.is_empty()),
+            ..self
+        }
+    }
+
+    pub fn namespace(&self) -> Option<&str> {
+        self.namespace.as_deref()
+    }
+
+    pub fn topology(&self) -> &RedisTopology {
+        &self.topology
+    }
+
+    fn namespaced_key(&self, key: &str) -> String {
+        namespaced_key(self.namespace.as_deref(), key)
+    }
+
+    fn namespaced_pattern(&self) -> Result<String, Error> {
+        let namespace = self.namespace.as_ref().ok_or(Error::UnscopedFlush)?;
+        let escaped: String = namespace
+            .chars()
+            .flat_map(|ch| {
+                if matches!(ch, '*' | '?' | '[' | ']' | '\\') {
+                    vec!['\\', ch]
+                } else {
+                    vec![ch]
+                }
+            })
+            .collect();
+        Ok(format!("{escaped}:*"))
+    }
+
+    fn flush_matching(connection: &mut ConnectionRef<'_>, pattern: &str) -> Result<(), Error> {
+        connection.scan(pattern, 1000, |connection, keys| {
+            if !keys.is_empty() {
+                connection
+                    .del::<_, usize>(keys)
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            Ok(true)
+        })
+    }
+
+    fn decode_response(&self, value: redis::Value) -> Result<Option<S::Value>, Error> {
+        match value {
+            redis::Value::Nil => Ok(None),
+            redis::Value::BulkString(bytes) => self.codec.decode(&bytes).map(Some),
+            redis::Value::SimpleString(text) => self.codec.decode(text.as_bytes()).map(Some),
+            _ => Err(Error::InvalidEntry),
+        }
+    }
+
+    fn decode_batch_response(&self, value: redis::Value) -> Result<BatchEntry<S::Value>, Error> {
+        match self.decode_response(value) {
+            Ok(Some(value)) => Ok(BatchEntry::Hit(value)),
+            Ok(None) => Ok(BatchEntry::Miss),
+            Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid),
+            Err(error) => Err(error),
+        }
     }
 
     fn ttl_seconds(ttl: Duration) -> u64 {
@@ -60,197 +222,409 @@ where
             .saturating_add(u64::from(ttl.subsec_nanos() > 0))
             .max(1)
     }
+}
 
-    fn run_blocking<T, F>(connection: Arc<Mutex<C>>, operation: F) -> CacheFuture<'static, T>
-    where
-        T: Send + 'static,
-        F: FnOnce(&mut C) -> Result<T, Error> + Send + 'static,
-    {
-        Box::pin(async move {
-            tokio::task::spawn_blocking(move || {
-                let mut connection = connection.lock().map_err(|_| Error::Unavailable)?;
-                operation(&mut connection)
-            })
-            .await
-            .map_err(|_| Error::Unavailable)?
-        })
+fn namespaced_key(namespace: Option<&str>, key: &str) -> String {
+    match namespace {
+        Some(namespace) if !key.starts_with(&format!("{namespace}:")) => {
+            format!("{namespace}:{key}")
+        }
+        _ => key.into(),
     }
 }
 
-impl<C> BaseCache for RedisCache<C>
+impl<S, C> BaseCache for RedisCache<S, C>
 where
+    S: CacheCodec,
     C: redis::ConnectionLike + Send + 'static,
 {
-    type Value = CacheEntry;
+    type Value = S::Value;
+    type Context = ExactCacheContext;
 
-    fn default_ttl(&self) -> Duration {
-        self.default_ttl
+    fn get_ttl(&self, context: &Self::Context) -> Option<Duration> {
+        context.ttl.or(Some(self.default_ttl))
     }
 
-    fn set_cache(&self, key: &str, value: Self::Value, kwargs: CacheKwargs) -> Result<(), Error> {
-        let payload = Self::encode(&value)?;
-        let ttl = Self::ttl_seconds(self.get_ttl(&kwargs));
-        self.connection()?
-            .set_ex::<_, _, ()>(Self::namespaced_key(key), payload, ttl)
-            .map_err(|_| Error::Unavailable)
-    }
-
-    fn get_cache(&self, key: &str, _: &CacheKwargs) -> Result<Option<Self::Value>, Error> {
-        self.connection()?
-            .get::<_, Option<Vec<u8>>>(Self::namespaced_key(key))
-            .map_err(|_| Error::Unavailable)?
-            .map(Self::decode)
-            .transpose()
-    }
-
-    fn delete_cache(&self, key: &str) -> Result<(), Error> {
-        self.connection()?
-            .del::<_, ()>(Self::namespaced_key(key))
-            .map_err(|_| Error::Unavailable)
-    }
-
-    fn flush_cache(&self) -> Result<(), Error> {
-        let mut connection = self.connection()?;
-        let keys = connection
-            .scan_match(Self::namespaced_pattern())
-            .map_err(|_| Error::Unavailable)?
-            .collect::<redis::RedisResult<Vec<String>>>()
-            .map_err(|_| Error::Unavailable)?;
-        if keys.is_empty() {
-            return Ok(());
-        }
-        connection
-            .del::<_, usize>(keys)
-            .map(|_| ())
-            .map_err(|_| Error::Unavailable)
-    }
-
-    fn async_set_cache<'a>(
-        &'a self,
-        key: &'a str,
+    fn set_cache(
+        &self,
+        key: &str,
         value: Self::Value,
-        kwargs: CacheKwargs,
-    ) -> CacheFuture<'a, ()> {
-        let payload = Self::encode(&value);
-        let key = Self::namespaced_key(key);
-        let ttl = Self::ttl_seconds(self.get_ttl(&kwargs));
-        Self::run_blocking(Arc::clone(&self.connection), move |connection| {
+        context: &ExactCacheContext,
+    ) -> Result<(), Error> {
+        let payload = self.codec.encode(&value)?;
+        let ttl = Self::ttl_seconds(self.get_ttl(context).unwrap_or(self.default_ttl));
+        let key = self.namespaced_key(key);
+        self.connections.execute(|connection| {
             connection
-                .set_ex::<_, _, ()>(key, payload?, ttl)
+                .set_ex::<_, _, ()>(key, payload, ttl)
                 .map_err(|_| Error::Unavailable)
         })
     }
 
-    fn async_get_cache<'a>(
-        &'a self,
-        key: &'a str,
-        _: &'a CacheKwargs,
-    ) -> CacheFuture<'a, Option<Self::Value>> {
-        let key = Self::namespaced_key(key);
-        Box::pin(async move {
-            Self::run_blocking(Arc::clone(&self.connection), move |connection| {
-                connection
-                    .get::<_, Option<Vec<u8>>>(key)
-                    .map_err(|_| Error::Unavailable)
-            })
-            .await?
-            .map(Self::decode)
-            .transpose()
-        })
+    fn get_cache(&self, key: &str, _: &ExactCacheContext) -> Result<Option<Self::Value>, Error> {
+        let key = self.namespaced_key(key);
+        let value = self.connections.execute(|connection| {
+            connection
+                .get::<_, redis::Value>(key)
+                .map_err(|_| Error::Unavailable)
+        })?;
+        self.decode_response(value)
     }
 
-    fn async_set_cache_pipeline<'a>(
-        &'a self,
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: ExactCacheContext,
+    ) -> Result<(), Error> {
+        let payload = self.codec.encode(&value)?;
+        let key = self.namespaced_key(key);
+        let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            connection
+                .set_ex::<_, _, ()>(key, payload, ttl)
+                .map_err(|_| Error::Unavailable)
+        })
+        .await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        _: &ExactCacheContext,
+    ) -> Result<Option<Self::Value>, Error> {
+        let key = self.namespaced_key(key);
+        let value = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            connection
+                .get::<_, redis::Value>(key)
+                .map_err(|_| Error::Unavailable)
+        })
+        .await?;
+        self.decode_response(value)
+    }
+
+    async fn async_set_cache_pipeline(
+        &self,
         cache_list: Vec<(String, Self::Value)>,
-        kwargs: CacheKwargs,
-    ) -> CacheFuture<'a, ()> {
+        context: ExactCacheContext,
+    ) -> Result<(), Error> {
         let entries = cache_list
             .into_iter()
             .map(|(key, value)| {
-                Self::encode(&value).map(|payload| (Self::namespaced_key(&key), payload))
+                self.codec
+                    .encode(&value)
+                    .map(|payload| (self.namespaced_key(&key), payload))
             })
-            .collect::<Result<Vec<_>, _>>();
-        let ttl = Self::ttl_seconds(self.get_ttl(&kwargs));
-        Self::run_blocking(Arc::clone(&self.connection), move |connection| {
-            for (key, payload) in entries? {
-                connection
-                    .set_ex::<_, _, ()>(key, payload, ttl)
-                    .map_err(|_| Error::Unavailable)?;
-            }
-            Ok(())
+            .collect::<Result<Vec<_>, _>>()?;
+        let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
+        if entries.is_empty() {
+            return Ok(());
+        }
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            let commands = entries
+                .into_iter()
+                .map(|(key, payload)| {
+                    let mut command = redis::cmd("SETEX");
+                    command.arg(key).arg(ttl).arg(payload);
+                    command
+                })
+                .collect();
+            connection.pipeline(commands).map(drop)
         })
+        .await
     }
 
-    fn async_delete_cache<'a>(&'a self, key: &'a str) -> CacheFuture<'a, ()> {
-        let key = Self::namespaced_key(key);
-        Self::run_blocking(Arc::clone(&self.connection), move |connection| {
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        match Connections::run_blocking(Arc::clone(&self.connections), |connection| {
+            Ok(match connection.ping() {
+                Ok(_) => CacheConnectionResult {
+                    status: CacheConnectionStatus::Success,
+                    message: "Redis cache connection test successful".into(),
+                    error: None,
+                },
+                Err(error) => CacheConnectionResult {
+                    status: CacheConnectionStatus::Failed,
+                    message: format!("Redis connection failed: {error}"),
+                    error: Some(error.to_string()),
+                },
+            })
+        })
+        .await
+        {
+            Ok(result) => Ok(result),
+            Err(error) => Ok(CacheConnectionResult {
+                status: CacheConnectionStatus::Failed,
+                message: format!("Redis connection failed: {error}"),
+                error: Some(error.to_string()),
+            }),
+        }
+    }
+}
+
+impl<S, C> BatchCache for RedisCache<S, C>
+where
+    S: CacheCodec,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    fn batch_get_cache(
+        &self,
+        keys: &[String],
+        _: &ExactCacheContext,
+    ) -> Result<Vec<BatchEntry<Self::Value>>, Error> {
+        let keys = keys
+            .iter()
+            .map(|key| self.namespaced_key(key))
+            .collect::<Vec<_>>();
+        let values = self.connections.execute(|connection| {
+            redis::cmd("MGET")
+                .arg(keys)
+                .query::<Vec<redis::Value>>(connection)
+                .map_err(|_| Error::Unavailable)
+        })?;
+        values
+            .into_iter()
+            .map(|value| self.decode_batch_response(value))
+            .collect()
+    }
+
+    async fn async_batch_get_cache(
+        &self,
+        keys: Vec<String>,
+        _: ExactCacheContext,
+    ) -> Result<Vec<BatchEntry<Self::Value>>, Error> {
+        let keys = keys
+            .iter()
+            .map(|key| self.namespaced_key(key))
+            .collect::<Vec<_>>();
+        let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            redis::cmd("MGET")
+                .arg(keys)
+                .query::<Vec<redis::Value>>(connection)
+                .map_err(|_| Error::Unavailable)
+        })
+        .await?;
+        values
+            .into_iter()
+            .map(|value| self.decode_batch_response(value))
+            .collect()
+    }
+}
+
+impl<S, C> DeleteCache for RedisCache<S, C>
+where
+    S: CacheCodec,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    fn delete_cache(&self, key: &str) -> Result<(), Error> {
+        let key = self.namespaced_key(key);
+        self.connections
+            .execute(|connection| connection.del::<_, ()>(key).map_err(|_| Error::Unavailable))
+    }
+
+    async fn async_delete_cache(&self, key: &str) -> Result<(), Error> {
+        let key = self.namespaced_key(key);
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
             connection.del::<_, ()>(key).map_err(|_| Error::Unavailable)
         })
+        .await
+    }
+}
+
+impl<S, C> FlushCache for RedisCache<S, C>
+where
+    S: CacheCodec,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    fn flush_cache(&self) -> Result<(), Error> {
+        let pattern = self.namespaced_pattern()?;
+        self.connections
+            .execute(|connection| Self::flush_matching(connection, &pattern))
     }
 
-    fn disconnect(&self) -> CacheFuture<'_, ()> {
-        Box::pin(async { Ok(()) })
-    }
-
-    fn test_connection(&self) -> CacheFuture<'_, CacheConnectionResult> {
-        Box::pin(async move {
-            Self::run_blocking(Arc::clone(&self.connection), |connection| {
-                redis::cmd("PING")
-                    .query::<String>(connection)
-                    .map_err(|_| Error::Unavailable)
-            })
-            .await?;
-            Ok(CacheConnectionResult {
-                status: CacheConnectionStatus::Success,
-                message: "Redis cache connection test successful".into(),
-                error: None,
-            })
+    async fn async_flush_cache(&self) -> Result<(), Error> {
+        let pattern = self.namespaced_pattern()?;
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            Self::flush_matching(connection, &pattern)
         })
+        .await
+    }
+}
+
+impl<S, C> CounterCache for RedisCache<S, C>
+where
+    S: CacheCodec<Value = f64>,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    fn increment_cache(
+        &self,
+        key: &str,
+        amount: f64,
+        context: ExactCacheContext,
+    ) -> Result<f64, Error> {
+        let key = self.namespaced_key(key);
+        let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
+        self.connections
+            .execute(|connection| increment(connection, key, amount, ttl))
+    }
+
+    async fn async_increment(
+        &self,
+        key: &str,
+        amount: f64,
+        context: ExactCacheContext,
+    ) -> Result<f64, Error> {
+        let key = self.namespaced_key(key);
+        let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            increment(connection, key, amount, ttl)
+        })
+        .await
+    }
+}
+
+fn increment(
+    connection: &mut ConnectionRef<'_>,
+    key: String,
+    amount: f64,
+    ttl: u64,
+) -> Result<f64, Error> {
+    redis::cmd("EVAL")
+        .arg(INCREMENT_SCRIPT)
+        .arg(1)
+        .arg(key)
+        .arg(amount)
+        .arg(ttl)
+        .query(connection)
+        .map_err(|_| Error::Unavailable)
+}
+
+fn stored_bytes(value: redis::Value) -> Result<Option<Vec<u8>>, Error> {
+    match value {
+        redis::Value::Nil => Ok(None),
+        redis::Value::BulkString(bytes) => Ok(Some(bytes)),
+        redis::Value::SimpleString(text) => Ok(Some(text.into_bytes())),
+        _ => Err(Error::InvalidEntry),
+    }
+}
+
+/// Eligibility is decided on decoded values, so a pin written by another encoder (Python's
+/// `json.dumps` spacing or key order) still matches. The write is a compare-and-set on the
+/// bytes that decision was made on, retried when another claimant wins the race.
+fn claim<S: CacheCodec>(
+    connection: &mut ConnectionRef<'_>,
+    codec: &S,
+    key: &str,
+    candidate: S::Value,
+    eligible: &[S::Value],
+    ttl: u64,
+) -> Result<S::Value, Error>
+where
+    S::Value: PartialEq,
+{
+    let payload = codec.encode(&candidate)?;
+    if payload.is_empty() {
+        return Err(Error::InvalidEntry);
+    }
+    for _ in 0..CLAIM_ATTEMPTS {
+        let current = stored_bytes(
+            connection
+                .get::<_, redis::Value>(key)
+                .map_err(|_| Error::Unavailable)?,
+        )?
+        .filter(|bytes| !bytes.is_empty());
+        let existing = current
+            .as_deref()
+            .and_then(|bytes| codec.decode(bytes).ok())
+            .filter(|existing| eligible.is_empty() || eligible.contains(existing));
+        let refresh = existing
+            .as_ref()
+            .is_some_and(|existing| !eligible.is_empty() || *existing == candidate);
+        let write: &[u8] = if existing.is_some() { b"" } else { &payload };
+        let applied = redis::cmd("EVAL")
+            .arg(CLAIM_SCRIPT)
+            .arg(1)
+            .arg(key)
+            .arg(current.as_deref().unwrap_or_default())
+            .arg(ttl)
+            .arg(write)
+            .arg(u8::from(refresh))
+            .query::<bool>(connection)
+            .map_err(|_| Error::Unavailable)?;
+        if applied {
+            return Ok(existing.unwrap_or(candidate));
+        }
+    }
+    Err(Error::Unavailable)
+}
+
+impl<S, C> ClaimCache for RedisCache<S, C>
+where
+    S: CacheCodec + Clone + 'static,
+    S::Value: PartialEq,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    fn claim_cache(
+        &self,
+        key: &str,
+        candidate: S::Value,
+        eligible: &[S::Value],
+        context: ExactCacheContext,
+    ) -> Result<S::Value, Error> {
+        let key = self.namespaced_key(key);
+        let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
+        self.connections
+            .execute(|connection| claim(connection, &self.codec, &key, candidate, eligible, ttl))
+    }
+
+    async fn async_claim_cache(
+        &self,
+        key: &str,
+        candidate: S::Value,
+        eligible: Vec<S::Value>,
+        context: ExactCacheContext,
+    ) -> Result<S::Value, Error> {
+        let key = self.namespaced_key(key);
+        let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
+        let codec = self.codec.clone();
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            claim(connection, &codec, &key, candidate, &eligible, ttl)
+        })
+        .await
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use super::RedisCache;
-    use litellm_cache::{BaseCache, CacheEntry, CacheKwargs};
-    use redis_test::{MockCmd, MockRedisConnection};
-    use serde_json::json;
     use std::time::Duration;
 
-    fn entry() -> CacheEntry {
-        CacheEntry {
-            timestamp: 123.0,
-            response: json!({"choices": [{"text": "cached"}]}),
-        }
-    }
+    use litellm_cache::{
+        BaseCache, CacheCodec, DeleteCache, ExactCacheContext, FlushCache, JsonCodec,
+    };
+    use redis_test::{MockCmd, MockRedisConnection};
+    use serde_json::json;
 
-    #[test]
-    fn cache_entries_round_trip_through_json() {
-        let entry = entry();
-        let encoded = RedisCache::<redis::Connection>::encode(&entry).unwrap();
-        assert_eq!(
-            RedisCache::<redis::Connection>::decode(encoded).unwrap(),
-            entry
-        );
-    }
+    use super::RedisCache;
 
-    #[test]
-    fn invalid_json_is_rejected() {
-        assert!(RedisCache::<redis::Connection>::decode(b"not json".to_vec()).is_err());
+    fn entry() -> serde_json::Value {
+        json!({"deployment": "model-a", "cooldown_seconds": 30})
     }
 
     #[test]
     fn ttl_seconds_rounds_up_and_keeps_expiration_positive() {
         assert_eq!(
-            RedisCache::<redis::Connection>::ttl_seconds(Duration::ZERO),
+            RedisCache::<JsonCodec<serde_json::Value>>::ttl_seconds(Duration::ZERO),
             1
         );
         assert_eq!(
-            RedisCache::<redis::Connection>::ttl_seconds(Duration::from_millis(1500)),
+            RedisCache::<JsonCodec<serde_json::Value>>::ttl_seconds(Duration::from_millis(1500)),
             2
         );
         assert_eq!(
-            RedisCache::<redis::Connection>::ttl_seconds(Duration::from_secs(15)),
+            RedisCache::<JsonCodec<serde_json::Value>>::ttl_seconds(Duration::from_secs(15)),
             15
         );
     }
@@ -258,7 +632,9 @@ mod tests {
     #[test]
     fn redis_commands_round_trip_entries_and_delete_only_namespaced_keys() {
         let value = entry();
-        let payload = RedisCache::<redis::Connection>::encode(&value).unwrap();
+        let payload = JsonCodec::<serde_json::Value>::new()
+            .encode(&value)
+            .unwrap();
         let connection = MockRedisConnection::new([
             MockCmd::new(
                 redis::cmd("SETEX")
@@ -271,13 +647,17 @@ mod tests {
             MockCmd::new(redis::cmd("DEL").arg("litellm-cache:key"), Ok(1u32)),
         ])
         .assert_all_commands_consumed();
-        let cache = RedisCache::with_connection(connection, None);
+        let cache =
+            RedisCache::with_connection(connection, None, JsonCodec::<serde_json::Value>::new())
+                .with_namespace(Some("litellm-cache".into()));
 
         cache
-            .set_cache("key", value.clone(), CacheKwargs::default())
+            .set_cache("key", value.clone(), &ExactCacheContext::default())
             .unwrap();
         assert_eq!(
-            cache.get_cache("key", &CacheKwargs::default()).unwrap(),
+            cache
+                .get_cache("key", &ExactCacheContext::default())
+                .unwrap(),
             Some(value)
         );
         cache.delete_cache("key").unwrap();
@@ -290,13 +670,17 @@ mod tests {
                 redis::cmd("SCAN")
                     .cursor_arg(0)
                     .arg("MATCH")
-                    .arg("litellm-cache:*"),
+                    .arg("litellm-cache:*")
+                    .arg("COUNT")
+                    .arg(1000),
                 Ok(redis_test::redis_value!(["0", ["litellm-cache:key"]])),
             ),
             MockCmd::new(redis::cmd("DEL").arg("litellm-cache:key"), Ok(1u32)),
         ])
         .assert_all_commands_consumed();
-        let cache = RedisCache::with_connection(connection, None);
+        let cache =
+            RedisCache::with_connection(connection, None, JsonCodec::<serde_json::Value>::new())
+                .with_namespace(Some("litellm-cache".into()));
 
         cache.flush_cache().unwrap();
     }
@@ -305,7 +689,9 @@ mod tests {
     async fn test_connection_runs_ping_off_executor() {
         let connection = MockRedisConnection::new([MockCmd::new(redis::cmd("PING"), Ok("PONG"))])
             .assert_all_commands_consumed();
-        let cache = RedisCache::with_connection(connection, None);
+        let cache =
+            RedisCache::with_connection(connection, None, JsonCodec::<serde_json::Value>::new())
+                .with_namespace(Some("litellm-cache".into()));
 
         assert_eq!(
             cache.test_connection().await.unwrap().status,

litellm-rust/crates/cache-redis/src/cache/connection.rs

@@ -0,0 +1,392 @@
+use std::collections::HashMap;
+
+use litellm_cache::Error;
+use redis::{
+    ConnectionAddr, ConnectionInfo, ConnectionLike, IntoConnectionInfo,
+    cluster::{ClusterClient, ClusterClientBuilder, ClusterConnection, NodeAddress},
+    cluster_routing::{
+        MultipleNodeRoutingInfo, ResponsePolicy, RoutingInfo, SingleNodeRoutingInfo, Slot,
+    },
+};
+
+use super::REDIS_TIMEOUT;
+use crate::topology::RedisNode;
+
+pub struct PooledConnection<C> {
+    pub(super) connection: C,
+    pub(super) failed: bool,
+}
+
+/// Pools connections without a checkout PING, which would double every operation's round trips.
+/// A timed-out command leaves its reply on the socket while redis still reports the connection
+/// open, so any connection whose operation failed is discarded instead of being reused.
+pub struct ConnectionManager(redis::Client);
+
+impl ConnectionManager {
+    pub(super) fn open(url: &str) -> Result<Self, Error> {
+        redis::Client::open(url)
+            .map(Self)
+            .map_err(|_| Error::Unavailable)
+    }
+}
+
+impl r2d2::ManageConnection for ConnectionManager {
+    type Connection = PooledConnection<redis::Connection>;
+    type Error = redis::RedisError;
+
+    fn connect(&self) -> Result<Self::Connection, redis::RedisError> {
+        let connection = self.0.get_connection()?;
+        connection.set_read_timeout(Some(REDIS_TIMEOUT))?;
+        connection.set_write_timeout(Some(REDIS_TIMEOUT))?;
+        Ok(PooledConnection {
+            connection,
+            failed: false,
+        })
+    }
+
+    fn is_valid(&self, connection: &mut Self::Connection) -> Result<(), redis::RedisError> {
+        redis::cmd("PING").query::<String>(&mut connection.connection)?;
+        Ok(())
+    }
+
+    fn has_broken(&self, connection: &mut Self::Connection) -> bool {
+        connection.failed || !redis::ConnectionLike::is_open(&connection.connection)
+    }
+}
+
+pub struct ClusterConnectionManager(ClusterClient);
+
+impl ClusterConnectionManager {
+    pub(super) fn open(url: &str, startup_nodes: &[RedisNode]) -> Result<Self, Error> {
+        if startup_nodes.is_empty() {
+            return Err(Error::Unavailable);
+        }
+        let info = url.into_connection_info().map_err(|_| Error::Unavailable)?;
+        let nodes = startup_nodes
+            .iter()
+            .map(|node| node_info(&info, node))
+            .collect::<Result<Vec<_>, _>>()?;
+        ClusterClientBuilder::new(nodes)
+            .connection_timeout(REDIS_TIMEOUT)
+            .response_timeout(REDIS_TIMEOUT)
+            .build()
+            .map(Self)
+            .map_err(|_| Error::Unavailable)
+    }
+}
+
+fn node_info(info: &ConnectionInfo, node: &RedisNode) -> Result<ConnectionInfo, Error> {
+    let addr = match info.addr() {
+        ConnectionAddr::Tcp(..) => ConnectionAddr::Tcp(node.host.clone(), node.port),
+        ConnectionAddr::TcpTls {
+            insecure,
+            tls_params,
+            ..
+        } => ConnectionAddr::TcpTls {
+            host: node.host.clone(),
+            port: node.port,
+            insecure: *insecure,
+            tls_params: tls_params.clone(),
+        },
+        _ => return Err(Error::Unavailable),
+    };
+    Ok(info.clone().set_addr(addr))
+}
+
+impl r2d2::ManageConnection for ClusterConnectionManager {
+    type Connection = PooledConnection<ClusterConnection>;
+    type Error = redis::RedisError;
+
+    fn connect(&self) -> Result<Self::Connection, redis::RedisError> {
+        let connection = self.0.get_connection()?;
+        connection.set_read_timeout(Some(REDIS_TIMEOUT))?;
+        connection.set_write_timeout(Some(REDIS_TIMEOUT))?;
+        Ok(PooledConnection {
+            connection,
+            failed: false,
+        })
+    }
+
+    fn is_valid(&self, connection: &mut Self::Connection) -> Result<(), redis::RedisError> {
+        redis::cmd("PING").query::<String>(&mut connection.connection)?;
+        Ok(())
+    }
+
+    fn has_broken(&self, connection: &mut Self::Connection) -> bool {
+        connection.failed || !redis::ConnectionLike::is_open(&connection.connection)
+    }
+}
+
+pub enum ConnectionRef<'a> {
+    Node(&'a mut dyn redis::ConnectionLike),
+    Cluster(&'a mut ClusterConnection),
+}
+
+impl redis::ConnectionLike for ConnectionRef<'_> {
+    fn req_packed_command(&mut self, cmd: &[u8]) -> redis::RedisResult<redis::Value> {
+        match self {
+            Self::Node(connection) => connection.req_packed_command(cmd),
+            Self::Cluster(connection) => connection.req_packed_command(cmd),
+        }
+    }
+
+    fn req_packed_commands(
+        &mut self,
+        cmd: &[u8],
+        offset: usize,
+        count: usize,
+    ) -> redis::RedisResult<Vec<redis::Value>> {
+        match self {
+            Self::Node(connection) => connection.req_packed_commands(cmd, offset, count),
+            Self::Cluster(connection) => connection.req_packed_commands(cmd, offset, count),
+        }
+    }
+
+    fn get_db(&self) -> i64 {
+        match self {
+            Self::Node(connection) => connection.get_db(),
+            Self::Cluster(connection) => redis::ConnectionLike::get_db(*connection),
+        }
+    }
+
+    fn supports_pipelining(&self) -> bool {
+        match self {
+            Self::Node(connection) => connection.supports_pipelining(),
+            Self::Cluster(connection) => redis::ConnectionLike::supports_pipelining(*connection),
+        }
+    }
+
+    fn check_connection(&mut self) -> bool {
+        match self {
+            Self::Node(connection) => connection.check_connection(),
+            Self::Cluster(connection) => connection.check_connection(),
+        }
+    }
+
+    fn is_open(&self) -> bool {
+        match self {
+            Self::Node(connection) => connection.is_open(),
+            Self::Cluster(connection) => redis::ConnectionLike::is_open(*connection),
+        }
+    }
+}
+
+impl ConnectionRef<'_> {
+    pub(crate) fn pipeline(
+        &mut self,
+        commands: Vec<redis::Cmd>,
+    ) -> Result<Vec<redis::Value>, Error> {
+        match self {
+            Self::Node(connection) => {
+                let mut pipeline = redis::pipe();
+                for command in &commands {
+                    pipeline.add_command(command.clone());
+                }
+                pipeline
+                    .query::<Vec<redis::Value>>(*connection)
+                    .map_err(|_| Error::Unavailable)
+            }
+            Self::Cluster(connection) => {
+                let mut replies: Vec<Option<redis::Value>> = vec![None; commands.len()];
+                for indices in slot_groups(&commands).into_values() {
+                    let mut pipeline = redis::pipe();
+                    for index in &indices {
+                        pipeline.add_command(commands[*index].clone());
+                    }
+                    let values = connection
+                        .req_packed_commands(&pipeline.get_packed_pipeline(), 0, indices.len())
+                        .map_err(|_| Error::Unavailable)?;
+                    if values.len() != indices.len() {
+                        return Err(Error::Unavailable);
+                    }
+                    for (index, value) in indices.into_iter().zip(values) {
+                        replies[index] = Some(value);
+                    }
+                }
+                replies
+                    .into_iter()
+                    .collect::<Option<Vec<_>>>()
+                    .ok_or(Error::Unavailable)
+            }
+        }
+    }
+
+    pub(crate) fn scan(
+        &mut self,
+        pattern: &str,
+        count: usize,
+        mut visit: impl FnMut(&mut Self, Vec<String>) -> Result<bool, Error>,
+    ) -> Result<(), Error> {
+        let pages = match self {
+            Self::Node(connection) => {
+                let page = scan_command(0, pattern, count)
+                    .query::<ScanPage>(*connection)
+                    .map_err(|_| Error::Unavailable)?;
+                vec![(None, page)]
+            }
+            Self::Cluster(connection) => connection
+                .route_command(
+                    &scan_command(0, pattern, count),
+                    RoutingInfo::MultiNode((
+                        MultipleNodeRoutingInfo::AllMasters,
+                        Some(ResponsePolicy::Special),
+                    )),
+                )
+                .map_err(|_| Error::Unavailable)
+                .and_then(primary_pages)?
+                .into_iter()
+                .map(|(node, page)| (Some(node), page))
+                .collect(),
+        };
+        for (node, (mut cursor, mut keys)) in pages {
+            loop {
+                if !visit(self, keys)? {
+                    return Ok(());
+                }
+                if cursor == 0 {
+                    break;
+                }
+                (cursor, keys) = self.scan_page(node.as_ref(), cursor, pattern, count)?;
+            }
+        }
+        Ok(())
+    }
+
+    pub(crate) fn ping(&mut self) -> Result<bool, redis::RedisError> {
+        let command = redis::cmd("PING");
+        match self {
+            Self::Node(connection) => command
+                .query::<String>(*connection)
+                .map(|response| response == "PONG"),
+            Self::Cluster(connection) => connection
+                .route_command(
+                    &command,
+                    RoutingInfo::MultiNode((
+                        MultipleNodeRoutingInfo::AllNodes,
+                        Some(ResponsePolicy::AllSucceeded),
+                    )),
+                )
+                .map(|_| true),
+        }
+    }
+
+    pub(crate) fn node_text(&mut self, command: &redis::Cmd) -> Result<String, Error> {
+        match self {
+            Self::Node(connection) => command.query(*connection).map_err(|_| Error::Unavailable),
+            Self::Cluster(connection) => {
+                let value = connection
+                    .route_command(
+                        command,
+                        RoutingInfo::MultiNode((
+                            MultipleNodeRoutingInfo::AllNodes,
+                            Some(ResponsePolicy::Special),
+                        )),
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+                let redis::Value::Map(entries) = value else {
+                    return Err(Error::Unavailable);
+                };
+                let mut replies = entries
+                    .into_iter()
+                    .map(|(node, reply)| {
+                        Ok((
+                            redis::from_redis_value::<String>(node)
+                                .map_err(|_| Error::Unavailable)?,
+                            redis::from_redis_value::<String>(reply)
+                                .map_err(|_| Error::Unavailable)?,
+                        ))
+                    })
+                    .collect::<Result<Vec<(String, String)>, Error>>()?;
+                replies.sort();
+                Ok(replies
+                    .into_iter()
+                    .map(|(_, reply)| reply)
+                    .collect::<Vec<_>>()
+                    .join("\n"))
+            }
+        }
+    }
+
+    pub(crate) fn flushall(&mut self) -> Result<(), Error> {
+        let command = redis::cmd("FLUSHALL");
+        match self {
+            Self::Node(connection) => command.query(*connection).map_err(|_| Error::Unavailable),
+            Self::Cluster(connection) => connection
+                .route_command(
+                    &command,
+                    RoutingInfo::MultiNode((
+                        MultipleNodeRoutingInfo::AllMasters,
+                        Some(ResponsePolicy::AllSucceeded),
+                    )),
+                )
+                .map(|_| ())
+                .map_err(|_| Error::Unavailable),
+        }
+    }
+
+    fn scan_page(
+        &mut self,
+        node: Option<&NodeAddress>,
+        cursor: u64,
+        pattern: &str,
+        count: usize,
+    ) -> Result<ScanPage, Error> {
+        let command = scan_command(cursor, pattern, count);
+        match (self, node) {
+            (Self::Node(connection), None) => {
+                command.query(*connection).map_err(|_| Error::Unavailable)
+            }
+            (Self::Cluster(connection), Some(node)) => connection
+                .route_command(
+                    &command,
+                    RoutingInfo::SingleNode(SingleNodeRoutingInfo::ByAddress {
+                        host: node.host().to_string(),
+                        port: node.port(),
+                    }),
+                )
+                .map_err(|_| Error::Unavailable)
+                .and_then(|value| redis::from_redis_value(value).map_err(|_| Error::Unavailable)),
+            _ => Err(Error::Unavailable),
+        }
+    }
+}
+
+type ScanPage = (u64, Vec<String>);
+
+fn primary_pages(value: redis::Value) -> Result<Vec<(NodeAddress, ScanPage)>, Error> {
+    let redis::Value::Map(entries) = value else {
+        return Err(Error::Unavailable);
+    };
+    entries
+        .into_iter()
+        .map(|(node, page)| {
+            let node = redis::from_redis_value::<String>(node).map_err(|_| Error::Unavailable)?;
+            let node = NodeAddress::try_from(node.as_str()).map_err(|_| Error::Unavailable)?;
+            let page = redis::from_redis_value::<ScanPage>(page).map_err(|_| Error::Unavailable)?;
+            Ok((node, page))
+        })
+        .collect()
+}
+
+fn scan_command(cursor: u64, pattern: &str, count: usize) -> redis::Cmd {
+    let mut command = redis::cmd("SCAN");
+    command
+        .cursor_arg(cursor)
+        .arg("MATCH")
+        .arg(pattern)
+        .arg("COUNT")
+        .arg(count);
+    command
+}
+
+fn slot_groups(commands: &[redis::Cmd]) -> HashMap<Slot, Vec<usize>> {
+    let mut groups: HashMap<Slot, Vec<usize>> = HashMap::new();
+    for (index, command) in commands.iter().enumerate() {
+        let key = match command.args_iter().nth(1) {
+            Some(redis::Arg::Simple(key)) => key,
+            _ => b"",
+        };
+        groups.entry(Slot::for_key(key)).or_default().push(index);
+    }
+    groups
+}

litellm-rust/crates/cache-redis/src/cache/operations.rs

@@ -0,0 +1,632 @@
+use std::{sync::Arc, time::Duration};
+
+use litellm_cache::{
+    CacheCodec, CacheScript, ClientInfoCache, Error, IncrementOperation, QueueCache, ScanCache,
+    ScriptCache, SetCache, TtlCache,
+};
+use redis::Commands;
+
+use super::{ConnectionRef, Connections, RedisCache, namespaced_key};
+
+const INCREMENT_WITH_FLOOR_SCRIPT: &str = concat!(
+    "local count = redis.call('INCRBY', KEYS[1], ARGV[1]); ",
+    "if count < 0 then count = redis.call('INCRBY', KEYS[1], -count); end; ",
+    "if redis.call('TTL', KEYS[1]) < 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; ",
+    "return count"
+);
+const SET_MAX_SCRIPT: &str = concat!(
+    "local current = redis.call('GET', KEYS[1]); ",
+    "if current == false or tonumber(current) < tonumber(ARGV[1]) then ",
+    "redis.call('SET', KEYS[1], ARGV[1]); ",
+    "if tonumber(ARGV[2]) > 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; ",
+    "return ARGV[1]; end; return current"
+);
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum RedisArg {
+    Bytes(Vec<u8>),
+    Integer(i64),
+    Float(f64),
+}
+
+impl From<&str> for RedisArg {
+    fn from(value: &str) -> Self {
+        Self::Bytes(value.as_bytes().to_vec())
+    }
+}
+
+impl From<String> for RedisArg {
+    fn from(value: String) -> Self {
+        Self::Bytes(value.into_bytes())
+    }
+}
+
+impl From<Vec<u8>> for RedisArg {
+    fn from(value: Vec<u8>) -> Self {
+        Self::Bytes(value)
+    }
+}
+
+impl From<i64> for RedisArg {
+    fn from(value: i64) -> Self {
+        Self::Integer(value)
+    }
+}
+
+impl From<f64> for RedisArg {
+    fn from(value: f64) -> Self {
+        Self::Float(value)
+    }
+}
+
+impl redis::ToRedisArgs for RedisArg {
+    fn write_redis_args<W>(&self, out: &mut W)
+    where
+        W: ?Sized + redis::RedisWrite,
+    {
+        match self {
+            Self::Bytes(value) => value.write_redis_args(out),
+            Self::Integer(value) => value.write_redis_args(out),
+            Self::Float(value) => value.write_redis_args(out),
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct RedisRpushOperation {
+    pub key: String,
+    pub values: Vec<RedisArg>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct RedisLpopOperation {
+    pub key: String,
+    pub count: Option<usize>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum RedisLpopResult {
+    Missing,
+    Value(Vec<u8>),
+    Values(Vec<Vec<u8>>),
+}
+
+pub struct RedisScript<C> {
+    connections: Arc<Connections<C>>,
+    namespace: Option<String>,
+    source: String,
+}
+
+impl<C> CacheScript for RedisScript<C>
+where
+    C: redis::ConnectionLike + Send + 'static,
+{
+    type Argument = RedisArg;
+    type Output = redis::Value;
+
+    async fn invoke(
+        &self,
+        keys: Vec<String>,
+        arguments: Vec<Self::Argument>,
+    ) -> Result<Self::Output, Error> {
+        let keys = keys
+            .into_iter()
+            .map(|key| namespaced_key(self.namespace.as_deref(), &key))
+            .collect::<Vec<_>>();
+        let connections = Arc::clone(&self.connections);
+        let source = self.source.clone();
+        tokio::task::spawn_blocking(move || {
+            connections.execute(|connection| {
+                redis::cmd("EVAL")
+                    .arg(source)
+                    .arg(keys.len())
+                    .arg(keys)
+                    .arg(arguments)
+                    .query(connection)
+                    .map_err(|_| Error::Unavailable)
+            })
+        })
+        .await
+        .map_err(|_| Error::Unavailable)?
+    }
+}
+
+impl<S, C> RedisCache<S, C>
+where
+    S: CacheCodec,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    pub async fn delete_cache_keys(&self, keys: Vec<String>) -> Result<usize, Error> {
+        if keys.is_empty() {
+            return Ok(0);
+        }
+        let keys = keys
+            .into_iter()
+            .map(|key| self.namespaced_key(&key))
+            .collect::<Vec<_>>();
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            connection.del(keys).map_err(|_| Error::Unavailable)
+        })
+        .await
+    }
+
+    pub fn batch_get_counts(&self, keys: &[String]) -> Result<Vec<Option<i64>>, Error> {
+        let keys = keys
+            .iter()
+            .map(|key| self.namespaced_key(key))
+            .collect::<Vec<_>>();
+        let values = self.connections.execute(|connection| {
+            redis::cmd("MGET")
+                .arg(keys)
+                .query::<Vec<redis::Value>>(connection)
+                .map_err(|_| Error::Unavailable)
+        })?;
+        values.into_iter().map(count).collect()
+    }
+
+    pub async fn async_batch_get_counts(
+        &self,
+        keys: Vec<String>,
+    ) -> Result<Vec<Option<i64>>, Error> {
+        let keys = keys
+            .iter()
+            .map(|key| self.namespaced_key(key))
+            .collect::<Vec<_>>();
+        let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            redis::cmd("MGET")
+                .arg(keys)
+                .query::<Vec<redis::Value>>(connection)
+                .map_err(|_| Error::Unavailable)
+        })
+        .await?;
+        values.into_iter().map(count).collect()
+    }
+
+    pub fn sync_ping(&self) -> Result<bool, Error> {
+        self.connections
+            .execute(|connection| connection.ping().map_err(|_| Error::Unavailable))
+    }
+
+    pub async fn ping(&self) -> Result<bool, Error> {
+        Connections::run_blocking(Arc::clone(&self.connections), |connection| {
+            connection.ping().map_err(|_| Error::Unavailable)
+        })
+        .await
+    }
+
+    pub async fn async_get_ttl(&self, key: &str) -> Result<Option<i64>, Error> {
+        let key = self.namespaced_key(key);
+        let ttl = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            redis::cmd("TTL")
+                .arg(key)
+                .query::<i64>(connection)
+                .map_err(|_| Error::Unavailable)
+        })
+        .await?;
+        Ok((ttl >= 0).then_some(ttl))
+    }
+
+    pub async fn async_scan_iter(&self, pattern: &str, count: usize) -> Result<Vec<String>, Error> {
+        let pattern = format!("{}*", self.namespaced_key(pattern));
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            let mut matches = Vec::new();
+            connection.scan(&pattern, count, |_, keys| {
+                matches.extend(keys);
+                Ok(matches.len() < count)
+            })?;
+            matches.truncate(count);
+            Ok(matches)
+        })
+        .await
+    }
+
+    pub async fn async_set_cache_sadd(
+        &self,
+        key: &str,
+        values: Vec<RedisArg>,
+        ttl: Option<Duration>,
+    ) -> Result<usize, Error> {
+        if values.is_empty() {
+            return Err(Error::InvalidEntry);
+        }
+        let key = self.namespaced_key(key);
+        let ttl = Self::ttl_seconds(ttl.unwrap_or(self.default_ttl));
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            let mut sadd = redis::cmd("SADD");
+            sadd.arg(&key).arg(values);
+            let mut expire = redis::cmd("EXPIRE");
+            expire.arg(&key).arg(ttl);
+            let replies = connection.pipeline(vec![sadd, expire])?;
+            replies
+                .into_iter()
+                .next()
+                .map(redis::from_redis_value::<usize>)
+                .transpose()
+                .map_err(|_| Error::Unavailable)?
+                .ok_or(Error::Unavailable)
+        })
+        .await
+    }
+
+    pub async fn async_rpush(&self, key: &str, values: Vec<RedisArg>) -> Result<usize, Error> {
+        if values.is_empty() {
+            return Err(Error::InvalidEntry);
+        }
+        let key = self.namespaced_key(key);
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            redis::cmd("RPUSH")
+                .arg(key)
+                .arg(values)
+                .query(connection)
+                .map_err(|_| Error::Unavailable)
+        })
+        .await
+    }
+
+    pub async fn async_rpush_pipeline(
+        &self,
+        operations: Vec<RedisRpushOperation>,
+    ) -> Result<Vec<usize>, Error> {
+        let operations = operations
+            .into_iter()
+            .map(|operation| {
+                if operation.values.is_empty() {
+                    return Err(Error::InvalidEntry);
+                }
+                Ok((self.namespaced_key(&operation.key), operation.values))
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+        if operations.is_empty() {
+            return Ok(Vec::new());
+        }
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            let commands = operations
+                .into_iter()
+                .map(|(key, values)| {
+                    let mut command = redis::cmd("RPUSH");
+                    command.arg(key).arg(values);
+                    command
+                })
+                .collect();
+            connection
+                .pipeline(commands)?
+                .into_iter()
+                .map(|value| redis::from_redis_value(value).map_err(|_| Error::Unavailable))
+                .collect()
+        })
+        .await
+    }
+
+    pub async fn async_lpop(
+        &self,
+        key: &str,
+        count: Option<usize>,
+    ) -> Result<RedisLpopResult, Error> {
+        let key = self.namespaced_key(key);
+        let multiple = count.is_some();
+        let value = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            let mut command = redis::cmd("LPOP");
+            command.arg(key);
+            if let Some(count) = count {
+                command.arg(count);
+            }
+            command
+                .query::<redis::Value>(connection)
+                .map_err(|_| Error::Unavailable)
+        })
+        .await?;
+        lpop_result(value, multiple)
+    }
+
+    pub async fn async_lpop_pipeline(
+        &self,
+        operations: Vec<RedisLpopOperation>,
+    ) -> Result<Vec<RedisLpopResult>, Error> {
+        let operations = operations
+            .into_iter()
+            .map(|operation| (self.namespaced_key(&operation.key), operation.count))
+            .collect::<Vec<_>>();
+        if operations.is_empty() {
+            return Ok(Vec::new());
+        }
+        let multiple = operations
+            .iter()
+            .map(|(_, count)| count.is_some())
+            .collect::<Vec<_>>();
+        let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            let commands = operations
+                .into_iter()
+                .map(|(key, count)| {
+                    let mut command = redis::cmd("LPOP");
+                    command.arg(key);
+                    if let Some(count) = count {
+                        command.arg(count);
+                    }
+                    command
+                })
+                .collect();
+            connection.pipeline(commands)
+        })
+        .await?;
+        values
+            .into_iter()
+            .zip(multiple)
+            .map(|(value, multiple)| lpop_result(value, multiple))
+            .collect()
+    }
+
+    pub async fn async_eval(
+        &self,
+        script: String,
+        keys: Vec<String>,
+        arguments: Vec<RedisArg>,
+    ) -> Result<redis::Value, Error> {
+        let keys = keys
+            .into_iter()
+            .map(|key| self.namespaced_key(&key))
+            .collect::<Vec<_>>();
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            redis::cmd("EVAL")
+                .arg(script)
+                .arg(keys.len())
+                .arg(keys)
+                .arg(arguments)
+                .query(connection)
+                .map_err(|_| Error::Unavailable)
+        })
+        .await
+    }
+
+    pub fn client_list(&self) -> Result<String, Error> {
+        self.connections
+            .execute(|connection| connection.node_text(redis::cmd("CLIENT").arg("LIST")))
+    }
+
+    pub fn info(&self) -> Result<String, Error> {
+        self.connections
+            .execute(|connection| connection.node_text(&redis::cmd("INFO")))
+    }
+
+    pub fn flushall(&self) -> Result<(), Error> {
+        self.connections.execute(|connection| connection.flushall())
+    }
+}
+
+impl<S, C> RedisCache<S, C>
+where
+    S: CacheCodec<Value = f64>,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    pub fn increment_with_floor(
+        &self,
+        key: &str,
+        amount: i64,
+        ttl: Duration,
+    ) -> Result<i64, Error> {
+        let key = self.namespaced_key(key);
+        let ttl = Self::ttl_seconds(ttl);
+        self.connections
+            .execute(|connection| increment_with_floor(connection, key, amount, ttl))
+    }
+
+    pub async fn async_increment_pipeline(
+        &self,
+        operations: Vec<IncrementOperation>,
+    ) -> Result<Vec<f64>, Error> {
+        let operations = operations
+            .into_iter()
+            .map(|operation| {
+                (
+                    self.namespaced_key(&operation.key),
+                    operation.amount,
+                    operation.ttl.map(Self::ttl_seconds),
+                )
+            })
+            .collect::<Vec<_>>();
+        if operations.is_empty() {
+            return Ok(Vec::new());
+        }
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            let mut commands = Vec::with_capacity(operations.len() * 2);
+            let mut increments = Vec::with_capacity(operations.len());
+            for (key, amount, ttl) in operations {
+                let mut increment = redis::cmd("INCRBYFLOAT");
+                increment.arg(&key).arg(amount);
+                increments.push(commands.len());
+                commands.push(increment);
+                if let Some(ttl) = ttl {
+                    let mut expire = redis::cmd("EXPIRE");
+                    expire.arg(key).arg(ttl);
+                    commands.push(expire);
+                }
+            }
+            let mut replies = connection.pipeline(commands)?;
+            increments
+                .into_iter()
+                .map(|index| {
+                    redis::from_redis_value(std::mem::take(&mut replies[index]))
+                        .map_err(|_| Error::Unavailable)
+                })
+                .collect()
+        })
+        .await
+    }
+
+    pub async fn async_increment_with_floor(
+        &self,
+        key: &str,
+        amount: i64,
+        ttl: Duration,
+    ) -> Result<i64, Error> {
+        let key = self.namespaced_key(key);
+        let ttl = Self::ttl_seconds(ttl);
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            increment_with_floor(connection, key, amount, ttl)
+        })
+        .await
+    }
+
+    pub async fn async_set_max(
+        &self,
+        key: &str,
+        value: f64,
+        ttl: Option<Duration>,
+    ) -> Result<f64, Error> {
+        let key = self.namespaced_key(key);
+        let ttl = Self::ttl_seconds(ttl.unwrap_or(self.default_ttl));
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            redis::cmd("EVAL")
+                .arg(SET_MAX_SCRIPT)
+                .arg(1)
+                .arg(key)
+                .arg(value)
+                .arg(ttl)
+                .query(connection)
+                .map_err(|_| Error::Unavailable)
+        })
+        .await
+    }
+}
+
+fn redis_bytes(value: redis::Value) -> Result<Vec<u8>, Error> {
+    match value {
+        redis::Value::BulkString(bytes) => Ok(bytes),
+        redis::Value::SimpleString(text) => Ok(text.into_bytes()),
+        _ => Err(Error::InvalidEntry),
+    }
+}
+
+fn lpop_result(value: redis::Value, multiple: bool) -> Result<RedisLpopResult, Error> {
+    match value {
+        redis::Value::Nil => Ok(RedisLpopResult::Missing),
+        redis::Value::Array(values) if multiple => values
+            .into_iter()
+            .map(redis_bytes)
+            .collect::<Result<Vec<_>, _>>()
+            .map(RedisLpopResult::Values),
+        value if !multiple => redis_bytes(value).map(RedisLpopResult::Value),
+        _ => Err(Error::InvalidEntry),
+    }
+}
+
+fn count(value: redis::Value) -> Result<Option<i64>, Error> {
+    match value {
+        redis::Value::Nil => Ok(None),
+        redis::Value::Int(value) => Ok(Some(value)),
+        redis::Value::BulkString(value) => std::str::from_utf8(&value)
+            .ok()
+            .and_then(|value| value.parse().ok())
+            .map(Some)
+            .ok_or(Error::InvalidEntry),
+        redis::Value::SimpleString(value) => {
+            value.parse().map(Some).map_err(|_| Error::InvalidEntry)
+        }
+        _ => Err(Error::InvalidEntry),
+    }
+}
+
+fn increment_with_floor(
+    connection: &mut ConnectionRef<'_>,
+    key: String,
+    amount: i64,
+    ttl: u64,
+) -> Result<i64, Error> {
+    redis::cmd("EVAL")
+        .arg(INCREMENT_WITH_FLOOR_SCRIPT)
+        .arg(1)
+        .arg(key)
+        .arg(amount)
+        .arg(ttl)
+        .query(connection)
+        .map_err(|_| Error::Unavailable)
+}
+
+impl<S, C> TtlCache for RedisCache<S, C>
+where
+    S: CacheCodec,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    async fn async_get_ttl(&self, key: &str) -> Result<Option<Duration>, Error> {
+        RedisCache::async_get_ttl(self, key)
+            .await
+            .map(|ttl| ttl.map(|seconds| Duration::from_secs(seconds as u64)))
+    }
+}
+
+impl<S, C> ScanCache for RedisCache<S, C>
+where
+    S: CacheCodec,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    async fn async_scan_iter(&self, pattern: &str, count: usize) -> Result<Vec<String>, Error> {
+        RedisCache::async_scan_iter(self, pattern, count).await
+    }
+}
+
+impl<S, C> ClientInfoCache for RedisCache<S, C>
+where
+    S: CacheCodec,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    type ClientList = String;
+    type Info = String;
+
+    fn client_list(&self) -> Result<Self::ClientList, Error> {
+        RedisCache::client_list(self)
+    }
+
+    fn info(&self) -> Result<Self::Info, Error> {
+        RedisCache::info(self)
+    }
+}
+
+impl<S, C> SetCache for RedisCache<S, C>
+where
+    S: CacheCodec,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    type SetValue = RedisArg;
+    type SetResult = usize;
+
+    async fn async_set_cache_sadd(
+        &self,
+        key: &str,
+        values: Vec<Self::SetValue>,
+        ttl: Option<Duration>,
+    ) -> Result<Self::SetResult, Error> {
+        RedisCache::async_set_cache_sadd(self, key, values, ttl).await
+    }
+}
+
+impl<S, C> QueueCache for RedisCache<S, C>
+where
+    S: CacheCodec,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    type QueueValue = RedisArg;
+    type PopResult = RedisLpopResult;
+
+    async fn async_rpush(&self, key: &str, values: Vec<Self::QueueValue>) -> Result<usize, Error> {
+        RedisCache::async_rpush(self, key, values).await
+    }
+
+    async fn async_lpop(&self, key: &str, count: Option<usize>) -> Result<Self::PopResult, Error> {
+        RedisCache::async_lpop(self, key, count).await
+    }
+}
+
+impl<S, C> ScriptCache for RedisCache<S, C>
+where
+    S: CacheCodec,
+    C: redis::ConnectionLike + Send + 'static,
+{
+    type Script = RedisScript<C>;
+
+    fn async_register_script(&self, source: String) -> Self::Script {
+        RedisScript {
+            connections: Arc::clone(&self.connections),
+            namespace: self.namespace.clone(),
+            source,
+        }
+    }
+}

litellm-rust/crates/cache-redis/src/lib.rs

@@ -1,3 +1,11 @@
 mod cache;
+mod topology;
 
-pub use cache::RedisCache;
+pub mod connection {
+    pub use crate::cache::{ConnectionRef, Connections};
+}
+
+pub use cache::{
+    RedisArg, RedisCache, RedisLpopOperation, RedisLpopResult, RedisRpushOperation, RedisScript,
+};
+pub use topology::{RedisNode, RedisTopology};

litellm-rust/crates/cache-redis/src/topology.rs

@@ -0,0 +1,14 @@
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct RedisNode {
+    pub host: String,
+    pub port: u16,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub enum RedisTopology {
+    #[default]
+    Standalone,
+    Cluster {
+        startup_nodes: Vec<RedisNode>,
+    },
+}

litellm-rust/crates/cache-redis/tests/cache.rs

@@ -1,6 +1,703 @@
-use litellm_cache_redis::RedisCache;
+use std::time::Duration;
+
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionStatus, CacheScript, ClaimCache,
+    CounterCache, DeleteCache, Error, ExactCacheContext, FlushCache, IncrementOperation, JsonCodec,
+    ScriptCache, get_cache, set_cache,
+};
+use litellm_cache_redis::{
+    RedisArg, RedisCache, RedisLpopOperation, RedisLpopResult, RedisRpushOperation,
+};
+use redis_test::{MockCmd, MockRedisConnection};
+
+struct TaggedByteCodec(u8);
+
+impl CacheCodec for TaggedByteCodec {
+    type Value = u8;
+
+    fn encode(&self, value: &u8) -> Result<Vec<u8>, Error> {
+        if *value > 127 {
+            return Err(Error::InvalidEntry);
+        }
+        Ok(vec![self.0, *value])
+    }
+
+    fn decode(&self, bytes: &[u8]) -> Result<u8, Error> {
+        match bytes {
+            [tag, value] if *tag == self.0 => Ok(*value),
+            _ => Err(Error::InvalidEntry),
+        }
+    }
+}
 
 #[test]
 fn constructor_rejects_invalid_urls() {
-    assert!(RedisCache::new("not a redis url", None).is_err());
+    assert!(RedisCache::new("not a redis url", None, JsonCodec::<String>::new()).is_err());
+}
+
+#[test]
+fn generic_helpers_use_the_injected_codec_and_ttl() {
+    let connection = MockRedisConnection::new([
+        MockCmd::new(
+            redis::cmd("SETEX")
+                .arg("counter")
+                .arg(2)
+                .arg([42u8, 7].as_slice()),
+            Ok("OK"),
+        ),
+        MockCmd::new(redis::cmd("GET").arg("counter"), Ok(vec![42u8, 7])),
+    ])
+    .assert_all_commands_consumed();
+    let cache = RedisCache::with_connection(connection, None, TaggedByteCodec(42));
+    let context = ExactCacheContext {
+        ttl: Some(Duration::from_millis(1500)),
+    };
+    set_cache(&cache, "counter", 7, &context).unwrap();
+    assert_eq!(get_cache(&cache, "counter", &context).unwrap(), Some(7));
+}
+
+#[tokio::test]
+async fn async_operations_preserve_codec_ttl_and_missing_values() {
+    let connection = MockRedisConnection::new([
+        MockCmd::new(
+            redis::cmd("SETEX")
+                .arg("counter")
+                .arg(9)
+                .arg([42u8, 7].as_slice()),
+            Ok("OK"),
+        ),
+        MockCmd::new(redis::cmd("GET").arg("counter"), Ok(vec![42u8, 7])),
+        MockCmd::new(
+            redis::cmd("SETEX")
+                .arg("batch")
+                .arg(2)
+                .arg([42u8, 8].as_slice()),
+            Ok("OK"),
+        ),
+        MockCmd::new(redis::cmd("DEL").arg("counter"), Ok(1u32)),
+        MockCmd::new(redis::cmd("GET").arg("counter"), Ok(redis::Value::Nil)),
+    ])
+    .assert_all_commands_consumed();
+    let cache = RedisCache::with_connection(
+        connection,
+        Some(Duration::from_secs(9)),
+        TaggedByteCodec(42),
+    );
+    let context = ExactCacheContext::default();
+    cache
+        .batch_cache_write("counter", 7, context.clone())
+        .await
+        .unwrap();
+    assert_eq!(
+        cache.async_get_cache("counter", &context).await.unwrap(),
+        Some(7)
+    );
+    cache
+        .async_set_cache_pipeline(
+            vec![("batch".into(), 8)],
+            ExactCacheContext {
+                ttl: Some(Duration::from_millis(1500)),
+            },
+        )
+        .await
+        .unwrap();
+    cache.async_delete_cache("counter").await.unwrap();
+    assert_eq!(
+        cache.async_get_cache("counter", &context).await.unwrap(),
+        None
+    );
+}
+
+#[tokio::test]
+async fn codec_errors_propagate_without_writing_partial_batches() {
+    let connection = MockRedisConnection::new([
+        MockCmd::new(redis::cmd("GET").arg("invalid"), Ok(vec![99u8, 7])),
+        MockCmd::new(redis::cmd("GET").arg("invalid"), Ok(vec![99u8, 7])),
+    ])
+    .assert_all_commands_consumed();
+    let cache = RedisCache::with_connection(connection, None, TaggedByteCodec(42));
+    let context = ExactCacheContext::default();
+    assert_eq!(
+        cache.set_cache("invalid", 255, &context),
+        Err(Error::InvalidEntry)
+    );
+    assert_eq!(
+        cache.async_set_cache("invalid", 255, context.clone()).await,
+        Err(Error::InvalidEntry)
+    );
+    assert_eq!(
+        cache
+            .async_set_cache_pipeline(
+                vec![("valid".into(), 7), ("invalid".into(), 255)],
+                context.clone(),
+            )
+            .await,
+        Err(Error::InvalidEntry)
+    );
+    assert_eq!(
+        cache.get_cache("invalid", &context),
+        Err(Error::InvalidEntry)
+    );
+    assert_eq!(
+        cache.async_get_cache("invalid", &context).await,
+        Err(Error::InvalidEntry)
+    );
+}
+
+#[test]
+fn namespaces_are_optional_and_existing_prefixes_are_not_duplicated() {
+    let connection = MockRedisConnection::new([
+        MockCmd::new(redis::cmd("GET").arg("team:key"), Ok(redis::Value::Nil)),
+        MockCmd::new(redis::cmd("GET").arg("team:key"), Ok(redis::Value::Nil)),
+    ])
+    .assert_all_commands_consumed();
+    let cache = RedisCache::with_connection(connection, None, JsonCodec::<String>::new())
+        .with_namespace(Some("team".into()));
+    assert_eq!(
+        cache
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap(),
+        None
+    );
+    assert_eq!(
+        cache
+            .get_cache("team:key", &ExactCacheContext::default())
+            .unwrap(),
+        None
+    );
+}
+
+#[test]
+fn flush_requires_a_namespace_and_escapes_glob_metacharacters() {
+    let unscoped = RedisCache::with_connection(
+        MockRedisConnection::new([]).assert_all_commands_consumed(),
+        None,
+        JsonCodec::<String>::new(),
+    );
+    assert_eq!(unscoped.flush_cache(), Err(Error::UnscopedFlush));
+    let connection = MockRedisConnection::new([
+        MockCmd::new(
+            redis::cmd("SCAN")
+                .cursor_arg(0)
+                .arg("MATCH")
+                .arg("team\\*:*")
+                .arg("COUNT")
+                .arg(1000),
+            Ok(redis_test::redis_value!(["0", ["team*:key"]])),
+        ),
+        MockCmd::new(redis::cmd("DEL").arg("team*:key"), Ok(1u32)),
+    ])
+    .assert_all_commands_consumed();
+    let scoped = RedisCache::with_connection(connection, None, JsonCodec::<String>::new())
+        .with_namespace(Some("team*".into()));
+    scoped.flush_cache().unwrap();
+}
+
+#[tokio::test]
+async fn connection_failures_use_the_python_result_contract() {
+    let error = redis::RedisError::from((redis::ErrorKind::Io, "connection refused"));
+    let connection =
+        MockRedisConnection::new([MockCmd::new(redis::cmd("PING"), Err::<String, _>(error))])
+            .assert_all_commands_consumed();
+    let cache = RedisCache::with_connection(connection, None, JsonCodec::<String>::new());
+
+    let result = cache.test_connection().await.unwrap();
+    assert_eq!(result.status, CacheConnectionStatus::Failed);
+    assert!(result.message.starts_with("Redis connection failed:"));
+    assert!(result.error.is_some());
+}
+
+#[tokio::test]
+async fn batch_reads_keep_order_and_treat_invalid_values_as_invalid_entries() {
+    let connection = MockRedisConnection::new([MockCmd::new(
+        redis::cmd("MGET").arg("hit").arg("miss").arg("invalid"),
+        Ok(vec![
+            redis::Value::BulkString(vec![42, 7]),
+            redis::Value::Nil,
+            redis::Value::BulkString(vec![99, 7]),
+        ]),
+    )])
+    .assert_all_commands_consumed();
+    let cache = RedisCache::with_connection(connection, None, TaggedByteCodec(42));
+
+    assert_eq!(
+        cache
+            .async_batch_get_cache(
+                vec!["hit".into(), "miss".into(), "invalid".into()],
+                ExactCacheContext::default(),
+            )
+            .await
+            .unwrap(),
+        vec![BatchEntry::Hit(7), BatchEntry::Miss, BatchEntry::Invalid]
+    );
+}
+
+#[tokio::test]
+async fn async_flush_deletes_each_scan_page_separately() {
+    let connection = MockRedisConnection::new([
+        MockCmd::new(
+            redis::cmd("SCAN")
+                .cursor_arg(0)
+                .arg("MATCH")
+                .arg("team:*")
+                .arg("COUNT")
+                .arg(1000),
+            Ok(redis_test::redis_value!(["7", ["team:a", "team:b"]])),
+        ),
+        MockCmd::new(redis::cmd("DEL").arg("team:a").arg("team:b"), Ok(2u32)),
+        MockCmd::new(
+            redis::cmd("SCAN")
+                .cursor_arg(7)
+                .arg("MATCH")
+                .arg("team:*")
+                .arg("COUNT")
+                .arg(1000),
+            Ok(redis_test::redis_value!(["0", ["team:c"]])),
+        ),
+        MockCmd::new(redis::cmd("DEL").arg("team:c"), Ok(1u32)),
+    ])
+    .assert_all_commands_consumed();
+    let cache = RedisCache::with_connection(connection, None, JsonCodec::<String>::new())
+        .with_namespace(Some("team".into()));
+
+    cache.async_flush_cache().await.unwrap();
+}
+
+#[tokio::test]
+async fn direct_redis_operations_preserve_namespace_values_and_missing_ttls() {
+    let mut sadd_pipeline = redis::pipe();
+    sadd_pipeline
+        .cmd("SADD")
+        .arg("team:members")
+        .arg("a")
+        .arg("b")
+        .cmd("EXPIRE")
+        .arg("team:members")
+        .arg(600u64)
+        .ignore();
+    let connection = MockRedisConnection::new([
+        MockCmd::new(
+            redis::cmd("MGET").arg("team:count").arg("team:missing"),
+            Ok(redis_test::redis_value!(["7", nil])),
+        ),
+        MockCmd::new(
+            redis::cmd("MGET").arg("team:count").arg("team:missing"),
+            Ok(redis_test::redis_value!(["7", nil])),
+        ),
+        MockCmd::new(redis::cmd("PING"), Ok("PONG")),
+        MockCmd::new(redis::cmd("PING"), Ok("PONG")),
+        MockCmd::new(redis::cmd("TTL").arg("team:missing"), Ok(-2i64)),
+        MockCmd::new(
+            redis::cmd("SCAN")
+                .cursor_arg(0)
+                .arg("MATCH")
+                .arg("team:job-*")
+                .arg("COUNT")
+                .arg(25),
+            Ok(redis_test::redis_value!(["4", ["team:job-a"]])),
+        ),
+        MockCmd::new(
+            redis::cmd("SCAN")
+                .cursor_arg(4)
+                .arg("MATCH")
+                .arg("team:job-*")
+                .arg("COUNT")
+                .arg(25),
+            Ok(redis_test::redis_value!(["0", ["team:job-b"]])),
+        ),
+        MockCmd::new(
+            redis::cmd("DEL").arg("team:job-a").arg("team:job-b"),
+            Ok(2u32),
+        ),
+        MockCmd::with_values(
+            sadd_pipeline,
+            Ok(vec![redis::Value::Int(2), redis::Value::Int(1)]),
+        ),
+        MockCmd::new(
+            redis::cmd("RPUSH").arg("team:queue").arg("a").arg("b"),
+            Ok(2u32),
+        ),
+        MockCmd::new(
+            redis::cmd("LPOP").arg("team:queue").arg(2usize),
+            Ok(redis_test::redis_value!(["a", "b"])),
+        ),
+        MockCmd::new(
+            redis::cmd("EVAL")
+                .arg("return KEYS[1]")
+                .arg(1usize)
+                .arg("team:key"),
+            Ok("team:key"),
+        ),
+        MockCmd::new(
+            redis::cmd("EVAL")
+                .arg("return KEYS[1]")
+                .arg(1usize)
+                .arg("team:key"),
+            Ok("team:key"),
+        ),
+        MockCmd::new(redis::cmd("CLIENT").arg("LIST"), Ok("id=1")),
+        MockCmd::new(redis::cmd("INFO"), Ok("redis_version:7")),
+        MockCmd::new(redis::cmd("FLUSHALL"), Ok("OK")),
+    ])
+    .assert_all_commands_consumed();
+    let cache = RedisCache::with_connection(connection, None, JsonCodec::<String>::new())
+        .with_namespace(Some("team".into()));
+
+    assert_eq!(
+        cache
+            .batch_get_counts(&["count".into(), "missing".into()])
+            .unwrap(),
+        [Some(7), None]
+    );
+    assert_eq!(
+        cache
+            .async_batch_get_counts(vec!["count".into(), "missing".into()])
+            .await
+            .unwrap(),
+        [Some(7), None]
+    );
+    assert!(cache.sync_ping().unwrap());
+    assert!(cache.ping().await.unwrap());
+    assert_eq!(cache.async_get_ttl("missing").await.unwrap(), None);
+    assert_eq!(
+        cache.async_scan_iter("job-", 25).await.unwrap(),
+        ["team:job-a", "team:job-b"]
+    );
+    assert_eq!(
+        cache
+            .delete_cache_keys(vec!["job-a".into(), "job-b".into()])
+            .await
+            .unwrap(),
+        2
+    );
+    assert_eq!(
+        cache
+            .async_set_cache_sadd("members", vec!["a".into(), "b".into()], None)
+            .await
+            .unwrap(),
+        2
+    );
+    assert_eq!(
+        cache
+            .async_rpush("queue", vec!["a".into(), "b".into()])
+            .await
+            .unwrap(),
+        2
+    );
+    assert_eq!(
+        cache.async_lpop("queue", Some(2)).await.unwrap(),
+        RedisLpopResult::Values(vec![b"a".to_vec(), b"b".to_vec()])
+    );
+    assert_eq!(
+        cache
+            .async_eval("return KEYS[1]".into(), vec!["key".into()], Vec::new())
+            .await
+            .unwrap(),
+        redis::Value::BulkString(b"team:key".to_vec())
+    );
+    assert_eq!(
+        cache
+            .async_register_script("return KEYS[1]".into())
+            .invoke(vec!["key".into()], Vec::new())
+            .await
+            .unwrap(),
+        redis::Value::BulkString(b"team:key".to_vec())
+    );
+    assert_eq!(cache.client_list().unwrap(), "id=1");
+    assert_eq!(cache.info().unwrap(), "redis_version:7");
+    cache.flushall().unwrap();
+}
+
+#[tokio::test]
+async fn direct_redis_pipelines_preserve_operation_order() {
+    let mut rpush_pipeline = redis::pipe();
+    rpush_pipeline
+        .cmd("RPUSH")
+        .arg("team:a")
+        .arg("one")
+        .cmd("RPUSH")
+        .arg("team:b")
+        .arg("two");
+    let mut lpop_pipeline = redis::pipe();
+    lpop_pipeline
+        .cmd("LPOP")
+        .arg("team:a")
+        .arg(2usize)
+        .cmd("LPOP")
+        .arg("team:b");
+    let connection = MockRedisConnection::new([
+        MockCmd::with_values(
+            rpush_pipeline,
+            Ok(vec![redis::Value::Int(1), redis::Value::Int(2)]),
+        ),
+        MockCmd::with_values(
+            lpop_pipeline,
+            Ok(vec![redis_test::redis_value!(["one"]), redis::Value::Nil]),
+        ),
+    ])
+    .assert_all_commands_consumed();
+    let queue = RedisCache::with_connection(connection, None, JsonCodec::<String>::new())
+        .with_namespace(Some("team".into()));
+
+    assert_eq!(
+        queue
+            .async_rpush_pipeline(vec![
+                RedisRpushOperation {
+                    key: "a".into(),
+                    values: vec![RedisArg::from("one")],
+                },
+                RedisRpushOperation {
+                    key: "b".into(),
+                    values: vec![RedisArg::from("two")],
+                },
+            ])
+            .await
+            .unwrap(),
+        [1, 2]
+    );
+    assert_eq!(
+        queue
+            .async_lpop_pipeline(vec![
+                RedisLpopOperation {
+                    key: "a".into(),
+                    count: Some(2),
+                },
+                RedisLpopOperation {
+                    key: "b".into(),
+                    count: None,
+                },
+            ])
+            .await
+            .unwrap(),
+        [
+            RedisLpopResult::Values(vec![b"one".to_vec()]),
+            RedisLpopResult::Missing,
+        ]
+    );
+
+    let mut increment_pipeline = redis::pipe();
+    increment_pipeline
+        .cmd("INCRBYFLOAT")
+        .arg("team:counter")
+        .arg(1.5f64)
+        .cmd("EXPIRE")
+        .arg("team:counter")
+        .arg(10u64)
+        .ignore()
+        .cmd("INCRBYFLOAT")
+        .arg("team:counter")
+        .arg(2.0f64);
+    let connection = MockRedisConnection::new([MockCmd::with_values(
+        increment_pipeline,
+        Ok(vec![
+            redis::Value::BulkString(b"1.5".to_vec()),
+            redis::Value::Int(1),
+            redis::Value::BulkString(b"3.5".to_vec()),
+        ]),
+    )])
+    .assert_all_commands_consumed();
+    let counters = RedisCache::with_connection(connection, None, JsonCodec::<f64>::new())
+        .with_namespace(Some("team".into()));
+    assert_eq!(
+        counters
+            .async_increment_pipeline(vec![
+                IncrementOperation {
+                    key: "counter".into(),
+                    amount: 1.5,
+                    ttl: Some(Duration::from_secs(10)),
+                },
+                IncrementOperation {
+                    key: "counter".into(),
+                    amount: 2.0,
+                    ttl: None,
+                },
+            ])
+            .await
+            .unwrap(),
+        [1.5, 3.5]
+    );
+}
+
+const INCREMENT_WITH_FLOOR_SCRIPT: &str = concat!(
+    "local count = redis.call('INCRBY', KEYS[1], ARGV[1]); ",
+    "if count < 0 then count = redis.call('INCRBY', KEYS[1], -count); end; ",
+    "if redis.call('TTL', KEYS[1]) < 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; ",
+    "return count"
+);
+const SET_MAX_SCRIPT: &str = concat!(
+    "local current = redis.call('GET', KEYS[1]); ",
+    "if current == false or tonumber(current) < tonumber(ARGV[1]) then ",
+    "redis.call('SET', KEYS[1], ARGV[1]); ",
+    "if tonumber(ARGV[2]) > 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; ",
+    "return ARGV[1]; end; return current"
+);
+
+#[tokio::test]
+async fn counter_repairs_are_atomic_and_use_default_ttl() {
+    let floor = || {
+        redis::cmd("EVAL")
+            .arg(INCREMENT_WITH_FLOOR_SCRIPT)
+            .arg(1)
+            .arg("team:counter")
+            .arg(-2i64)
+            .arg(30u64)
+            .clone()
+    };
+    let connection = MockRedisConnection::new([
+        MockCmd::new(floor(), Ok(0i64)),
+        MockCmd::new(floor(), Ok(0i64)),
+        MockCmd::new(
+            redis::cmd("EVAL")
+                .arg(SET_MAX_SCRIPT)
+                .arg(1)
+                .arg("team:counter")
+                .arg(4.5f64)
+                .arg(600u64),
+            Ok("4.5"),
+        ),
+    ])
+    .assert_all_commands_consumed();
+    let cache = RedisCache::with_connection(connection, None, JsonCodec::<f64>::new())
+        .with_namespace(Some("team".into()));
+
+    assert_eq!(
+        cache
+            .increment_with_floor("counter", -2, Duration::from_secs(30))
+            .unwrap(),
+        0
+    );
+    assert_eq!(
+        cache
+            .async_increment_with_floor("counter", -2, Duration::from_secs(30))
+            .await
+            .unwrap(),
+        0
+    );
+    assert_eq!(
+        cache.async_set_max("counter", 4.5, None).await.unwrap(),
+        4.5
+    );
+}
+
+const CLAIM_SCRIPT: &str = concat!(
+    "local current = redis.call('GET', KEYS[1]); ",
+    "if ARGV[1] == '' then if current ~= false and current ~= '' then return 0; end; ",
+    "elseif current ~= ARGV[1] then return 0; end; ",
+    "if ARGV[3] ~= '' then redis.call('SET', KEYS[1], ARGV[3], 'EX', ARGV[2]); ",
+    "elseif ARGV[4] == '1' then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; return 1"
+);
+
+fn claim_eval(expected: &str, write: &str, refresh: bool) -> redis::Cmd {
+    let mut cmd = redis::cmd("EVAL");
+    cmd.arg(CLAIM_SCRIPT)
+        .arg(1)
+        .arg("pin")
+        .arg(expected)
+        .arg(600)
+        .arg(write)
+        .arg(u8::from(refresh));
+    cmd
+}
+
+#[tokio::test]
+async fn claims_match_eligible_values_written_by_another_encoder() {
+    let python_payload = r#"{"model_id": "a", "deployment": "east"}"#;
+    let stored = serde_json::json!({"deployment": "east", "model_id": "a"});
+    let candidate = serde_json::json!({"model_id": "b"});
+    let connection = MockRedisConnection::new([
+        MockCmd::new(redis::cmd("GET").arg("pin"), Ok(python_payload)),
+        MockCmd::new(claim_eval(python_payload, "", true), Ok(1)),
+    ])
+    .assert_all_commands_consumed();
+    let cache =
+        RedisCache::with_connection(connection, None, JsonCodec::<serde_json::Value>::new());
+
+    assert_eq!(
+        cache
+            .async_claim_cache(
+                "pin",
+                candidate,
+                vec![stored.clone()],
+                ExactCacheContext::default()
+            )
+            .await
+            .unwrap(),
+        stored
+    );
+}
+
+#[test]
+fn claims_retry_when_the_key_changes_and_replace_ineligible_winners() {
+    let candidate = serde_json::json!({"model_id": "b"});
+    let payload = r#"{"model_id":"b"}"#;
+    let connection = MockRedisConnection::new([
+        MockCmd::new(redis::cmd("GET").arg("pin"), Ok(redis::Value::Nil)),
+        MockCmd::new(claim_eval("", payload, false), Ok(0)),
+        MockCmd::new(redis::cmd("GET").arg("pin"), Ok(r#"{"model_id":"gone"}"#)),
+        MockCmd::new(claim_eval(r#"{"model_id":"gone"}"#, payload, false), Ok(1)),
+    ])
+    .assert_all_commands_consumed();
+    let cache =
+        RedisCache::with_connection(connection, None, JsonCodec::<serde_json::Value>::new());
+
+    assert_eq!(
+        cache
+            .claim_cache(
+                "pin",
+                candidate.clone(),
+                &[serde_json::json!({"model_id": "a"})],
+                ExactCacheContext::default()
+            )
+            .unwrap(),
+        candidate
+    );
+}
+
+#[test]
+fn claims_without_eligible_values_keep_the_winner_without_refreshing_its_ttl() {
+    let stored = r#"{"model_id": "a"}"#;
+    let connection = MockRedisConnection::new([
+        MockCmd::new(redis::cmd("GET").arg("pin"), Ok(stored)),
+        MockCmd::new(claim_eval(stored, "", false), Ok(1)),
+    ])
+    .assert_all_commands_consumed();
+    let cache =
+        RedisCache::with_connection(connection, None, JsonCodec::<serde_json::Value>::new());
+
+    assert_eq!(
+        cache
+            .claim_cache(
+                "pin",
+                serde_json::json!({"model_id": "b"}),
+                &[],
+                ExactCacheContext::default()
+            )
+            .unwrap(),
+        serde_json::json!({"model_id": "a"})
+    );
+}
+
+#[tokio::test]
+async fn async_increment_runs_the_atomic_script() {
+    let mut eval = redis::cmd("EVAL");
+    eval.arg(concat!(
+        "local value = redis.call('INCRBYFLOAT', KEYS[1], ARGV[1]); ",
+        "if redis.call('TTL', KEYS[1]) == -1 then ",
+        "redis.call('EXPIRE', KEYS[1], ARGV[2]); end; return value"
+    ))
+    .arg(1)
+    .arg("counter")
+    .arg(2.5f64)
+    .arg(600);
+    let connection =
+        MockRedisConnection::new([MockCmd::new(eval, Ok("4.5"))]).assert_all_commands_consumed();
+    let cache = RedisCache::with_connection(connection, None, JsonCodec::<f64>::new());
+
+    assert_eq!(
+        cache
+            .async_increment("counter", 2.5, ExactCacheContext::default())
+            .await
+            .unwrap(),
+        4.5
+    );
 }

litellm-rust/crates/cache-redis/tests/cluster.rs

@@ -0,0 +1,492 @@
+//! Contract tests against a real Redis Cluster. Set `LITELLM_TEST_REDIS_CLUSTER_NODES` to a
+//! comma separated `host:port` list (for example `127.0.0.1:7000,127.0.0.1:7001`) to run them.
+
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheConnectionStatus, CacheScript, ClaimCache,
+    CounterCache, DeleteCache, Error, ExactCacheContext, FlushCache, IncrementOperation, JsonCodec,
+    ScriptCache,
+};
+use litellm_cache_redis::{
+    RedisArg, RedisCache, RedisLpopOperation, RedisLpopResult, RedisNode, RedisRpushOperation,
+    RedisTopology,
+};
+use redis::cluster_routing::Slot;
+
+type Cache = RedisCache<JsonCodec<serde_json::Value>>;
+
+fn topology() -> Option<RedisTopology> {
+    let nodes = std::env::var("LITELLM_TEST_REDIS_CLUSTER_NODES").ok()?;
+    let startup_nodes = nodes
+        .split(',')
+        .map(|node| {
+            let (host, port) = node.trim().rsplit_once(':').expect("host:port");
+            RedisNode {
+                host: host.to_string(),
+                port: port.parse().expect("port"),
+            }
+        })
+        .collect();
+    Some(RedisTopology::Cluster { startup_nodes })
+}
+
+fn namespace(label: &str) -> String {
+    let nanos = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap()
+        .as_nanos();
+    format!("cluster-test:{label}:{nanos}")
+}
+
+fn cluster_url() -> String {
+    std::env::var("LITELLM_TEST_REDIS_CLUSTER_URL")
+        .unwrap_or_else(|_| "redis://127.0.0.1:7000".into())
+}
+
+fn cluster_cache(label: &str) -> Option<Cache> {
+    let topology = topology()?;
+    Some(
+        Cache::connect(
+            &cluster_url(),
+            &topology,
+            Some(Duration::from_secs(120)),
+            JsonCodec::new(),
+        )
+        .expect("cluster connection")
+        .with_namespace(Some(namespace(label))),
+    )
+}
+
+fn counter_cache(label: &str) -> Option<RedisCache<JsonCodec<f64>>> {
+    let topology = topology()?;
+    Some(
+        RedisCache::connect(
+            &cluster_url(),
+            &topology,
+            Some(Duration::from_secs(60)),
+            JsonCodec::new(),
+        )
+        .expect("cluster connection")
+        .with_namespace(Some(namespace(label))),
+    )
+}
+
+fn multi_slot_keys(count: usize) -> Vec<String> {
+    let keys: Vec<String> = (0..count).map(|index| format!("key-{index}")).collect();
+    let slots: std::collections::HashSet<Slot> = keys.iter().map(Slot::for_key).collect();
+    assert!(slots.len() > 1, "keys must span multiple slots");
+    keys
+}
+
+macro_rules! cluster_or_skip {
+    ($label:expr) => {
+        match cluster_cache($label) {
+            Some(cache) => cache,
+            None => return,
+        }
+    };
+}
+
+#[test]
+fn constructor_rejects_clusters_without_startup_nodes() {
+    let error = Cache::connect(
+        "redis://127.0.0.1:7000",
+        &RedisTopology::Cluster {
+            startup_nodes: Vec::new(),
+        },
+        None,
+        JsonCodec::new(),
+    )
+    .err();
+    assert!(matches!(error, Some(Error::Unavailable)));
+}
+
+#[test]
+fn constructor_rejects_unix_socket_urls_for_clusters() {
+    let error = Cache::connect(
+        "redis+unix:///tmp/redis.sock",
+        &RedisTopology::Cluster {
+            startup_nodes: vec![RedisNode {
+                host: "127.0.0.1".into(),
+                port: 7000,
+            }],
+        },
+        None,
+        JsonCodec::new(),
+    )
+    .err();
+    assert!(matches!(error, Some(Error::Unavailable)));
+}
+
+#[test]
+fn single_key_operations_round_trip_with_ttl_rounding() {
+    let cache = cluster_or_skip!("single");
+    let context = ExactCacheContext {
+        ttl: Some(Duration::from_millis(1500)),
+    };
+    let keys = multi_slot_keys(12);
+    for (index, key) in keys.iter().enumerate() {
+        cache
+            .set_cache(key, serde_json::json!({ "index": index }), &context)
+            .unwrap();
+    }
+    for (index, key) in keys.iter().enumerate() {
+        assert_eq!(
+            cache.get_cache(key, &context).unwrap(),
+            Some(serde_json::json!({ "index": index }))
+        );
+    }
+    let runtime = tokio::runtime::Runtime::new().unwrap();
+    let ttl = runtime.block_on(cache.async_get_ttl(&keys[0])).unwrap();
+    assert_eq!(ttl, Some(2));
+    cache.delete_cache(&keys[0]).unwrap();
+    assert_eq!(cache.get_cache(&keys[0], &context).unwrap(), None);
+    assert!(cache.sync_ping().unwrap());
+}
+
+#[tokio::test]
+async fn batch_reads_span_slots_and_preserve_order_with_malformed_entries() {
+    let cache = cluster_or_skip!("batch");
+    let context = ExactCacheContext::default();
+    let keys = multi_slot_keys(40);
+    for (index, key) in keys.iter().enumerate() {
+        if index % 5 == 0 {
+            continue;
+        }
+        cache
+            .async_set_cache(key, serde_json::json!(index), context.clone())
+            .await
+            .unwrap();
+    }
+    let mut raw = redis::cluster::ClusterClient::new(vec![cluster_url()])
+        .unwrap()
+        .get_connection()
+        .unwrap();
+    let malformed = format!("{}:{}", cache.namespace().unwrap(), keys[1]);
+    redis::cmd("SET")
+        .arg(&malformed)
+        .arg("not json")
+        .exec(&mut raw)
+        .unwrap();
+
+    let entries = cache
+        .async_batch_get_cache(keys.clone(), context.clone())
+        .await
+        .unwrap();
+    assert_eq!(entries.len(), keys.len());
+    for (index, entry) in entries.iter().enumerate() {
+        let expected = if index == 1 {
+            BatchEntry::Invalid
+        } else if index % 5 == 0 {
+            BatchEntry::Miss
+        } else {
+            BatchEntry::Hit(serde_json::json!(index))
+        };
+        assert_eq!(*entry, expected, "entry {index}");
+    }
+    let sync_entries = cache.batch_get_cache(&keys, &context).unwrap();
+    assert_eq!(sync_entries, entries);
+
+    cache.delete_cache_keys(keys.clone()).await.unwrap();
+    let entries = cache.async_batch_get_cache(keys, context).await.unwrap();
+    assert!(entries.iter().all(|entry| *entry == BatchEntry::Miss));
+}
+
+#[tokio::test]
+async fn pipelines_group_by_slot_and_return_results_in_submission_order() {
+    let cache = cluster_or_skip!("pipeline");
+    let keys = multi_slot_keys(30);
+    let entries = keys
+        .iter()
+        .enumerate()
+        .map(|(index, key)| (key.clone(), serde_json::json!(index)))
+        .collect();
+    cache
+        .async_set_cache_pipeline(entries, ExactCacheContext::default())
+        .await
+        .unwrap();
+    let hits = cache
+        .async_batch_get_cache(keys.clone(), ExactCacheContext::default())
+        .await
+        .unwrap();
+    assert!(
+        hits.iter()
+            .enumerate()
+            .all(|(index, entry)| *entry == BatchEntry::Hit(serde_json::json!(index)))
+    );
+
+    let queues: Vec<String> = keys.iter().map(|key| format!("queue:{key}")).collect();
+    let pushed = cache
+        .async_rpush_pipeline(
+            queues
+                .iter()
+                .enumerate()
+                .map(|(index, key)| RedisRpushOperation {
+                    key: key.clone(),
+                    values: (0..=index)
+                        .map(|value| RedisArg::Integer(value as i64))
+                        .collect(),
+                })
+                .collect(),
+        )
+        .await
+        .unwrap();
+    assert_eq!(pushed, (1..=keys.len()).collect::<Vec<_>>());
+    let popped = cache
+        .async_lpop_pipeline(
+            queues
+                .iter()
+                .enumerate()
+                .map(|(index, key)| RedisLpopOperation {
+                    key: key.clone(),
+                    count: (index % 2 == 0).then_some(2),
+                })
+                .collect(),
+        )
+        .await
+        .unwrap();
+    for (index, result) in popped.into_iter().enumerate() {
+        match result {
+            RedisLpopResult::Value(value) => {
+                assert_eq!(index % 2, 1, "queue {index}");
+                assert_eq!(value, b"0");
+            }
+            RedisLpopResult::Values(values) => {
+                assert_eq!(index % 2, 0, "queue {index}");
+                let expected: Vec<Vec<u8>> = (0..=index)
+                    .take(2)
+                    .map(|value| value.to_string().into_bytes())
+                    .collect();
+                assert_eq!(values, expected);
+            }
+            other => panic!("queue {index}: {other:?}"),
+        }
+    }
+
+    let counters: Vec<String> = keys.iter().map(|key| format!("counter:{key}")).collect();
+    let Some(counter) = counter_cache("counter") else {
+        return;
+    };
+    let totals = counter
+        .async_increment_pipeline(
+            counters
+                .iter()
+                .enumerate()
+                .map(|(index, key)| IncrementOperation {
+                    key: key.clone(),
+                    amount: index as f64 + 0.5,
+                    ttl: (index % 3 == 0).then_some(Duration::from_secs(30)),
+                })
+                .collect(),
+        )
+        .await
+        .unwrap();
+    let expected: Vec<f64> = (0..keys.len()).map(|index| index as f64 + 0.5).collect();
+    assert_eq!(totals, expected);
+    assert_eq!(counter.async_get_ttl(&counters[0]).await.unwrap(), Some(30));
+    assert_eq!(counter.async_get_ttl(&counters[1]).await.unwrap(), None);
+    counter.async_flush_cache().await.unwrap();
+    cache.async_flush_cache().await.unwrap();
+}
+
+#[tokio::test]
+async fn scan_and_scoped_flush_cover_every_primary() {
+    let cache = cluster_or_skip!("flush");
+    let other = cluster_or_skip!("other");
+    let context = ExactCacheContext::default();
+    let keys = multi_slot_keys(60);
+    for key in &keys {
+        cache
+            .async_set_cache(key, serde_json::json!(true), context.clone())
+            .await
+            .unwrap();
+        other
+            .async_set_cache(key, serde_json::json!(true), context.clone())
+            .await
+            .unwrap();
+    }
+    let mut scanned = cache.async_scan_iter("key-", 1000).await.unwrap();
+    scanned.sort();
+    let mut expected: Vec<String> = keys
+        .iter()
+        .map(|key| format!("{}:{key}", cache.namespace().unwrap()))
+        .collect();
+    expected.sort();
+    assert_eq!(scanned, expected);
+    assert_eq!(cache.async_scan_iter("key-", 7).await.unwrap().len(), 7);
+
+    cache.flush_cache().unwrap();
+    let flushed = cache
+        .async_batch_get_cache(keys.clone(), context.clone())
+        .await
+        .unwrap();
+    assert!(flushed.iter().all(|entry| *entry == BatchEntry::Miss));
+    let kept = other.async_batch_get_cache(keys, context).await.unwrap();
+    assert!(
+        kept.iter()
+            .all(|entry| *entry == BatchEntry::Hit(serde_json::json!(true)))
+    );
+    other.async_flush_cache().await.unwrap();
+}
+
+fn ping_calls_per_node(startup: &redis::Client) -> Vec<(String, u64)> {
+    let mut connection = startup.get_connection().unwrap();
+    let nodes: String = redis::cmd("CLUSTER")
+        .arg("NODES")
+        .query(&mut connection)
+        .unwrap();
+    let mut counts: Vec<(String, u64)> = nodes
+        .lines()
+        .map(|line| {
+            let address = line.split_whitespace().nth(1).unwrap();
+            let address = address.split('@').next().unwrap();
+            let mut node = redis::Client::open(format!("redis://{address}"))
+                .unwrap()
+                .get_connection()
+                .unwrap();
+            let stats: String = redis::cmd("INFO")
+                .arg("commandstats")
+                .query(&mut node)
+                .unwrap();
+            let calls = stats
+                .lines()
+                .find_map(|stat| stat.strip_prefix("cmdstat_ping:calls="))
+                .and_then(|rest| rest.split(',').next())
+                .map_or(0, |calls| calls.parse().unwrap());
+            (address.to_string(), calls)
+        })
+        .collect();
+    counts.sort();
+    counts
+}
+
+#[tokio::test]
+async fn ping_reaches_every_node() {
+    let cache = cluster_or_skip!("ping");
+    let startup = redis::Client::open(cluster_url()).unwrap();
+    let before = ping_calls_per_node(&startup);
+    assert!(before.len() >= 2, "{before:?}");
+    assert!(cache.ping().await.unwrap());
+    let after = ping_calls_per_node(&startup);
+    for ((node, calls_before), (_, calls_after)) in before.iter().zip(&after) {
+        assert!(calls_after > calls_before, "{node} was not pinged");
+    }
+    assert!(cache.sync_ping().unwrap());
+    let result = cache.test_connection().await.unwrap();
+    assert_eq!(result.status, CacheConnectionStatus::Success);
+}
+
+#[tokio::test]
+async fn counters_claims_scripts_and_sets_work_on_the_cluster() {
+    let Some(counter) = counter_cache("counter") else {
+        return;
+    };
+    let context = ExactCacheContext::default();
+    assert_eq!(
+        counter
+            .increment_cache("spend", 1.5, context.clone())
+            .unwrap(),
+        1.5
+    );
+    assert_eq!(
+        counter
+            .async_increment("spend", 2.0, context.clone())
+            .await
+            .unwrap(),
+        3.5
+    );
+    assert_eq!(
+        counter
+            .increment_with_floor("budget", -3, Duration::from_secs(30))
+            .unwrap(),
+        0
+    );
+    assert_eq!(
+        counter
+            .async_increment_with_floor("budget", 7, Duration::from_secs(30))
+            .await
+            .unwrap(),
+        7
+    );
+    assert_eq!(counter.async_set_max("peak", 4.0, None).await.unwrap(), 4.0);
+    assert_eq!(counter.async_set_max("peak", 2.0, None).await.unwrap(), 4.0);
+    counter.flush_cache().unwrap();
+
+    let cache = cluster_or_skip!("claim");
+    let owner = serde_json::json!("owner-a");
+    let rival = serde_json::json!("owner-b");
+    assert_eq!(
+        cache
+            .claim_cache("lock", owner.clone(), &[], context.clone())
+            .unwrap(),
+        owner
+    );
+    assert_eq!(
+        cache
+            .async_claim_cache("lock", rival.clone(), vec![owner.clone()], context.clone())
+            .await
+            .unwrap(),
+        owner
+    );
+    assert_eq!(
+        cache
+            .claim_cache("lock", rival.clone(), &[], context.clone())
+            .unwrap(),
+        owner
+    );
+    assert_eq!(
+        cache
+            .async_claim_cache("lock", rival.clone(), vec![rival.clone()], context.clone())
+            .await
+            .unwrap(),
+        rival
+    );
+
+    let script = cache
+        .async_register_script("return redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2])".into());
+    let reply = script
+        .invoke(
+            vec!["scripted".into()],
+            vec![RedisArg::Bytes(b"payload".to_vec()), RedisArg::Integer(5)],
+        )
+        .await
+        .unwrap();
+    assert_eq!(reply, redis::Value::Okay);
+    assert_eq!(cache.async_get_ttl("scripted").await.unwrap(), Some(5));
+    let evaluated: redis::Value = cache
+        .async_eval(
+            "return redis.call('GET', KEYS[1])".into(),
+            vec!["scripted".into()],
+            Vec::new(),
+        )
+        .await
+        .unwrap();
+    assert_eq!(evaluated, redis::Value::BulkString(b"payload".to_vec()));
+
+    assert_eq!(
+        cache
+            .async_set_cache_sadd(
+                "members",
+                vec![
+                    RedisArg::Bytes(b"a".to_vec()),
+                    RedisArg::Bytes(b"b".to_vec())
+                ],
+                Some(Duration::from_secs(9)),
+            )
+            .await
+            .unwrap(),
+        2
+    );
+    assert_eq!(cache.async_get_ttl("members").await.unwrap(), Some(9));
+
+    let result = cache.test_connection().await.unwrap();
+    assert_eq!(result.status, CacheConnectionStatus::Success);
+    assert!(cache.ping().await.unwrap());
+    let info = cache.info().unwrap();
+    assert!(info.matches("redis_version").count() > 1, "{info}");
+    assert!(cache.client_list().unwrap().contains("id="));
+    cache.async_flush_cache().await.unwrap();
+    assert_eq!(cache.async_get_ttl("members").await.unwrap(), None);
+    assert_eq!(cache.get_cache("lock", &context).unwrap(), None);
+}

litellm-rust/crates/cache-response/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "litellm-cache-response"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-cache.workspace = true
+py_literal = "0.4.0"
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+
+[dev-dependencies]
+litellm-cache-memory.workspace = true
+litellm-cache-redis.workspace = true
+redis = "1.7.0"
+redis-test = "1.0.4"
+tokio.workspace = true

litellm-rust/crates/cache-response/README.md

@@ -0,0 +1,63 @@
+# Response cache foundation
+
+`ResponseCache<B>` adds request keys, independent read/write controls, response envelopes, and freshness checks to any `B: BaseCache<Value = CacheEntry>`
+
+## Ownership
+
+`litellm-cache` defines typed storage and codec traits. Memory and Redis implement those traits without depending on response policy. Other consumers can store their own value types using the same backend implementations
+
+`litellm-cache-response` owns response keys, controls, entries, the Python-compatible response codec, and `WriteBuffer`, the backend-neutral deferred-write policy. It has no runtime dependency on a specific cache backend or Python
+
+The Python bridge constructs backends and selects them through its private `NativeResponseCache` enum, which only dispatches. Generic Rust callers inject their backend directly. A native gateway can construct the same generic response service in its own host
+
+## Native Rust use
+
+```rust
+use std::{sync::Arc, time::Duration};
+use litellm_cache_memory::InMemoryCache;
+use litellm_cache_response::{CacheKeyInput, ResponseCache, ResponseCacheRequest};
+use serde_json::json;
+
+let cache = ResponseCache::new(Arc::new(InMemoryCache::default()));
+let request = ResponseCacheRequest::new(CacheKeyInput {
+    preset: Some("example:key".into()),
+    ..Default::default()
+});
+let now = Duration::from_secs(100);
+cache.store(&request, json!({"answer": 7}), now)?;
+assert_eq!(cache.async_lookup(&request, now).await?, Some(json!({"answer": 7})));
+```
+
+For Redis, inject `RedisCache::new(url, ttl, ResponseCacheCodec)` instead. Namespaces are optional and existing namespace prefixes are preserved. Sync operations check out independent connections from a bounded pool, while async callers, including counters and claims, move that blocking work off the executor. The pool skips the checkout PING and instead discards any connection whose command failed
+
+Callers supply Unix time for response freshness. Backend TTL uses its own clock. A read can reject an entry through `max_age` even while the backend still retains it
+
+## Python integration boundary
+
+The extension keeps a private test harness for memory and Redis single and batch response lookup and storage. Batch lookup returns ordered values plus missing indices for embedding partial-hit wiring. No bridge-only cache type is part of the public API
+
+The bridge also exposes a production-shaped response cache runtime selected through the Rust catalog. Its shipped rule set is empty, so current SDK, Router, and proxy calls stay on Python and do not construct native cache resources. Tests can inject a rule and build the native memory runtime from an ordinary Python `Cache` configuration without changing the legacy cache classes
+
+Object responses are written as they are, and every other response shape is written as a serialized string, which is the pair of shapes Python reads. A string on the wire is therefore always a serialized response, so string-valued responses round trip. Typed backends such as memory never pass through the codec
+
+The resolver reads the namespace's `cache` attribute each time it resolves. A captured binding retains the selected service for its operation, including background writes. `None` disables caching. Custom Python cache objects keep their original methods, arguments, returned awaitables, exceptions, and caller-task execution
+
+Python callbacks use the built-in `Cache` API, so a `Cache` subclass works unchanged. A batch lookup takes one original kwargs mapping per request and returns the list of `get_cache` or gathered `async_get_cache` results, while native bindings return `{values, missing_indices}`. A batch store hands the caller's original result to `async_add_cache_pipeline`. `ping` calls `ping`, and a flush goes to the facade's backend
+
+The private facade test harness checks object identity, method overrides, effective TTL, Redis namespace, memory capacity, and later configuration changes before selecting native execution. Its snapshot includes Redis connection settings, so a later `redis_kwargs` change, including an SSL option, selects Python callback execution. Buffered async writes honor `redis_flush_size`. Public activation must construct the shared native service from the initial Python Redis settings, including `litellm.default_redis_ttl` and SSL options. A buffered entry keeps the time it was produced, and a failed flush drops its batch instead of growing the buffer during an outage. The harness does not migrate entries or replace Python methods. Until activation configures one shared service, the Python facade and native test service can hold separate data. Existing public cache constructors remain on Python
+
+Native cache handles must be recreated after fork. The bridge releases the GIL around native operations, and Redis runs blocking connection operations off the async executor. Native errors propagate to the host, which owns the existing fail-open and logging policy
+
+The Redis backend also provides the primitives needed to preserve its direct Python surface later: TLS URLs, ping, bulk delete, counter batches, TTL, scan, set membership, raw queue push and pop, queue and counter pipelines, counter floor and maximum operations, script evaluation, client information, namespaced flush, and full flush. These are backend operations only and are not exported to Python by this PR. Memory provides TTL, oldest-key, and counter-pipeline operations
+
+## Adding another backend
+
+Implement `BaseCache` for the backend with its associated value type, and accept a `CacheCodec` when wire serialization is needed. `ResponseCache<B>` then works without another response implementation. Add a concrete bridge enum variant and constructor only when exposing that backend to Python
+
+Verify typed values, TTL precedence, missing entries, serialization failures, namespaces, batch ordering, and sync/async behavior. Run response fixtures with `ResponseCacheCodec`, including both Python envelope encodings, before enabling a public facade
+
+## Follow-up scope
+
+Public SDK, Router, and proxy activation still need constructor parity, stream replay, embedding partial-batch integration, response reconstruction, callback scheduling, and failure-policy integration. This foundation does not switch those request paths
+
+Redis cluster, disk, and cloud stores remain follow-ups. Semantic backends plug in through `SemanticCacheContext`, which carries the prompt inputs and metadata alongside the cache TTL. The generic dual cache takes read, write, and remote-failure policies, runs its async operations through the async L2 methods, and provides L2-first counters and atomic affinity claims. Errors propagate by default, and `RemoteFailurePolicy::UseLocal` opts key-value operations and claims into the local tier when L2 is unavailable. Claims compare decoded values, so a pin written by Python still matches. Public Router integration remains follow-up work. Reservations and pubsub still need explicit capabilities owned by their consuming features. Adding a cache backend does not establish those guarantees

litellm-rust/crates/cache-response/src/buffer.rs

@@ -0,0 +1,45 @@
+use std::{sync::Mutex, time::Duration};
+
+use litellm_cache::Error;
+use serde_json::Value;
+
+use crate::{ExactResponseCache, ResponseCacheRequest};
+
+pub struct WriteBuffer {
+    flush_size: usize,
+    entries: Mutex<Vec<(ResponseCacheRequest, Value, Duration)>>,
+}
+
+impl WriteBuffer {
+    pub fn new(flush_size: usize) -> Self {
+        Self {
+            flush_size: flush_size.max(1),
+            entries: Mutex::new(Vec::new()),
+        }
+    }
+
+    pub async fn async_store(
+        &self,
+        cache: &dyn ExactResponseCache,
+        request: &ResponseCacheRequest,
+        response: Value,
+        now: Duration,
+    ) -> Result<(), Error> {
+        let pending = {
+            let mut entries = self.entries.lock().map_err(|_| Error::Unavailable)?;
+            entries.push((request.clone(), response, now));
+            (entries.len() >= self.flush_size).then(|| std::mem::take(&mut *entries))
+        };
+        // A failed flush drops its batch, as Python does. Requeueing would grow the
+        // buffer and re-send an ever larger pipeline on every write during an outage.
+        match pending {
+            Some(pending) => cache.async_store_entries(pending).await,
+            None => Ok(()),
+        }
+    }
+
+    pub fn clear(&self) -> Result<(), Error> {
+        self.entries.lock().map_err(|_| Error::Unavailable)?.clear();
+        Ok(())
+    }
+}

litellm-rust/crates/cache-response/src/caching.rs

@@ -0,0 +1,147 @@
+use std::time::Duration;
+
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+
+#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
+pub enum CacheMode {
+    #[default]
+    #[serde(rename = "default_on")]
+    DefaultOn,
+    #[serde(rename = "default_off")]
+    DefaultOff,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct CacheKeyField {
+    pub name: String,
+    pub value: Option<String>,
+    pub api_parameter: bool,
+    pub internal_parameter: bool,
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+#[serde(default)]
+pub struct CacheKeyInput {
+    pub fields: Vec<CacheKeyField>,
+    pub preset: Option<String>,
+    pub namespace: Option<String>,
+    pub include_provider_parameters: bool,
+}
+
+#[derive(Default)]
+pub struct CacheKeyContext {
+    pub model_group: Option<String>,
+    pub caching_groups: Vec<(Vec<String>, String)>,
+    pub file_checksum: Option<String>,
+    pub file_object_name: Option<String>,
+    pub metadata_file_name: Option<String>,
+    pub parameters_file_name: Option<String>,
+}
+
+impl CacheKeyContext {
+    pub fn apply(self, input: &mut CacheKeyInput) {
+        let group = self.model_group.as_ref().and_then(|model| {
+            self.caching_groups
+                .iter()
+                .find(|(models, _)| models.contains(model))
+        });
+        for field in &mut input.fields {
+            match field.name.as_str() {
+                "model" => {
+                    field.value = group
+                        .map(|(_, formatted)| formatted.clone())
+                        .or_else(|| self.model_group.clone())
+                        .or_else(|| field.value.take())
+                }
+                "file" => {
+                    field.value = self
+                        .file_checksum
+                        .clone()
+                        .or_else(|| self.file_object_name.clone())
+                        .or_else(|| self.metadata_file_name.clone())
+                        .or_else(|| self.parameters_file_name.clone())
+                }
+                _ => {}
+            }
+        }
+    }
+}
+
+pub fn get_cache_key(input: &CacheKeyInput) -> String {
+    cache_key(input)
+}
+
+pub fn cache_key(input: &CacheKeyInput) -> String {
+    if let Some(preset) = &input.preset {
+        return preset.clone();
+    }
+    let mut digest = Sha256::new();
+    for field in &input.fields {
+        if (field.api_parameter || (input.include_provider_parameters && !field.internal_parameter))
+            && let Some(value) = &field.value
+        {
+            digest.update(field.name.as_bytes());
+            digest.update(b": ");
+            digest.update(value.as_bytes());
+        }
+    }
+    let hash = format!("{:x}", digest.finalize());
+    input
+        .namespace
+        .as_deref()
+        .filter(|namespace| !namespace.is_empty())
+        .map_or(hash.clone(), |namespace| format!("{namespace}:{hash}"))
+}
+
+#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize)]
+pub struct CacheControls {
+    pub supported_call_type: bool,
+    pub configured: bool,
+    pub native_backend: bool,
+    pub default_on: bool,
+    pub caching: Option<bool>,
+    pub no_cache: bool,
+    pub no_store: bool,
+    #[serde(default)]
+    pub use_cache: bool,
+}
+
+impl CacheControls {
+    pub fn reads(self) -> bool {
+        self.supported_call_type
+            && self.configured
+            && self.caching.unwrap_or(true)
+            && !self.no_cache
+            && (self.default_on || self.use_cache)
+    }
+
+    pub fn writes(self) -> bool {
+        self.supported_call_type
+            && self.configured
+            && self.caching.unwrap_or(true)
+            && !self.no_store
+            && (self.default_on || self.use_cache)
+    }
+}
+
+pub fn should_use_cache(controls: CacheControls) -> bool {
+    controls.reads() || controls.writes()
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct CacheEntry {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub timestamp: Option<f64>,
+    pub response: Value,
+}
+
+impl CacheEntry {
+    pub fn fresh(&self, now: Duration, max_age: Option<Duration>) -> bool {
+        self.timestamp.is_none_or(|timestamp| {
+            timestamp.is_finite()
+                && max_age.is_none_or(|age| now.as_secs_f64() - timestamp <= age.as_secs_f64())
+        })
+    }
+}

litellm-rust/crates/cache-response/src/codec.rs

@@ -0,0 +1,129 @@
+use litellm_cache::{CacheCodec, Error};
+use serde_json::Value;
+
+use crate::CacheEntry;
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct ResponseCacheCodec;
+
+impl CacheCodec for ResponseCacheCodec {
+    type Value = CacheEntry;
+
+    fn encode(&self, value: &CacheEntry) -> Result<Vec<u8>, Error> {
+        if value
+            .timestamp
+            .is_some_and(|timestamp| !timestamp.is_finite())
+        {
+            return Err(Error::InvalidEntry);
+        }
+        // Python reads a `response` that is either a dict or a serialized string, so every
+        // other shape is written serialized. A string on the wire is therefore always a
+        // serialized response, which keeps string-valued responses unambiguous.
+        if value.timestamp.is_none() || value.response.is_object() {
+            return serde_json::to_vec(value).map_err(|_| Error::InvalidEntry);
+        }
+        let response = serde_json::to_string(&value.response).map_err(|_| Error::InvalidEntry)?;
+        serde_json::to_vec(&CacheEntry {
+            timestamp: value.timestamp,
+            response: Value::String(response),
+        })
+        .map_err(|_| Error::InvalidEntry)
+    }
+
+    fn decode(&self, bytes: &[u8]) -> Result<CacheEntry, Error> {
+        let text = std::str::from_utf8(bytes).map_err(|_| Error::InvalidEntry)?;
+        let value = decode_value(text)?;
+        let Some(timestamp) = value.get("timestamp") else {
+            return Ok(CacheEntry {
+                timestamp: None,
+                response: value,
+            });
+        };
+        let Some(timestamp) = timestamp.as_f64().filter(|timestamp| timestamp.is_finite()) else {
+            return Err(Error::InvalidEntry);
+        };
+        let response = match value.get("response").ok_or(Error::InvalidEntry)? {
+            Value::String(text) => decode_value(text)?,
+            response => response.clone(),
+        };
+        Ok(CacheEntry {
+            timestamp: Some(timestamp),
+            response,
+        })
+    }
+}
+
+fn decode_value(text: &str) -> Result<Value, Error> {
+    if let Ok(value) = serde_json::from_str(text) {
+        return Ok(value);
+    }
+    check_literal_depth(text)?;
+    let literal: py_literal::Value = text.parse().map_err(|_| Error::InvalidEntry)?;
+    literal_value(literal, 0)
+}
+
+fn literal_value(value: py_literal::Value, depth: usize) -> Result<Value, Error> {
+    use py_literal::Value as Literal;
+    if depth > 128 {
+        return Err(Error::InvalidEntry);
+    }
+    match value {
+        Literal::String(text) => Ok(Value::String(text)),
+        Literal::Boolean(value) => Ok(Value::Bool(value)),
+        Literal::None => Ok(Value::Null),
+        Literal::Integer(value) => {
+            serde_json::from_str(&value.to_string()).map_err(|_| Error::InvalidEntry)
+        }
+        Literal::Float(value) => serde_json::Number::from_f64(value)
+            .map(Value::Number)
+            .ok_or(Error::InvalidEntry),
+        Literal::List(values) | Literal::Tuple(values) => values
+            .into_iter()
+            .map(|value| literal_value(value, depth + 1))
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::Array),
+        Literal::Dict(entries) => entries
+            .into_iter()
+            .map(|(key, value)| {
+                let Literal::String(key) = key else {
+                    return Err(Error::InvalidEntry);
+                };
+                Ok((key, literal_value(value, depth + 1)?))
+            })
+            .collect::<Result<serde_json::Map<_, _>, _>>()
+            .map(Value::Object),
+        _ => Err(Error::InvalidEntry),
+    }
+}
+
+fn check_literal_depth(text: &str) -> Result<(), Error> {
+    let mut quote = None;
+    let mut escaped = false;
+    let mut depth = 0usize;
+    for ch in text.chars() {
+        if escaped {
+            escaped = false;
+            continue;
+        }
+        if let Some(delimiter) = quote {
+            if ch == '\\' {
+                escaped = true;
+            } else if ch == delimiter {
+                quote = None;
+            }
+            continue;
+        }
+        match ch {
+            '\'' | '"' => quote = Some(ch),
+            '[' | '{' | '(' => {
+                depth += 1;
+                if depth > 128 {
+                    return Err(Error::InvalidEntry);
+                }
+            }
+            ']' | '}' | ')' => depth = depth.saturating_sub(1),
+            _ => {}
+        }
+    }
+    Ok(())
+}

litellm-rust/crates/cache-response/src/embedding.rs

@@ -0,0 +1,22 @@
+use serde::Serialize;
+use serde_json::Value;
+
+#[derive(Clone, Debug, PartialEq, Serialize)]
+pub struct PartialHits {
+    pub values: Vec<Option<Value>>,
+    pub missing_indices: Vec<usize>,
+}
+
+impl PartialHits {
+    pub fn new(values: Vec<Option<Value>>) -> Self {
+        let missing_indices = values
+            .iter()
+            .enumerate()
+            .filter_map(|(index, value)| value.is_none().then_some(index))
+            .collect();
+        Self {
+            values,
+            missing_indices,
+        }
+    }
+}

litellm-rust/crates/cache-response/src/exact.rs

@@ -0,0 +1,148 @@
+use std::{future::Future, pin::Pin, time::Duration};
+
+use litellm_cache::{
+    BaseCache, BatchCache, CacheConnectionResult, Error, ExactCacheContext, FlushCache,
+};
+use serde_json::Value;
+
+use crate::{CacheEntry, PartialHits, ResponseCache, ResponseCacheRequest};
+
+type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
+
+/// Object-safe view of a `ResponseCache` over an exact-match backend, so hosts can hold every
+/// exact backend behind one pointer without erasing which backend it is elsewhere.
+pub trait ExactResponseCache: Send + Sync {
+    fn default_ttl(&self) -> Option<Duration>;
+
+    fn lookup(&self, request: &ResponseCacheRequest, now: Duration)
+    -> Result<Option<Value>, Error>;
+
+    fn store(
+        &self,
+        request: &ResponseCacheRequest,
+        response: Value,
+        now: Duration,
+    ) -> Result<(), Error>;
+
+    fn lookup_batch(
+        &self,
+        requests: &[ResponseCacheRequest],
+        now: Duration,
+    ) -> Result<PartialHits, Error>;
+
+    fn async_lookup<'a>(
+        &'a self,
+        request: &'a ResponseCacheRequest,
+        now: Duration,
+    ) -> BoxFuture<'a, Result<Option<Value>, Error>>;
+
+    fn async_store<'a>(
+        &'a self,
+        request: &'a ResponseCacheRequest,
+        response: Value,
+        now: Duration,
+    ) -> BoxFuture<'a, Result<(), Error>>;
+
+    fn async_lookup_batch<'a>(
+        &'a self,
+        requests: &'a [ResponseCacheRequest],
+        now: Duration,
+    ) -> BoxFuture<'a, Result<PartialHits, Error>>;
+
+    fn async_store_batch<'a>(
+        &'a self,
+        entries: Vec<(ResponseCacheRequest, Value)>,
+        now: Duration,
+    ) -> BoxFuture<'a, Result<(), Error>>;
+
+    fn async_store_entries<'a>(
+        &'a self,
+        entries: Vec<(ResponseCacheRequest, Value, Duration)>,
+    ) -> BoxFuture<'a, Result<(), Error>>;
+
+    fn async_flush<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>>;
+
+    fn test_connection<'a>(&'a self) -> BoxFuture<'a, Result<CacheConnectionResult, Error>>;
+}
+
+impl<B> ExactResponseCache for ResponseCache<B>
+where
+    B: BaseCache<Value = CacheEntry, Context = ExactCacheContext> + BatchCache + FlushCache,
+{
+    fn default_ttl(&self) -> Option<Duration> {
+        ResponseCache::default_ttl(self)
+    }
+
+    fn lookup(
+        &self,
+        request: &ResponseCacheRequest,
+        now: Duration,
+    ) -> Result<Option<Value>, Error> {
+        ResponseCache::lookup(self, request, now)
+    }
+
+    fn store(
+        &self,
+        request: &ResponseCacheRequest,
+        response: Value,
+        now: Duration,
+    ) -> Result<(), Error> {
+        ResponseCache::store(self, request, response, now)
+    }
+
+    fn lookup_batch(
+        &self,
+        requests: &[ResponseCacheRequest],
+        now: Duration,
+    ) -> Result<PartialHits, Error> {
+        ResponseCache::lookup_batch(self, requests, now)
+    }
+
+    fn async_lookup<'a>(
+        &'a self,
+        request: &'a ResponseCacheRequest,
+        now: Duration,
+    ) -> BoxFuture<'a, Result<Option<Value>, Error>> {
+        Box::pin(ResponseCache::async_lookup(self, request, now))
+    }
+
+    fn async_store<'a>(
+        &'a self,
+        request: &'a ResponseCacheRequest,
+        response: Value,
+        now: Duration,
+    ) -> BoxFuture<'a, Result<(), Error>> {
+        Box::pin(ResponseCache::async_store(self, request, response, now))
+    }
+
+    fn async_lookup_batch<'a>(
+        &'a self,
+        requests: &'a [ResponseCacheRequest],
+        now: Duration,
+    ) -> BoxFuture<'a, Result<PartialHits, Error>> {
+        Box::pin(ResponseCache::async_lookup_batch(self, requests, now))
+    }
+
+    fn async_store_batch<'a>(
+        &'a self,
+        entries: Vec<(ResponseCacheRequest, Value)>,
+        now: Duration,
+    ) -> BoxFuture<'a, Result<(), Error>> {
+        Box::pin(ResponseCache::async_store_batch(self, entries, now))
+    }
+
+    fn async_store_entries<'a>(
+        &'a self,
+        entries: Vec<(ResponseCacheRequest, Value, Duration)>,
+    ) -> BoxFuture<'a, Result<(), Error>> {
+        Box::pin(ResponseCache::async_store_entries(self, entries))
+    }
+
+    fn async_flush<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>> {
+        Box::pin(ResponseCache::async_flush(self))
+    }
+
+    fn test_connection<'a>(&'a self) -> BoxFuture<'a, Result<CacheConnectionResult, Error>> {
+        Box::pin(ResponseCache::test_connection(self))
+    }
+}

litellm-rust/crates/cache-response/src/lib.rs

@@ -0,0 +1,16 @@
+mod buffer;
+mod caching;
+mod codec;
+mod embedding;
+mod exact;
+mod response;
+
+pub use buffer::WriteBuffer;
+pub use caching::{
+    CacheControls, CacheEntry, CacheKeyContext, CacheKeyField, CacheKeyInput, CacheMode, cache_key,
+    get_cache_key, should_use_cache,
+};
+pub use codec::ResponseCacheCodec;
+pub use embedding::PartialHits;
+pub use exact::ExactResponseCache;
+pub use response::{ResponseCache, ResponseCacheRequest};

litellm-rust/crates/cache-response/src/response.rs

@@ -0,0 +1,302 @@
+use std::{sync::Arc, time::Duration};
+
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheConnectionResult, CacheContext, Error, FlushCache,
+};
+use serde_json::Value;
+
+use crate::{CacheControls, CacheEntry, CacheKeyInput, PartialHits, cache_key};
+
+#[derive(Clone)]
+pub struct ResponseCacheRequest<C: CacheContext = litellm_cache::ExactCacheContext> {
+    pub key: CacheKeyInput,
+    pub controls: CacheControls,
+    pub context: C,
+    pub max_age: Option<Duration>,
+}
+
+impl<C: CacheContext + Default> ResponseCacheRequest<C> {
+    pub fn new(key: CacheKeyInput) -> Self {
+        Self {
+            key,
+            controls: CacheControls {
+                configured: true,
+                supported_call_type: true,
+                native_backend: true,
+                default_on: true,
+                ..Default::default()
+            },
+            context: C::default(),
+            max_age: None,
+        }
+    }
+}
+
+impl<C: CacheContext> ResponseCacheRequest<C> {
+    pub fn with_context<D: CacheContext>(self, context: D) -> ResponseCacheRequest<D> {
+        ResponseCacheRequest {
+            key: self.key,
+            controls: self.controls,
+            context,
+            max_age: self.max_age,
+        }
+    }
+}
+
+pub struct ResponseCache<B: BaseCache<Value = CacheEntry>>
+where
+    B::Context: Default + PartialEq,
+{
+    backend: Arc<B>,
+}
+
+impl<B> ResponseCache<B>
+where
+    B: BaseCache<Value = CacheEntry>,
+    B::Context: Default + PartialEq,
+{
+    pub fn new(backend: Arc<B>) -> Self {
+        Self { backend }
+    }
+
+    pub fn backend(&self) -> &B {
+        &self.backend
+    }
+
+    pub fn backend_arc(&self) -> &Arc<B> {
+        &self.backend
+    }
+
+    pub fn default_ttl(&self) -> Option<Duration> {
+        self.backend.get_ttl(&B::Context::default())
+    }
+
+    pub async fn async_flush(&self) -> Result<(), Error>
+    where
+        B: FlushCache,
+    {
+        self.backend.async_flush_cache().await
+    }
+
+    pub async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        self.backend.test_connection().await
+    }
+
+    pub fn lookup(
+        &self,
+        request: &ResponseCacheRequest<B::Context>,
+        now: Duration,
+    ) -> Result<Option<Value>, Error> {
+        if !request.controls.reads() {
+            return Ok(None);
+        }
+        let entry = match self
+            .backend
+            .get_cache(&cache_key(&request.key), &request.context)
+        {
+            Ok(entry) => entry,
+            Err(Error::InvalidEntry) => None,
+            Err(error) => return Err(error),
+        };
+        Ok(Self::fresh_or_miss(entry, now, request.max_age))
+    }
+
+    pub async fn async_lookup(
+        &self,
+        request: &ResponseCacheRequest<B::Context>,
+        now: Duration,
+    ) -> Result<Option<Value>, Error> {
+        if !request.controls.reads() {
+            return Ok(None);
+        }
+        let entry = match self
+            .backend
+            .async_get_cache(&cache_key(&request.key), &request.context)
+            .await
+        {
+            Ok(entry) => entry,
+            Err(Error::InvalidEntry) => None,
+            Err(error) => return Err(error),
+        };
+        Ok(Self::fresh_or_miss(entry, now, request.max_age))
+    }
+
+    pub fn lookup_batch(
+        &self,
+        requests: &[ResponseCacheRequest<B::Context>],
+        now: Duration,
+    ) -> Result<PartialHits, Error>
+    where
+        B: BatchCache,
+    {
+        let readable = requests
+            .iter()
+            .enumerate()
+            .filter(|(_, request)| request.controls.reads())
+            .collect::<Vec<_>>();
+        let keys = readable
+            .iter()
+            .map(|(_, request)| cache_key(&request.key))
+            .collect::<Vec<_>>();
+        let entries = if let Some((_, request)) = readable.first() {
+            self.backend.batch_get_cache(&keys, &request.context)?
+        } else {
+            Vec::new()
+        };
+        Self::partial_hits(requests, readable, entries, now)
+    }
+
+    pub async fn async_lookup_batch(
+        &self,
+        requests: &[ResponseCacheRequest<B::Context>],
+        now: Duration,
+    ) -> Result<PartialHits, Error>
+    where
+        B: BatchCache,
+    {
+        let readable = requests
+            .iter()
+            .enumerate()
+            .filter(|(_, request)| request.controls.reads())
+            .collect::<Vec<_>>();
+        let keys = readable
+            .iter()
+            .map(|(_, request)| cache_key(&request.key))
+            .collect::<Vec<_>>();
+        let entries = if let Some((_, request)) = readable.first() {
+            self.backend
+                .async_batch_get_cache(keys, request.context.clone())
+                .await?
+        } else {
+            Vec::new()
+        };
+        Self::partial_hits(requests, readable, entries, now)
+    }
+
+    pub fn store(
+        &self,
+        request: &ResponseCacheRequest<B::Context>,
+        response: Value,
+        now: Duration,
+    ) -> Result<(), Error> {
+        if !request.controls.writes() {
+            return Ok(());
+        }
+        self.backend.set_cache(
+            &cache_key(&request.key),
+            CacheEntry {
+                timestamp: Some(now.as_secs_f64()),
+                response,
+            },
+            &request.context,
+        )
+    }
+
+    pub async fn async_store(
+        &self,
+        request: &ResponseCacheRequest<B::Context>,
+        response: Value,
+        now: Duration,
+    ) -> Result<(), Error> {
+        if !request.controls.writes() {
+            return Ok(());
+        }
+        self.backend
+            .async_set_cache(
+                &cache_key(&request.key),
+                CacheEntry {
+                    timestamp: Some(now.as_secs_f64()),
+                    response,
+                },
+                request.context.clone(),
+            )
+            .await
+    }
+
+    pub async fn async_store_batch(
+        &self,
+        entries: Vec<(ResponseCacheRequest<B::Context>, Value)>,
+        now: Duration,
+    ) -> Result<(), Error> {
+        self.async_store_entries(
+            entries
+                .into_iter()
+                .map(|(request, response)| (request, response, now))
+                .collect(),
+        )
+        .await
+    }
+
+    /// Stores entries that each carry the time they were produced, so a deferred write keeps
+    /// the freshness of its original response.
+    pub async fn async_store_entries(
+        &self,
+        entries: Vec<(ResponseCacheRequest<B::Context>, Value, Duration)>,
+    ) -> Result<(), Error> {
+        let writable = entries
+            .into_iter()
+            .filter(|(request, _, _)| request.controls.writes())
+            .map(|(request, response, now)| {
+                (
+                    cache_key(&request.key),
+                    CacheEntry {
+                        timestamp: Some(now.as_secs_f64()),
+                        response,
+                    },
+                    request.context,
+                )
+            })
+            .collect::<Vec<_>>();
+        let Some((_, _, first_kwargs)) = writable.first() else {
+            return Ok(());
+        };
+        if writable
+            .iter()
+            .all(|(_, _, context)| context == first_kwargs)
+        {
+            let context = first_kwargs.clone();
+            let cache_list = writable
+                .into_iter()
+                .map(|(key, entry, _)| (key, entry))
+                .collect();
+            return self
+                .backend
+                .async_set_cache_pipeline(cache_list, context)
+                .await;
+        }
+        for (key, entry, context) in writable {
+            self.backend.async_set_cache(&key, entry, context).await?;
+        }
+        Ok(())
+    }
+
+    fn partial_hits(
+        requests: &[ResponseCacheRequest<B::Context>],
+        readable: Vec<(usize, &ResponseCacheRequest<B::Context>)>,
+        entries: Vec<BatchEntry<CacheEntry>>,
+        now: Duration,
+    ) -> Result<PartialHits, Error> {
+        if readable.len() != entries.len() {
+            return Err(Error::Unavailable);
+        }
+        let mut values = vec![None; requests.len()];
+        for ((index, request), entry) in readable.into_iter().zip(entries) {
+            let response = match entry {
+                BatchEntry::Hit(entry) => Self::fresh_or_miss(Some(entry), now, request.max_age),
+                BatchEntry::Miss | BatchEntry::Invalid => None,
+            };
+            values[index] = response;
+        }
+        Ok(PartialHits::new(values))
+    }
+
+    fn fresh_or_miss(
+        entry: Option<CacheEntry>,
+        now: Duration,
+        max_age: Option<Duration>,
+    ) -> Option<Value> {
+        entry
+            .filter(|entry| entry.fresh(now, max_age))
+            .map(|entry| entry.response)
+    }
+}

litellm-rust/crates/cache-response/tests/caching.rs

@@ -0,0 +1,90 @@
+use litellm_cache_response::{
+    CacheControls, CacheKeyContext, CacheKeyField, CacheKeyInput, cache_key, get_cache_key,
+};
+use sha2::{Digest, Sha256};
+
+#[test]
+fn keys_match_python_order_groups_files_presets_and_namespaces() {
+    let mut input = CacheKeyInput {
+        fields: vec![
+            CacheKeyField {
+                name: "model".into(),
+                value: Some("deployment".into()),
+                api_parameter: true,
+                internal_parameter: false,
+            },
+            CacheKeyField {
+                name: "file".into(),
+                value: None,
+                api_parameter: true,
+                internal_parameter: false,
+            },
+        ],
+        namespace: Some("team".into()),
+        ..Default::default()
+    };
+    CacheKeyContext {
+        model_group: Some("group".into()),
+        caching_groups: vec![(vec!["group".into()], "['group']".into())],
+        file_checksum: Some("checksum".into()),
+        ..Default::default()
+    }
+    .apply(&mut input);
+    assert_eq!(
+        cache_key(&input),
+        format!(
+            "team:{:x}",
+            Sha256::digest(b"model: ['group']file: checksum")
+        )
+    );
+    input.preset = Some("preset".into());
+    assert_eq!(get_cache_key(&input), "preset");
+}
+
+#[test]
+fn cache_controls_honor_default_modes_and_directives() {
+    let enabled = CacheControls {
+        supported_call_type: true,
+        configured: true,
+        default_on: true,
+        ..Default::default()
+    };
+    assert!(enabled.reads());
+    assert!(enabled.writes());
+    assert!(
+        !CacheControls {
+            default_on: false,
+            ..enabled
+        }
+        .reads()
+    );
+    assert!(
+        CacheControls {
+            default_on: false,
+            use_cache: true,
+            ..enabled
+        }
+        .reads()
+    );
+    assert!(
+        !CacheControls {
+            no_cache: true,
+            ..enabled
+        }
+        .reads()
+    );
+    assert!(
+        !CacheControls {
+            no_store: true,
+            ..enabled
+        }
+        .writes()
+    );
+    assert!(
+        !CacheControls {
+            caching: Some(false),
+            ..enabled
+        }
+        .writes()
+    );
+}