Merge remote-tracking branch 'origin/main' into litellm_techdebt_20260906

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

# Conflicts:
#	litellm/router_strategy/complexity_router/jev_classifier.py
#	litellm/types/utils.py

.circleci/config.yml

@@ -6,6 +6,9 @@ parameters:
   migration_candidate_image:
     type: string
     default: ""
+  migration_baseline_image:
+    type: string
+    default: "ghcr.io/berriai/litellm-database:v1.102.0"
   migration_source_sha:
     type: string
     default: ""
@@ -1508,7 +1511,7 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver"
+            uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not legacy_resolver"
 
   installing_litellm_on_python_3_13:
     docker:
@@ -1532,7 +1535,7 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver"
+            uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not legacy_resolver"
 
   installing_litellm_on_python_v2_migration_resolver:
     docker:
@@ -1561,10 +1564,11 @@ jobs:
           url: tcp://localhost:5432
           timeout: "60"
       - run:
-          name: Run v2 migration resolver proxy smoke test
+          name: Run both migration resolvers against Postgres
           command: |
             uv run --no-sync python -m pytest -vv \
-              tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver
+              tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings \
+              tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_legacy_resolver
 
   helm_chart_testing:
     machine:
@@ -2945,7 +2949,10 @@ jobs:
     parameters:
       suite:
         type: enum
-        enum: [startup, recovery, legacy]
+        enum: [startup, recovery, legacy, upgrade, shaped]
+      baseline:
+        type: boolean
+        default: false
     machine:
       image: ubuntu-2204:2024.04.1
     resource_class: large
@@ -2953,6 +2960,7 @@ jobs:
     environment:
       LITELLM_MIGRATION_TESTS: "1"
       LITELLM_MIGRATION_TEST_IMAGE: litellm-docker-database:ci
+      LITELLM_MIGRATION_BASELINE_IMAGE: << pipeline.parameters.migration_baseline_image >>
       MIGRATION_TEST_ADMIN_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres
       MIGRATION_TEST_CONTAINER_ADMIN_URL: postgresql://postgres:postgres@host.docker.internal:5432/postgres
       MIGRATION_TEST_OUTPUT: /tmp/migration-results
@@ -2980,6 +2988,16 @@ jobs:
       - wait_for_service:
           url: tcp://localhost:5432
           timeout: "60"
+      - when:
+          condition: << parameters.baseline >>
+          steps:
+            - run:
+                name: Pull the baseline release the upgrade starts from
+                environment:
+                  BASELINE_IMAGE: << pipeline.parameters.migration_baseline_image >>
+                command: |
+                  [[ "$BASELINE_IMAGE" =~ ^ghcr.io/berriai/[a-z0-9._/-]+(@sha256:[0-9a-f]{64}|:v[0-9][0-9a-z.-]*)$ ]] || exit 1
+                  docker pull "$BASELINE_IMAGE"
       - run:
           name: Run migration startup regressions
           environment:
@@ -3032,28 +3050,29 @@ jobs:
       - run:
           name: Run Docker container with bad DATABASE_URL
           command: |
+            set +e
             docker run --name my-app \
               -p 4000:4000 \
               -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
               -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
               -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
               myapp:latest \
-              --port 4000 > docker_output.log 2>&1 || true
+              --port 4000 > docker_output.log 2>&1
+            echo "$?" > docker_exit_code
+            set -e
       - run:
           name: Display Docker logs
           command: cat docker_output.log
       - run:
-          name: Check for expected error
+          name: Proxy must refuse to serve on an unreachable database
           command: |
-            if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \
-               (grep -q "Database setup failed after multiple retries" docker_output.log || \
-                grep -q "ERROR:    Application startup failed. Exiting." docker_output.log); then
-              echo "Expected error found. Test passed."
-            else
-              echo "Expected error not found. Test failed."
-              cat docker_output.log
-              exit 1
-            fi
+            fail() { echo "FAILED: $1"; cat docker_output.log; exit 1; }
+            exit_code="$(cat docker_exit_code)"
+            [ "$exit_code" -ne 0 ] || fail "proxy exited 0 with an unreachable database"
+            grep -q "P1001" docker_output.log || fail "log does not name the unreachable database server"
+            ! grep -q "Application startup complete" docker_output.log || fail "proxy reached serving state"
+            ! docker exec my-app true 2>/dev/null || fail "container is still running"
+            echo "Proxy refused to serve (exit $exit_code) and never reached startup. Test passed."
 
   provider_replay_harness:
     docker:
@@ -3142,6 +3161,33 @@ jobs:
       - store_artifacts:
           path: test-results
 
+  unit:
+    machine:
+      image: ubuntu-2204:2024.04.1
+    resource_class: large
+    working_directory: ~/project
+    steps:
+      - setup_litellm_test_deps
+      - run:
+          name: Generate Prisma client
+          command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+      - run:
+          name: Run unit tests
+          command: |
+            mkdir -p test-results/unit
+            mapfile -t files < <(find tests/unit -name 'test_*.py' | sort)
+            if [ "${#files[@]}" -eq 0 ]; then echo "tests/unit holds no test_*.py files; nothing to run"; exit 0; fi
+            set +e
+            LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short --junitxml=test-results/unit/junit.xml
+            status=$?
+            set -e
+            if [ "$status" -eq 5 ]; then echo "pytest collected no tests from tests/unit; passing"; exit 0; fi
+            exit "$status"
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test-results
+
 workflows:
   migration_startup:
     when: << pipeline.parameters.run_migration_tests >>
@@ -3160,6 +3206,16 @@ workflows:
           name: migration-legacy-and-pooling
           suite: legacy
           requires: [build_docker_database_image]
+      - migration_startup_tests:
+          name: migration-upgrade
+          suite: upgrade
+          baseline: true
+          requires: [build_docker_database_image]
+      - migration_startup_tests:
+          name: migration-upgrade-shaped
+          suite: shaped
+          baseline: true
+          requires: [build_docker_database_image]
   migration_startup_scheduled:
     triggers:
       - schedule:
@@ -3190,6 +3246,8 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - unit:
+          filters: *main_branches
       - provider_replay_harness
       - base_sdk_install:
           filters: *main_branches

.circleci/scripts/run_integration.sh

@@ -121,6 +121,10 @@ start_proxy() {
       "LITELLM_MODEL_COST_MAP_URL=$INTEGRATION_UPSTREAM_URL/_cost_map"
       "MODEL_COST_MAP_MIN_MODEL_COUNT=1"
       "MODEL_COST_MAP_MAX_SHRINK_RATIO=0"
+      "GEMINI_API_BASE=$INTEGRATION_UPSTREAM_URL"
+      "ANTHROPIC_API_BASE=$INTEGRATION_UPSTREAM_URL"
+      "GEMINI_API_KEY=sk-scripted-provider"
+      "ANTHROPIC_API_KEY=sk-scripted-provider"
     )
   else
     cost_map_env=("LITELLM_LOCAL_MODEL_COST_MAP=True")

.circleci/scripts/run_migration_tests.py

@@ -13,6 +13,8 @@ SUITES: Final = {
     "startup": (("test_startup.py",), 12),
     "recovery": (("test_recovery.py",), 15),
     "legacy": (("test_legacy.py", "test_pooling.py"), 11),
+    "upgrade": (("test_upgrade.py", "test_rolling_upgrade.py"), 5),
+    "shaped": (("test_shaped_database.py",), 1),
 }
 
 
@@ -93,6 +95,7 @@ def main() -> int:
             {
                 **metadata,
                 "suite": suite,
+                "baseline_image": os.environ.get("LITELLM_MIGRATION_BASELINE_IMAGE", ""),
                 "expected_cases": expected,
                 "passed": passed,
                 "pytest_exit_code": result.returncode,

.github/actions/cache-cargo-build/action.yml

@@ -15,6 +15,12 @@ description: >-
   cache the same directory for different workloads, and a shared key would let
   whichever ran first deny the others a save.
 
+inputs:
+  profile:
+    description: "Cargo profile the build uses (dev or release)"
+    required: false
+    default: "dev"
+
 runs:
   using: composite
   steps:
@@ -25,6 +31,6 @@ runs:
           ~/.cargo/registry
           ~/.cargo/git
           litellm-rust/target
-        key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }}
+        key: ${{ runner.os }}-maturin-${{ inputs.profile }}-${{ hashFiles('litellm-rust/Cargo.lock') }}
         restore-keys: |
-          ${{ runner.os }}-maturin-dev-
+          ${{ runner.os }}-maturin-${{ inputs.profile }}-

.github/e2e-stack/redact_output.py

@@ -0,0 +1,83 @@
+import argparse
+import os
+import sys
+from functools import reduce
+from pathlib import Path
+from typing import Final
+from xml.sax.saxutils import escape
+
+from pydantic import JsonValue, TypeAdapter, ValidationError
+from secrets_to_env import MIN_MASKED_LENGTH
+
+REDACTED: Final = "***"
+json_adapter: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
+
+
+def string_leaves(node: JsonValue) -> tuple[str, ...]:
+    match node:
+        case str():
+            return (node,)
+        case list():
+            return tuple(leaf for child in node for leaf in string_leaves(child))
+        case dict():
+            return tuple(leaf for child in node.values() for leaf in string_leaves(child))
+    return ()
+
+
+def field_lines(value: str) -> tuple[str, ...]:
+    try:
+        return tuple(line for leaf in string_leaves(json_adapter.validate_json(value)) for line in leaf.splitlines())
+    except ValidationError:
+        return ()
+
+
+def masked_values(values_files: tuple[Path, ...]) -> tuple[str, ...]:
+    values: Final = frozenset(
+        line.split("=", 1)[1].strip().strip("'")
+        for path in values_files
+        for line in path.read_text().splitlines()
+        if "=" in line
+    )
+    texts: Final = frozenset(text for value in values for text in (value, *field_lines(value)))
+    renderings: Final = frozenset(
+        rendering
+        for text in texts
+        if len(text) >= MIN_MASKED_LENGTH
+        for rendering in (text, escape(text), escape(text, {'"': """}))
+    )
+    return tuple(sorted(renderings, key=lambda rendering: (-len(rendering), rendering)))
+
+
+def redact(text: str, values: tuple[str, ...]) -> str:
+    return reduce(lambda redacted, value: redacted.replace(value, REDACTED), values, text)
+
+
+def write_redacted(source: Path, out_dir: Path, values: tuple[str, ...]) -> None:
+    target: Final = out_dir / source.name
+    with os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "w") as handle:
+        _ = handle.write(redact(source.read_text(errors="replace"), values))
+
+
+def main() -> int:
+    parser: Final = argparse.ArgumentParser()
+    _ = parser.add_argument("--values", action="append", type=Path, required=True)
+    _ = parser.add_argument("--out", type=Path, required=True)
+    _ = parser.add_argument("files", nargs="*", type=Path)
+    args: Final = parser.parse_args()
+    values_files: Final = tuple(args.values)
+    out_dir: Final[Path] = args.out
+    sources: Final = tuple(args.files)
+    try:
+        values: Final = masked_values(values_files)
+        out_dir.mkdir(mode=0o700, exist_ok=True)
+        for source in sources:
+            write_redacted(source, out_dir, values)
+    except OSError as error:
+        _ = sys.stderr.write(f"could not redact {error.filename}\n")
+        return 1
+    _ = sys.stdout.write(f"redacted {len(sources)} file(s) into {out_dir}\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/secrets_to_env.py

@@ -9,6 +9,7 @@ from pydantic import TypeAdapter, ValidationError
 secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
 ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
 MIN_MASKED_LENGTH: Final = 8
+ACTIONS_RUNNER_FLAG: Final = "GITHUB_ACTIONS"
 
 
 def main() -> int:
@@ -30,10 +31,15 @@ def main() -> int:
             f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n"
         )
         return 1
-    for value in secrets.values():
-        if len(value) >= MIN_MASKED_LENGTH:
-            _ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n")
-    sys.stdout.flush()
+    if os.environ.get(ACTIONS_RUNNER_FLAG) == "true":
+        _ = sys.stdout.write(
+            "".join(
+                f"::add-mask::{value.replace('%', '%25')}\n"
+                for value in secrets.values()
+                if len(value) >= MIN_MASKED_LENGTH
+            )
+        )
+        sys.stdout.flush()
     lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
     try:
         with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle:

.github/e2e-stack/select_tests.py

@@ -9,6 +9,7 @@ UNSUPPORTED: Final = re.compile(
     r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
     r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
     r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
+    r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$"
 )
 HARNESS: Final = re.compile(
     r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"

.github/e2e-stack/up.sh

@@ -143,7 +143,7 @@ env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/m
 
 start_server() {
   local name="$1"; shift
-  env "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
+  env -u AWS_ROLE_NAME "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
   echo $! > "${PIDS_DIR}/${name}.pid"
 }
 

.github/scripts/verify_linux_native_wheel.py

@@ -134,7 +134,16 @@ def main(
         uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members)
         native_path: Final = wheel.parent / "native" / Path(native_member.filename).name
         native_path.parent.mkdir(parents=True, exist_ok=True)
-        native_path.write_bytes(archive.read(native_member))
+        native_bytes: Final = archive.read(native_member)
+        native_path.write_bytes(native_bytes)
+        duplicated_vocabularies: Final = tuple(
+            member.filename
+            for member in wheel_members
+            if member.filename.startswith("litellm/litellm_core_utils/tokenizers/")
+            and re.fullmatch(r"[0-9a-f]{40}", PurePosixPath(member.filename).name)
+            and member.file_size > 0
+            and archive.read(member) in native_bytes
+        )
 
     wheel_metadata_tags_match: Final = (
         len(wheel_metadata_tags) == len(expanded_filename_tags)
@@ -205,7 +214,7 @@ def main(
     native_module: Final = load_native_module(native_path)
     native_module_loads: Final = native_module is not None
     panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
-    native_size_limit: Final = 25_000_000
+    native_size_limit: Final = 35_000_000
     native_size_within_limit: Final = native_member.file_size <= native_size_limit
     validations: Final = (
         (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
@@ -222,7 +231,8 @@ def main(
         ("Python extension entry point is present", extension_entry_point_present),
         ("Native module loads", native_module_loads),
         ("Production module omits the panic test hook", panic_test_hook_absent),
-        ("Native extension does not exceed 25 MB", native_size_within_limit),
+        (f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit),
+        ("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies),
         ("Wheel contents are valid", not unexpected_members),
     )
 
@@ -267,7 +277,8 @@ def main(
             ),
             (
                 not native_size_within_limit,
-                f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB",
+                f"native extension exceeds {native_size_limit / 1_000_000:.0f} MB: "
+                f"{native_member.file_size / 1_000_000:.2f} MB",
             ),
             (bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"),
         )

.github/workflows/_test-unit-base.yml

@@ -165,33 +165,41 @@ jobs:
           DIST: ${{ inputs.dist }}
           COVERAGE_CORE: sysmon
         run: |
-          if [ "${WORKERS}" = "0" ]; then
-            uv run --no-sync pytest ${TEST_PATH:?} \
-              --tb=short -vv \
-              --maxfail="${MAX_FAILURES}" \
-              --reruns "${RERUNS}" \
-              --reruns-delay 1 \
-              --timeout="${TEST_TIMEOUT_SECONDS}" \
-              --rerun-except "from pytest-timeout" \
-              --durations=20 \
-              --cov=./litellm --cov=./enterprise/litellm_enterprise \
-              --cov-report=xml:coverage.xml \
-              --cov-config=pyproject.toml
-          else
-            uv run --no-sync pytest ${TEST_PATH:?} \
-              --tb=short -vv \
-              --maxfail="${MAX_FAILURES}" \
-              -n "${WORKERS}" \
-              --reruns "${RERUNS}" \
-              --reruns-delay 1 \
-              --timeout="${TEST_TIMEOUT_SECONDS}" \
-              --rerun-except "from pytest-timeout" \
-              --dist="${DIST}" \
-              --durations=20 \
-              --cov=./litellm --cov=./enterprise/litellm_enterprise \
-              --cov-report=xml:coverage.xml \
-              --cov-config=pyproject.toml
+          found_path=false
+          for path in ${TEST_PATH}; do
+            if [ -e "${path%%::*}" ]; then
+              found_path=true
+              break
+            fi
+          done
+          if [ "$found_path" = false ]; then
+            echo "No path in TEST_PATH exists (${TEST_PATH}); nothing to run"
+            exit 0
           fi
+          xdist_args=()
+          if [ "${WORKERS}" != "0" ]; then
+            xdist_args=(-n "${WORKERS}" --dist="${DIST}")
+          fi
+          set +e
+          uv run --no-sync pytest ${TEST_PATH:?} \
+            --tb=short -vv \
+            --maxfail="${MAX_FAILURES}" \
+            "${xdist_args[@]}" \
+            --reruns "${RERUNS}" \
+            --reruns-delay 1 \
+            --timeout="${TEST_TIMEOUT_SECONDS}" \
+            --rerun-except "from pytest-timeout" \
+            --durations=20 \
+            --cov=./litellm --cov=./enterprise/litellm_enterprise \
+            --cov-report=xml:coverage.xml \
+            --cov-config=pyproject.toml
+          status=$?
+          set -e
+          if [ "$status" -eq 5 ]; then
+            echo "pytest collected no tests from ${TEST_PATH}; passing"
+            exit 0
+          fi
+          exit "$status"
 
       - name: Save coverage report
         if: always() && steps.changes.outputs.decision != 'skip'

.github/workflows/codspeed.yml

@@ -13,6 +13,7 @@ on:
       - ".github/workflows/codspeed.yml"
       - ".github/actions/setup-uv-with-retries/**"
       - ".github/actions/cache-cargo-build/**"
+      - ".github/scripts/uv_sync_with_retries.sh"
   pull_request:
     branches:
       - main
@@ -25,6 +26,7 @@ on:
       - ".github/workflows/codspeed.yml"
       - ".github/actions/setup-uv-with-retries/**"
       - ".github/actions/cache-cargo-build/**"
+      - ".github/scripts/uv_sync_with_retries.sh"
   # Allow CodSpeed to trigger backtest performance analysis
   # in order to generate initial data
   workflow_dispatch:
@@ -59,19 +61,27 @@ jobs:
 
       - name: Cache the Rust build
         uses: ./.github/actions/cache-cargo-build
+        with:
+          profile: release
 
       # Build the wheel and resolve every dependency outside the CodSpeed
       # runner: the same maturin build took 42 minutes inside `codspeed run`
       # versus under 3 minutes as a plain step (LIT-6183)
-      - name: Build environment
+      - name: Build the release wheel
+        run: uv build --wheel --out-dir dist
+
+      - name: Install the wheel into the benchmark environment
+        run: |
+          UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/benchmark-venv" .github/scripts/uv_sync_with_retries.sh --frozen --no-default-groups --group benchmarks --no-install-project --python 3.12
+          uv pip install --python "${RUNNER_TEMP}/benchmark-venv/bin/python" --no-deps dist/*.whl
+
+      - name: Collect benchmarks
+        env:
+          PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
+          LITELLM_REQUIRE_INSTALLED_WHEEL: "1"
         run: >
-          env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
-          uv run --frozen --no-default-groups
-          --with pytest==8.3.5
-          --with pytest-codspeed==4.3.0
-          --with "mcp>=2.2.0,<3.0"
-          --with "a2a-sdk>=1.1.0,<2.0"
-          pytest
+          "${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
+          --import-mode=importlib
           -p pytest_codspeed.plugin
           tests/benchmarks/
           --codspeed
@@ -82,13 +92,9 @@ jobs:
         with:
           mode: simulation
           run: >
-            env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
-            uv run --frozen --no-default-groups
-            --with pytest==8.3.5
-            --with pytest-codspeed==4.3.0
-            --with "mcp>=2.2.0,<3.0"
-            --with "a2a-sdk>=1.1.0,<2.0"
-            pytest
+            env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 LITELLM_REQUIRE_INSTALLED_WHEEL=1
+            "${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
+            --import-mode=importlib
             -p pytest_codspeed.plugin
             tests/benchmarks/
             --codspeed

.github/workflows/test-e2e-changed.yml

@@ -175,6 +175,8 @@ jobs:
         env:
           TESTS: ${{ needs.detect.outputs.tests }}
           E2E_FIXTURE_MODE: live
+          E2E_PROVIDER_EDGE_HOST_REACHABLE: '1'
+          COLUMNS: '400'
         run: |
           umask 077
           read -r -a test_files <<< "${TESTS}"
@@ -189,6 +191,7 @@ jobs:
             uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"
             verified=$?
             set -e
+            grep -E '^(FAILED|ERROR) ' "${log}" || true
             grep -E '^=+ .* in [0-9.]+s( \([0-9:]+\))? =+$' "${log}" | tail -n 1
             echo "::endgroup::"
             if [ "${status}" = "5" ]; then
@@ -206,6 +209,24 @@ jobs:
             echo "pass ${pass} of 3 passed"
           done
 
+      - name: Redact the pytest output
+        if: always() && steps.boot.outcome == 'success'
+        run: |
+          umask 077
+          shopt -s nullglob
+          uv run --no-sync python .github/e2e-stack/redact_output.py \
+            --values tests/e2e/.env --values "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" \
+            --out "${RUNNER_TEMP}/e2e-redacted" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
+
+      - name: Keep the redacted pytest output
+        if: always() && steps.boot.outcome == 'success'
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: e2e-changed-pytest-output-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/e2e-redacted
+          retention-days: 14
+          if-no-files-found: ignore
+
       - name: Stop the stack
         if: always() && steps.boot.outcome != 'skipped'
         run: bash .github/e2e-stack/down.sh
@@ -214,7 +235,7 @@ jobs:
         if: always()
         run: |
           rm -f tests/e2e/.env "${RUNNER_TEMP}/e2e-boot.log" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
-          rm -rf "${RUNNER_TEMP}/litellm-e2e-stack"
+          rm -rf "${RUNNER_TEMP}/litellm-e2e-stack" "${RUNNER_TEMP}/e2e-redacted"
 
   gate:
     name: e2e-changed-tests

.github/workflows/test-linting.yml

@@ -130,6 +130,10 @@ jobs:
           echo "File content around line 43:"
           head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
 
+      - name: Check MCP operation boundary
+        if: steps.changes.outputs.decision != 'skip'
+        run: uv run --no-sync python scripts/check_mcp_operation_boundary.py
+
       - name: Run Ruff linting
         if: steps.changes.outputs.decision != 'skip'
         run: |

.github/workflows/test-rust.yml

@@ -120,6 +120,20 @@ jobs:
 
       - run: cargo test --workspace --doc --locked
 
+      - name: Test token counter feature combinations
+        run: |
+          for features in '' fast huggingface tiktoken fast,huggingface fast,tiktoken huggingface,tiktoken fast,huggingface,tiktoken; do
+            cargo test -p litellm-token-counter --locked --no-default-features --features "$features"
+            cargo check -p litellm-python-bridge --locked --no-default-features --features "abi3${features:+,$features}"
+          done
+
+      - name: Test secret manager feature combinations
+        run: |
+          cargo test -p litellm-auth-gcp --locked --no-default-features
+          for features in '' aws google hashicorp azure cyberark aws,google aws,azure google,azure aws,google,azure aws,google,cyberark aws,google,azure,cyberark aws,google,hashicorp,azure,cyberark; do
+            cargo test -p litellm-secrets --locked --no-default-features --features "$features"
+          done
+
   rust-wheel:
     runs-on: ubuntu-latest
     timeout-minutes: 30

.github/workflows/test-unit.yml

@@ -51,7 +51,7 @@ jobs:
         include:
           - shard: mcp-integration
             artifact-name: mcp-integration
-            test-path: "tests/mcp_tests"
+            test-path: "tests/mcp_tests tests/test_litellm/experimental_mcp_client"
             workers: 2
             reruns: 0
             timeout-minutes: 20
@@ -113,7 +113,6 @@ jobs:
               tests/test_litellm/compression
               tests/test_litellm/containers
               tests/test_litellm/endpoints
-              tests/test_litellm/experimental_mcp_client
               tests/test_litellm/models
               tests/test_litellm/repositories
               tests/test_litellm/images

Makefile

@@ -164,6 +164,7 @@ lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
 
 # Linting targets
 lint-ruff: $(LINT_DEP_INSTALL)
+	$(UV_RUN) python scripts/check_mcp_operation_boundary.py
 	cd litellm && $(UV_RUN) ruff check . && cd ..
 	$(UV_RUN) ruff check --config ruff-tests.toml tests
 

README.md

@@ -307,6 +307,7 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse
 | [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ |  |  | ✅ |  |  |  |  |
 | [DeepInfra (`deepinfra`)](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |
 | [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ |  |  |  |
 | [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ |  | ✅ |  |  |  |  |  |
@@ -356,7 +357,7 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse
 | [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Pinstripes (`pinstripes`)](https://docs.litellm.ai/docs/providers/pinstripes) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Predibase (`predibase`)](https://docs.litellm.ai/docs/providers/predibase) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
-| [Qwen AI Platform (`qwen_ai_platform`)](https://docs.litellm.ai/docs/providers/qwencloud) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  | ✅ |
+| [Qianwen AI Platform (`qwen_ai_platform`)](https://docs.litellm.ai/docs/providers/qwencloud) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  | ✅ |
 | [QwenCloud (`qwencloud`)](https://docs.litellm.ai/docs/providers/qwencloud) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  | ✅ |
 | [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) |  |  |  |  | ✅ |  |  |  |  |  |
 | [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |

backend/Dockerfile

@@ -61,6 +61,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra saml \
         --python python3.13
 
+RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
+
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
     npm_config_cache=/root/.npm \
     prisma generate --schema=./schema.prisma

backend/routes/allowlist.py

@@ -51,6 +51,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/cache_settings",
     "/coordination_redis/",
     "/cost_tracking",
+    "/cost_optimization/",
     "/cost/",
     "/credentials",
     "/credential",

ci_cd/cost_map_guard.py

@@ -1,8 +1,11 @@
 """Guard the cost map on pull requests.
 
-Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file,
-and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named
-litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models.
+Every pull request whose diff against its merge base touches one of the three cost map files gets the file
+checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the
+map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies
+and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot
+(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three
+files and may only add or update models.
 """
 
 from __future__ import annotations
@@ -108,20 +111,37 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
     )
 
 
+def touches_cost_map(changed_files: Sequence[str]) -> bool:
+    return any(path in GUARDED_PATHS for path in changed_files)
+
+
+def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
+    if bot:
+        return "bot contract enforced"
+    return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
+
+
 def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]:
+    if not bot and not touches_cost_map(changed_files):
+        return ()
     head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH)
     if isinstance(head_map, str):
         return (head_map,)
     return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ()))
 
 
-def _git(*args: str) -> str:
+def _git(*args: str) -> str | None:
     result: Final = subprocess.run(("git", *args), check=False, capture_output=True, text=True)
-    return result.stdout if result.returncode == 0 else ""
+    return result.stdout if result.returncode == 0 else None
 
 
 def snapshot(revision: str) -> Snapshot:
-    return Snapshot(*(_git("show", f"{revision}:{path}") for path in GUARDED_PATHS))
+    return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS))
+
+
+def changed_files(base: str, head: str) -> tuple[str, ...] | None:
+    diff: Final = _git("diff", "--name-only", "--no-renames", base, head)
+    return None if diff is None else tuple(diff.splitlines())
 
 
 def main(argv: Sequence[str]) -> int:
@@ -131,9 +151,12 @@ def main(argv: Sequence[str]) -> int:
     parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
     args: Final = parser.parse_args(argv)
     bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
-    changed_files: Final = tuple(_git("diff", "--name-only", args.base, args.head).splitlines())
-    failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed_files, bot)
-    contract: Final = "bot contract enforced" if bot else "human PR, file checks only"
+    changed: Final = changed_files(args.base, args.head)
+    if changed is None:
+        print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown")
+        return 1
+    failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot)
+    contract: Final = contract_for(bot, changed)
     if failures:
         print(f"cost map guard failed ({contract}):")
         print("\n".join(f"- {failure}" for failure in failures))

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json

@@ -6267,6 +6267,63 @@
       ],
       "title": "Spend update queue sizes (litellm_<queue>_size)",
       "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "description": "Requests that carried usage but were logged at $0 on a model whose pricing entry has a non-zero rate, by requested model and reason",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineWidth": 1,
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 430
+      },
+      "id": 110,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(litellm_zero_cost_requests_total[$__rate_interval])) by (requested_model, reason)",
+          "legendFormat": "{{requested_model}} / {{reason}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "litellm_zero_cost_requests rate",
+      "type": "timeseries"
     }
   ],
   "preload": false,

docker/README.md

@@ -2,6 +2,17 @@
 
 This guide provides instructions for building and running the LiteLLM application using Docker and Docker Compose.
 
+> **Just want to run LiteLLM?** This guide builds from source. To run the published
+> image instead, use `docker-compose.quickstart.yml` in this directory — the
+> two-service stack (gateway + Postgres) that the
+> [Docker quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start) documents:
+>
+> ```bash
+> curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml
+> printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env
+> docker compose -f docker-compose.quickstart.yml up -d
+> ```
+
 ## Prerequisites
 
 - Docker

docker/docker-compose.quickstart.yml

@@ -0,0 +1,41 @@
+# LiteLLM quickstart stack: the gateway plus a Postgres database that stores
+# models, virtual keys, and spend logs. Used by
+# https://docs.litellm.ai/docs/proxy/docker_quick_start
+#
+#   curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml
+#   printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env
+#   docker compose -f docker-compose.quickstart.yml up -d
+#
+# Compose reads .env from this directory. Keep it: regenerating LITELLM_SALT_KEY
+# makes credentials already stored in the database unreadable. For anything
+# beyond local evaluation, pin the image to a specific release tag.
+services:
+  litellm:
+    image: docker.litellm.ai/berriai/litellm:main-stable
+    ports:
+      - "4000:4000"
+    environment:
+      LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?set it in .env - see the header of this file}
+      LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?set it in .env - see the header of this file}
+      DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
+      STORE_MODEL_IN_DB: "True"
+    depends_on:
+      db:
+        condition: service_healthy
+
+  db:
+    image: postgres:16
+    environment:
+      POSTGRES_USER: litellm
+      POSTGRES_PASSWORD: litellm
+      POSTGRES_DB: litellm
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U litellm"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+
+volumes:
+  postgres_data:

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -2,10 +2,11 @@
 Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if the cost has been tracked.
 """
 
+from collections.abc import Sequence
 from dataclasses import replace as dataclasses_replace
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, List, Literal, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tuple, cast
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -18,8 +19,8 @@ if TYPE_CHECKING:
     from prisma import models as prisma_models
 
     from litellm.integrations.prometheus import PrometheusLogger
-    from litellm.proxy._types import LiteLLM_ManagedObjectTable
     from litellm.proxy.utils import PrismaClient, ProxyLogging
+    from litellm.repositories.prisma_protocols import TableActions
     from litellm.router import Router
     from litellm.types.router import Deployment
     from litellm.types.utils import LiteLLMBatch
@@ -41,6 +42,42 @@ TERMINAL_MANAGED_OBJECT_STATUSES: Final[Tuple[str, ...]] = (
 )
 
 
+class _ManagedObjectRow(Protocol):
+    @property
+    def id(self) -> str: ...
+
+    @property
+    def unified_object_id(self) -> str: ...
+
+    @property
+    def created_by(self) -> str | None: ...
+
+    @property
+    def file_object(self) -> object: ...
+
+
+def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]":
+    table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable
+    return table
+
+
+def _user_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_UserTable]":
+    table: Final[TableActions[prisma_models.LiteLLM_UserTable]] = prisma_client.db.litellm_usertable
+    return table
+
+
+def _token_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_VerificationToken]":
+    table: Final[TableActions[prisma_models.LiteLLM_VerificationToken]] = (
+        prisma_client.db.litellm_verificationtoken
+    )
+    return table
+
+
+def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_TeamTable]":
+    table: Final[TableActions[prisma_models.LiteLLM_TeamTable]] = prisma_client.db.litellm_teamtable
+    return table
+
+
 class CheckBatchCost:
     def __init__(
         self,
@@ -73,7 +110,7 @@ class CheckBatchCost:
         inline for a batch the first poll cycle then accounts again.
         """
         try:
-            await self.prisma_client.db.litellm_managedobjecttable.find_first(
+            await _managed_object_table(self.prisma_client).find_first(
                 where={"file_purpose": "batch", "batch_processed": False}
             )
         except Exception as probe_err:
@@ -97,10 +134,8 @@ class CheckBatchCost:
         if not user_id:
             return {}
         try:
-            user_row: prisma_models.LiteLLM_UserTable | None = (
-                await self.prisma_client.db.litellm_usertable.find_unique(
-                    where={"user_id": user_id}
-                )
+            user_row: prisma_models.LiteLLM_UserTable | None = await _user_table(self.prisma_client).find_unique(
+                where={"user_id": user_id}
             )
             if user_row is None:
                 return {}
@@ -117,11 +152,9 @@ class CheckBatchCost:
         if not api_key:
             return None
         try:
-            key_row: prisma_models.LiteLLM_VerificationToken | None = (
-                await self.prisma_client.db.litellm_verificationtoken.find_unique(
-                    where={"token": api_key}
-                )
-            )
+            key_row: prisma_models.LiteLLM_VerificationToken | None = await _token_table(
+                self.prisma_client
+            ).find_unique(where={"token": api_key})
             return getattr(key_row, "key_alias", None) if key_row is not None else None
         except Exception as e:
             verbose_proxy_logger.error(f"CheckBatchCost: could not look up key alias for batch {batch_id}: {e}")
@@ -132,17 +165,15 @@ class CheckBatchCost:
         if not team_id:
             return None
         try:
-            team_row: prisma_models.LiteLLM_TeamTable | None = (
-                await self.prisma_client.db.litellm_teamtable.find_unique(
-                    where={"team_id": team_id}
-                )
+            team_row: prisma_models.LiteLLM_TeamTable | None = await _team_table(self.prisma_client).find_unique(
+                where={"team_id": team_id}
             )
             return getattr(team_row, "team_alias", None) if team_row is not None else None
         except Exception as e:
             verbose_proxy_logger.error(f"CheckBatchCost: could not look up team alias for team {team_id}: {e}")
             return None
 
-    async def _get_org_id(self, job: "LiteLLM_ManagedObjectTable", batch_id: str) -> str | None:
+    async def _get_org_id(self, job: "_ManagedObjectRow", batch_id: str) -> str | None:
         org_id = getattr(job, "org_id", None)
         if org_id:
             return org_id
@@ -150,11 +181,9 @@ class CheckBatchCost:
         team_id = getattr(job, "team_id", None)
         if api_key:
             try:
-                key_row: prisma_models.LiteLLM_VerificationToken | None = (
-                    await self.prisma_client.db.litellm_verificationtoken.find_unique(
-                        where={"token": api_key}
-                    )
-                )
+                key_row: prisma_models.LiteLLM_VerificationToken | None = await _token_table(
+                    self.prisma_client
+                ).find_unique(where={"token": api_key})
                 key_org_id = getattr(key_row, "organization_id", None) if key_row is not None else None
                 if key_org_id:
                     return key_org_id
@@ -166,10 +195,8 @@ class CheckBatchCost:
         if not team_id:
             return None
         try:
-            team_row: prisma_models.LiteLLM_TeamTable | None = (
-                await self.prisma_client.db.litellm_teamtable.find_unique(
-                    where={"team_id": team_id}
-                )
+            team_row: prisma_models.LiteLLM_TeamTable | None = await _team_table(self.prisma_client).find_unique(
+                where={"team_id": team_id}
             )
             return getattr(team_row, "organization_id", None) if team_row is not None else None
         except Exception as e:
@@ -177,7 +204,7 @@ class CheckBatchCost:
             return None
 
     async def _build_creator_attribution_metadata(
-        self, job: "LiteLLM_ManagedObjectTable", batch_id: str
+        self, job: "_ManagedObjectRow", batch_id: str
     ) -> dict[str, object]:
         """
         Rebuild the spend-tracking metadata for the key, team, and tags that created the
@@ -225,7 +252,7 @@ class CheckBatchCost:
         should not be polled.
         """
         cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
-        result: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
+        result: Final = await _managed_object_table(self.prisma_client).update_many(
             where={
                 "file_purpose": "batch",
                 "status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)},
@@ -244,7 +271,7 @@ class CheckBatchCost:
 
         # A row already in a terminal status is never rewritten by the sweep above, so
         # without this it keeps a poll-page slot forever and starves newer batches.
-        retired: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
+        retired: Final = await _managed_object_table(self.prisma_client).update_many(
             where={
                 "file_purpose": "batch",
                 "batch_processed": False,
@@ -259,9 +286,9 @@ class CheckBatchCost:
                 f"{MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days that were never costed"
             )
 
-    async def _fallback_find_jobs(self) -> list:
+    async def _fallback_find_jobs(self) -> "Sequence[_ManagedObjectRow]":
         """Query batch jobs without the batch_processed filter (for older schemas)."""
-        return await self.prisma_client.db.litellm_managedobjecttable.find_many(
+        return await _managed_object_table(self.prisma_client).find_many(
             where={
                 "file_purpose": "batch",
                 "status": {
@@ -279,7 +306,7 @@ class CheckBatchCost:
             order={"created_at": "asc"},
         )
 
-    async def _retire_job(self, job: "LiteLLM_ManagedObjectTable", reason: str) -> None:
+    async def _retire_job(self, job: "_ManagedObjectRow", reason: str) -> None:
         """
         Take a row that can never be costed out of the poll page. Leaving it selectable
         would burn one of the MAX_OBJECTS_PER_POLL_CYCLE slots on every future cycle, and
@@ -292,7 +319,7 @@ class CheckBatchCost:
             else {"status": "stale_expired"}
         )
         try:
-            await self.prisma_client.db.litellm_managedobjecttable.update(
+            await _managed_object_table(self.prisma_client).update(
                 where={"id": job.id},
                 data=data,
             )
@@ -306,7 +333,7 @@ class CheckBatchCost:
             "so it will no longer be polled"
         )
 
-    async def _claim_job_for_costing(self, job: "LiteLLM_ManagedObjectTable") -> bool:
+    async def _claim_job_for_costing(self, job: "_ManagedObjectRow") -> bool:
         """
         Atomically flip batch_processed from false to true, returning whether this pod won
         the row. Every pod and uvicorn worker schedules its own poller against the shared
@@ -321,7 +348,7 @@ class CheckBatchCost:
         if not self._has_batch_processed_column:
             return True
         try:
-            claimed: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
+            claimed: Final = await _managed_object_table(self.prisma_client).update_many(
                 where={"id": job.id, "batch_processed": False},
                 data={"batch_processed": True},
             )
@@ -332,7 +359,7 @@ class CheckBatchCost:
             return False
         return claimed > 0
 
-    async def _release_job_claim(self, job: "LiteLLM_ManagedObjectTable") -> None:
+    async def _release_job_claim(self, job: "_ManagedObjectRow") -> None:
         """Give a claimed row back once billing it failed, so a later poll cycle retries it.
 
         Safe to match on batch_processed=True: while this poller is active the retrieve
@@ -342,7 +369,7 @@ class CheckBatchCost:
         if not self._has_batch_processed_column:
             return
         try:
-            await self.prisma_client.db.litellm_managedobjecttable.update_many(
+            await _managed_object_table(self.prisma_client).update_many(
                 where={"id": job.id, "batch_processed": True},
                 data={"batch_processed": False},
             )
@@ -353,7 +380,7 @@ class CheckBatchCost:
             )
 
     @staticmethod
-    def _has_unified_id_without_model(job: "LiteLLM_ManagedObjectTable") -> bool:
+    def _has_unified_id_without_model(job: "_ManagedObjectRow") -> bool:
         """A unified id that decodes but carries no model_id can never be routed."""
         from litellm.proxy.openai_files_endpoints.common_utils import (
             convert_b64_uid_to_unified_uid,
@@ -402,7 +429,7 @@ class CheckBatchCost:
         return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
 
     async def _finalize_unbilled_terminal_job(
-        self, job: "prisma_models.LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
+        self, job: "_ManagedObjectRow", response: "LiteLLMBatch"
     ) -> None:
         """Persist a terminal batch that has nothing billable, converting any raw
         provider file ids to managed ids, and take it out of the poll page."""
@@ -426,7 +453,7 @@ class CheckBatchCost:
                 "file_object": response.model_dump_json(),
                 **({"batch_processed": True} if self._has_batch_processed_column else {}),
             }
-            await self.prisma_client.db.litellm_managedobjecttable.update(
+            await _managed_object_table(self.prisma_client).update(
                 where={"id": job.id},
                 data=update_data,
             )
@@ -447,7 +474,7 @@ class CheckBatchCost:
 
     def _resolve_job_routing(
         self,
-        job: "LiteLLM_ManagedObjectTable",
+        job: "_ManagedObjectRow",
         prom_logger: Optional["PrometheusLogger"],
     ) -> Optional[Tuple[str, str]]:
         """
@@ -524,7 +551,7 @@ class CheckBatchCost:
 
     def _resolve_unmanaged_provider_routing(
         self,
-        job: "LiteLLM_ManagedObjectTable",
+        job: "_ManagedObjectRow",
         prom_logger: Optional["PrometheusLogger"],
         llm_provider: str,
         bare_model_name: str,
@@ -620,7 +647,7 @@ class CheckBatchCost:
     @classmethod
     def _get_managed_file_model_name(
         cls,
-        job: "LiteLLM_ManagedObjectTable",
+        job: "_ManagedObjectRow",
         deployment_info: "Deployment",
     ) -> Optional[str]:
         """
@@ -640,7 +667,7 @@ class CheckBatchCost:
         )
 
     @staticmethod
-    def _get_input_file_id(job: "LiteLLM_ManagedObjectTable") -> Optional[str]:
+    def _get_input_file_id(job: "_ManagedObjectRow") -> Optional[str]:
         import json
 
         from litellm.types.utils import LiteLLMBatch
@@ -660,7 +687,7 @@ class CheckBatchCost:
 
     async def _track_completed_batch_cost(
         self,
-        job: "LiteLLM_ManagedObjectTable",
+        job: "_ManagedObjectRow",
         response: "LiteLLMBatch",
         model_id: str,
         batch_id: str,
@@ -936,7 +963,7 @@ class CheckBatchCost:
                 # endpoint may transition a batch to "complete" before
                 # CheckBatchCost runs.  The batch_processed=False filter
                 # already prevents reprocessing finished batches.
-                jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many(
+                jobs = await _managed_object_table(self.prisma_client).find_many(
                     where={
                         "file_purpose": "batch",
                         "batch_processed": False,
@@ -1038,7 +1065,7 @@ class CheckBatchCost:
                     }
                     if self._has_batch_processed_column:
                         update_data["batch_processed"] = True
-                    await self.prisma_client.db.litellm_managedobjecttable.update(
+                    await _managed_object_table(self.prisma_client).update(
                         where={"id": job.id},
                         data=update_data,
                     )

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -6,7 +6,7 @@ same route are non-inference and free.
 """
 
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Dict, Optional, cast
+from typing import TYPE_CHECKING, Dict, Final, Optional, Protocol, cast
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -22,11 +22,31 @@ from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
 
 if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient, ProxyLogging
+    from litellm.repositories.prisma_protocols import TableActions
     from litellm.router import Router
 
 TERMINAL_RESPONSE_STATUSES = frozenset({"completed", "failed", "cancelled", "incomplete"})
 
 
+class _ManagedObjectRow(Protocol):
+    @property
+    def id(self) -> str: ...
+
+    @property
+    def unified_object_id(self) -> str: ...
+
+    @property
+    def created_by(self) -> str | None: ...
+
+    @property
+    def file_object(self) -> object: ...
+
+
+def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]":
+    table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable
+    return table
+
+
 class CheckResponsesCost:
     def __init__(
         self,
@@ -128,7 +148,7 @@ class CheckResponsesCost:
                 f"CheckResponsesCost: stale cleanup failed (poll will continue): {cleanup_err}"
             )
 
-        jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many(
+        jobs = await _managed_object_table(self.prisma_client).find_many(
             where={
                 "status": {"in": ["queued", "in_progress"]},
                 "file_purpose": "response",
@@ -138,7 +158,7 @@ class CheckResponsesCost:
         )
         
         verbose_proxy_logger.debug(f"Found {len(jobs)} response jobs to check")
-        completed_jobs = []
+        completed_jobs: Final[list[_ManagedObjectRow]] = []
 
         for job in jobs:
             unified_object_id = job.unified_object_id
@@ -189,7 +209,7 @@ class CheckResponsesCost:
 
         # Mark completed jobs in the database
         if len(completed_jobs) > 0:
-            await self.prisma_client.db.litellm_managedobjecttable.update_many(
+            await _managed_object_table(self.prisma_client).update_many(
                 where={"id": {"in": [job.id for job in completed_jobs]}},
                 data={"status": "completed"},
             )

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -481,10 +481,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         """
         if self.prisma_client is None:
             return
-        managed_object = (
-            await self.prisma_client.db.litellm_managedobjecttable.find_first(
-                where={"OR": [{"unified_object_id": object_id}, {"model_object_id": object_id}]}
-            )
+        managed_object = await _managed_object_table(self.prisma_client).find_first(
+            where={"OR": [{"unified_object_id": object_id}, {"model_object_id": object_id}]}
         )
         if managed_object is None:
             return
@@ -509,10 +507,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         """
         if self.prisma_client is None:
             return
-        managed_file = (
-            await self.prisma_client.db.litellm_managedfiletable.find_first(
-                where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]}
-            )
+        managed_file = await _managed_file_table(self.prisma_client).find_first(
+            where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]}
         )
         if managed_file is None:
             return
@@ -535,8 +531,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         provider_file_ids = tuple(
             file_id
             for file_id in (
-                getattr(response, "output_file_id", None),
-                getattr(response, "error_file_id", None),
+                response.output_file_id,
+                response.error_file_id,
             )
             if file_id and not _is_base64_encoded_unified_file_id(file_id)
         )
@@ -544,10 +540,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             return
         if self.prisma_client is None:
             return
-        batch_row = (
-            await self.prisma_client.db.litellm_managedobjecttable.find_first(
-                where={"unified_object_id": response.id}
-            )
+        batch_row = await _managed_object_table(self.prisma_client).find_first(
+            where={"unified_object_id": response.id}
         )
         if batch_row is None or (
             batch_row.created_by is None and batch_row.team_id is None

gateway/routes/allowlist.py

@@ -96,16 +96,19 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/assemblyai/",
     "/eu.assemblyai/",
     "/deepgram/",
+    "/fal_ai/",
     "/langfuse/",
     "/vllm/",
     "/mistral/",
     "/typesafe/",
+    "/openrouter/",
     "/nvidia_nim/",
     "/groq/",
     "/voyage/",
     "/cursor/",
     "/milvus/",
     "/openai_passthrough/",
+    "/tinyfish/",
     # Dynamic provider / toolset passthrough (path templates)
     "/{provider}/",
     "/toolset/",

helm/litellm/templates/backend/deployment.yaml

@@ -7,6 +7,9 @@ metadata:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: backend
 spec:
+  {{- if and (not .Values.backend.hpa.enabled) (not (kindIs "invalid" .Values.backend.replicaCount)) }}
+  replicas: {{ .Values.backend.replicaCount }}
+  {{- end }}
   {{- with .Values.backend.strategy }}
   strategy:
     {{- toYaml . | nindent 4 }}

helm/litellm/templates/gateway/deployment.yaml

@@ -7,6 +7,9 @@ metadata:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: gateway
 spec:
+  {{- if and (not .Values.gateway.hpa.enabled) (not (kindIs "invalid" .Values.gateway.replicaCount)) }}
+  replicas: {{ .Values.gateway.replicaCount }}
+  {{- end }}
   {{- with .Values.gateway.strategy }}
   strategy:
     {{- toYaml . | nindent 4 }}

helm/litellm/templates/ui/deployment.yaml

@@ -7,6 +7,9 @@ metadata:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: ui
 spec:
+  {{- if and (not .Values.ui.hpa.enabled) (not (kindIs "invalid" .Values.ui.replicaCount)) }}
+  replicas: {{ .Values.ui.replicaCount }}
+  {{- end }}
   {{- with .Values.ui.strategy }}
   strategy:
     {{- toYaml . | nindent 4 }}

helm/litellm/tests/replica_count_tests.yaml

@@ -0,0 +1,100 @@
+suite: test fixed replica count when HPA is disabled
+templates:
+  - gateway/deployment.yaml
+  - gateway/configmap.yaml
+  - backend/deployment.yaml
+  - ui/deployment.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: gateway renders replicaCount into spec.replicas when its HPA is disabled
+    template: gateway/deployment.yaml
+    set:
+      gateway.hpa.enabled: false
+      gateway.replicaCount: 3
+    asserts:
+      - isKind:
+          of: Deployment
+      - equal:
+          path: spec.replicas
+          value: 3
+
+  - it: backend renders replicaCount into spec.replicas when its HPA is disabled
+    template: backend/deployment.yaml
+    set:
+      backend.hpa.enabled: false
+      backend.replicaCount: 2
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 2
+
+  - it: ui renders replicaCount into spec.replicas when its HPA is disabled
+    template: ui/deployment.yaml
+    set:
+      ui.hpa.enabled: false
+      ui.replicaCount: 2
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 2
+
+  - it: replicaCount 0 scales the gateway to zero instead of being treated as unset
+    template: gateway/deployment.yaml
+    set:
+      gateway.hpa.enabled: false
+      gateway.replicaCount: 0
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 0
+
+  - it: a component with HPA disabled but no replicaCount set keeps omitting spec.replicas, so upgrades do not reset a hand-scaled Deployment
+    set:
+      gateway.hpa.enabled: false
+      backend.hpa.enabled: false
+      ui.hpa.enabled: false
+    asserts:
+      - notExists:
+          path: spec.replicas
+        template: gateway/deployment.yaml
+      - notExists:
+          path: spec.replicas
+        template: backend/deployment.yaml
+      - notExists:
+          path: spec.replicas
+        template: ui/deployment.yaml
+
+  - it: every component omits spec.replicas when its HPA is enabled, so the autoscaler owns the count
+    set:
+      gateway.hpa.enabled: true
+      gateway.replicaCount: 3
+      backend.hpa.enabled: true
+      backend.replicaCount: 3
+      ui.hpa.enabled: true
+      ui.replicaCount: 3
+    asserts:
+      - notExists:
+          path: spec.replicas
+        template: gateway/deployment.yaml
+      - notExists:
+          path: spec.replicas
+        template: backend/deployment.yaml
+      - notExists:
+          path: spec.replicas
+        template: ui/deployment.yaml
+
+  - it: a component with HPA disabled renders replicas while a sibling with HPA enabled does not
+    set:
+      gateway.hpa.enabled: false
+      gateway.replicaCount: 4
+      backend.hpa.enabled: true
+      backend.replicaCount: 4
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 4
+        template: gateway/deployment.yaml
+      - notExists:
+          path: spec.replicas
+        template: backend/deployment.yaml

helm/litellm/values.yaml

@@ -397,6 +397,11 @@ gateway:
   #   failureThreshold: 30
   #   periodSeconds: 10
   startupProbe: {}
+  # Optional fixed pod count, rendered into the Deployment's spec.replicas only
+  # when hpa.enabled is false. Unset by default so an existing Deployment keeps
+  # its current count; with the HPA on, the autoscaler owns the count, e.g.:
+  #   replicaCount: 3
+  replicaCount:
   hpa:
     enabled: true
     minReplicas: 1
@@ -524,6 +529,8 @@ backend:
   strategy: {}
   # Optional startupProbe; same shape as gateway.startupProbe. Empty by default.
   startupProbe: {}
+  # Same semantics as gateway.replicaCount.
+  replicaCount:
   hpa:
     enabled: true
     minReplicas: 1
@@ -590,6 +597,8 @@ ui:
   strategy: {}
   # Optional startupProbe; same shape as gateway.startupProbe. Empty by default.
   startupProbe: {}
+  # Same semantics as gateway.replicaCount.
+  replicaCount:
   hpa:
     enabled: false
     minReplicas: 1

litellm-proxy-extras/litellm_proxy_extras/migrations/20260917000000_add_agent_access_group_ids/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN IF NOT EXISTS "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[];

litellm-proxy-extras/litellm_proxy_extras/migrations/20260919000000_add_autorouter_user_session_rollup/migration.sql

@@ -0,0 +1,42 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_AutoRouterUserSession" (
+    "user_id" TEXT NOT NULL,
+    "api_key" TEXT NOT NULL,
+    "session_id" TEXT NOT NULL,
+    "router_name" TEXT NOT NULL,
+    "router_type" TEXT NOT NULL,
+    "first_turn_at" TIMESTAMP(3) NOT NULL,
+    "last_turn_at" TIMESTAMP(3) NOT NULL,
+    "last_model" TEXT NOT NULL,
+    "models" JSONB NOT NULL DEFAULT '{}',
+    "turns" INTEGER NOT NULL DEFAULT 0,
+    "unordered_turns" INTEGER NOT NULL DEFAULT 0,
+    "covered_turns" INTEGER NOT NULL DEFAULT 0,
+    "cache_hits" INTEGER NOT NULL DEFAULT 0,
+    "same_model_turns" INTEGER NOT NULL DEFAULT 0,
+    "same_model_hits" INTEGER NOT NULL DEFAULT 0,
+    "first_visit_turns" INTEGER NOT NULL DEFAULT 0,
+    "first_visit_hits" INTEGER NOT NULL DEFAULT 0,
+    "return_turns" INTEGER NOT NULL DEFAULT 0,
+    "return_hits" INTEGER NOT NULL DEFAULT 0,
+    "return_expired_misses" INTEGER NOT NULL DEFAULT 0,
+    "return_within_ttl_misses" INTEGER NOT NULL DEFAULT 0,
+    "ttl_5m_turns" INTEGER NOT NULL DEFAULT 0,
+    "ttl_1h_turns" INTEGER NOT NULL DEFAULT 0,
+    "total_tokens" BIGINT NOT NULL DEFAULT 0,
+    "spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "savings_estimated_turns" INTEGER NOT NULL DEFAULT 0,
+    "savings_estimated_actual_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "savings_estimated_saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "savings_estimated_baseline_models" JSONB NOT NULL DEFAULT '{}',
+    "classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "classifier_cost_recorded_turns" INTEGER NOT NULL DEFAULT 0,
+    "tier_turns" JSONB NOT NULL DEFAULT '{}',
+    "baseline_models" JSONB NOT NULL DEFAULT '{}',
+
+    CONSTRAINT "LiteLLM_AutoRouterUserSession_pkey" PRIMARY KEY ("user_id", "api_key", "session_id", "router_name")
+);
+
+CREATE INDEX IF NOT EXISTS "idx_autorouter_user_session_last_turn" ON "LiteLLM_AutoRouterUserSession"("last_turn_at");
+
+CREATE INDEX IF NOT EXISTS "idx_autorouter_user_session_user_last_turn" ON "LiteLLM_AutoRouterUserSession"("user_id", "last_turn_at");

litellm-proxy-extras/litellm_proxy_extras/migrations/20260920041500_add_policy_attachment_is_default/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "LiteLLM_PolicyAttachmentTable" ADD COLUMN IF NOT EXISTS "is_default" BOOLEAN NOT NULL DEFAULT false;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260921000000_add_password_reset_columns/migration.sql

@@ -0,0 +1,3 @@
+ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "password_reset_required" BOOLEAN;
+
+ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "last_breach_check_at" TIMESTAMP(3);

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -73,6 +73,7 @@ model LiteLLM_AgentsTable {
   static_headers       Json?          @default("{}")
   extra_headers        String[]       @default([])
   agent_access_groups String[]       @default([])
+  access_group_ids     String[]       @default([])
   object_permission_id String?
   object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
   spend      Float    @default(0.0)
@@ -246,6 +247,8 @@ model LiteLLM_UserTable {
     organization_id String?
     object_permission_id String?
     password  String?
+    password_reset_required Boolean?
+    last_breach_check_at DateTime?
     teams    String[] @default([])
     user_role  String?
 		max_budget Float?
@@ -1419,6 +1422,7 @@ model LiteLLM_PolicyAttachmentTable {
   models           String[]  @default([])  // Model names or patterns
   tags             String[]  @default([])  // Tag patterns (e.g., ["healthcare", "prod-*"])
   priority         Int?                    // Explicit execution order
+  is_default       Boolean   @default(false)  // Applied only when no non-default attachment matches
   created_at       DateTime  @default(now())
   created_by       String?
   updated_at       DateTime  @default(now()) @updatedAt
@@ -1620,6 +1624,47 @@ model LiteLLM_AutoRouterSession {
   @@index([last_turn_at], map: "idx_autorouter_session_last_turn")
 }
 
+model LiteLLM_AutoRouterUserSession {
+  user_id               String
+  api_key               String
+  session_id            String
+  router_name           String
+  router_type           String
+  first_turn_at         DateTime
+  last_turn_at          DateTime
+  last_model            String
+  models                Json     @default("{}")
+  turns                 Int      @default(0)
+  unordered_turns       Int      @default(0)
+  covered_turns         Int      @default(0)
+  cache_hits            Int      @default(0)
+  same_model_turns      Int      @default(0)
+  same_model_hits       Int      @default(0)
+  first_visit_turns     Int      @default(0)
+  first_visit_hits      Int      @default(0)
+  return_turns          Int      @default(0)
+  return_hits           Int      @default(0)
+  return_expired_misses Int      @default(0)
+  return_within_ttl_misses  Int      @default(0)
+  ttl_5m_turns          Int      @default(0)
+  ttl_1h_turns          Int      @default(0)
+  total_tokens          BigInt   @default(0)
+  spend                 Float    @default(0)
+  saved_spend           Float    @default(0)
+  savings_estimated_turns Int    @default(0)
+  savings_estimated_actual_spend Float @default(0)
+  savings_estimated_saved_spend Float @default(0)
+  savings_estimated_baseline_models Json @default("{}")
+  classifier_cost       Float    @default(0)
+  classifier_cost_recorded_turns Int @default(0)
+  tier_turns            Json     @default("{}")
+  baseline_models       Json     @default("{}")
+
+  @@id([user_id, api_key, session_id, router_name])
+  @@index([last_turn_at], map: "idx_autorouter_user_session_last_turn")
+  @@index([user_id, last_turn_at], map: "idx_autorouter_user_session_user_last_turn")
+}
+
 // Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in
 // either direction. forward duplicates the requests the keys did not route through the
 // router through it, answering whether they should adopt it; reverse duplicates the

litellm-rust/Cargo.toml

@@ -14,20 +14,41 @@ litellm-host = { path = "crates/host" }
 litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
 litellm-framing = { path = "crates/framer" }
 litellm-auth = { path = "crates/auth" }
+litellm-auth-types = { path = "crates/auth-types" }
 litellm-auth-aws = { path = "crates/auth-aws" }
 litellm-auth-azure = { path = "crates/auth-azure" }
 litellm-auth-gcp = { path = "crates/auth-gcp" }
+litellm-secrets = { path = "crates/secrets" }
+litellm-secrets-types = { path = "crates/secrets-types" }
+litellm-secrets-aws = { path = "crates/secrets-aws" }
+litellm-secrets-google = { path = "crates/secrets-google" }
+litellm-secrets-hashicorp = { path = "crates/secrets-hashicorp" }
+litellm-secrets-azure = { path = "crates/secrets-azure" }
+litellm-secrets-cyberark = { path = "crates/secrets-cyberark" }
 litellm-http = { path = "crates/http" }
 litellm-llms = { path = "crates/llms" }
 litellm-types = { path = "crates/types" }
 litellm-core-utils = { path = "crates/core-utils" }
 litellm-cache = { path = "crates/cache" }
+litellm-cache-azure-blob = { path = "crates/cache-azure-blob" }
 litellm-cache-memory = { path = "crates/cache-memory" }
+litellm-cache-redis = { path = "crates/cache-redis" }
+litellm-cache-s3 = { path = "crates/cache-s3" }
+litellm-cache-gcs = { path = "crates/cache-gcs" }
+litellm-cache-disk = { path = "crates/cache-disk" }
+litellm-cache-redis-semantic = { path = "crates/cache-redis-semantic" }
+litellm-cache-response = { path = "crates/cache-response" }
+litellm-cache-qdrant-semantic = { path = "crates/cache-qdrant-semantic" }
 litellm-token-counter = { path = "crates/token-counter" }
+litellm-token-counter-fast = { path = "crates/token-counter-fast" }
+litellm-token-counter-huggingface = { path = "crates/token-counter-huggingface" }
+litellm-token-counter-tiktoken = { path = "crates/token-counter-tiktoken" }
 litellm-host-python = { path = "crates/host-python" }
 
 bytes = "1"
 http = "1"
+google-cloud-auth = { version = "1.16.0", default-features = false }
+jsonwebtoken = { version = "11.1.0", default-features = false }
 hyper-util = { version = "0.1.20", default-features = false, features = ["client-proxy"] }
 proptest = "1.7.0"
 pyo3 = "0.29.2"
@@ -35,9 +56,14 @@ pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] }
 pythonize = "0.29.0"
 rand = "0.8"
 reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls", "http2", "stream"] }
+qdrant-client = { version = "1.19.0", default-features = false }
+uuid = { version = "1", features = ["v4"] }
 rstest = "0.26.1"
 rstest_reuse = "0.7.0"
 rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] }
+rustify = "=0.7.0"
+rustify_derive = "=0.5.5"
+vaultrs = { version = "=0.8.0", default-features = false, features = ["rustls"] }
 rustls-native-certs = "0.8"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["float_roundtrip"] }
@@ -45,6 +71,8 @@ serde_with = { version = "=3.16.1", default-features = false, features = ["std",
 sha2 = "0.10"
 subtle = "2"
 thiserror = "2.0"
+tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] }
+tiktoken-rs = "0.12.0"
 tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] }
 tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }
 futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }
@@ -52,6 +80,7 @@ base64 = "0.22"
 moka = { version = "0.12.16", features = ["future"] }
 strum = { version = "0.28.0", features = ["derive"] }
 url = "2.5.8"
+percent-encoding = "2.3"
 webpki-roots = "1"
 time = { version = "0.3.53", features = ["parsing"] }
 criterion = "0.8.2"
@@ -60,7 +89,7 @@ veil = "0.3.0"
 
 [profile.release]
 opt-level = 3
-lto = "thin"
+lto = "fat"
 codegen-units = 1
 panic = "unwind"
 debug = false

litellm-rust/crates/auth-aws/Cargo.toml

@@ -6,7 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
-litellm-auth.workspace = true
+litellm-auth-types.workspace = true
 litellm-http.workspace = true
 
 moka = { workspace = true, features = ["sync"] }

litellm-rust/crates/auth-aws/src/aws.rs

@@ -621,6 +621,26 @@ mod tests {
         None
     }
 
+    #[test]
+    fn secret_names_cover_environment_reads() {
+        let seen = std::sync::Arc::new(std::sync::Mutex::new(
+            std::collections::BTreeSet::<String>::new(),
+        ));
+        let recorded = seen.clone();
+        let env = |name: &str| {
+            recorded.lock().unwrap().insert(name.to_string());
+            None
+        };
+        resolve_aws_region(None, &Map::new(), &env);
+        aws_auth_config(&Map::new(), &env);
+        assert!(
+            seen.lock()
+                .unwrap()
+                .iter()
+                .all(|name| crate::constants::SECRET_NAMES.contains(&name.as_str()))
+        );
+    }
+
     #[test]
     fn a_region_comes_from_the_call_then_the_model_then_the_environment() {
         let params = Map::from_iter([("aws_region_name".to_string(), Value::from("eu-west-1"))]);

litellm-rust/crates/auth-aws/src/constants.rs

@@ -3,6 +3,8 @@ pub const AWS_SECRET_ACCESS_KEY: &str = "AWS_SECRET_ACCESS_KEY";
 pub const AWS_SESSION_TOKEN: &str = "AWS_SESSION_TOKEN";
 pub const AWS_REGION_NAME: &str = "AWS_REGION_NAME";
 pub const AWS_REGION: &str = "AWS_REGION";
+pub const AWS_DEFAULT_REGION: &str = "AWS_DEFAULT_REGION";
+pub const AWS_BEDROCK_RUNTIME_ENDPOINT: &str = "AWS_BEDROCK_RUNTIME_ENDPOINT";
 pub const AWS_SESSION_NAME: &str = "AWS_SESSION_NAME";
 pub const AWS_PROFILE_NAME: &str = "AWS_PROFILE_NAME";
 pub const AWS_ROLE_NAME: &str = "AWS_ROLE_NAME";
@@ -12,6 +14,19 @@ pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
 pub const AWS_STS_ENDPOINT: &str = "AWS_STS_ENDPOINT";
 pub const AWS_EXTERNAL_ID: &str = "AWS_EXTERNAL_ID";
 pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK";
+pub const SECRET_NAMES: &[&str] = &[
+    AWS_ACCESS_KEY_ID,
+    AWS_SECRET_ACCESS_KEY,
+    AWS_SESSION_TOKEN,
+    AWS_REGION_NAME,
+    AWS_REGION,
+    AWS_SESSION_NAME,
+    AWS_PROFILE_NAME,
+    AWS_ROLE_NAME,
+    AWS_WEB_IDENTITY_TOKEN,
+    AWS_STS_ENDPOINT,
+    AWS_EXTERNAL_ID,
+];
 
 /// Headers SigV4 covers, beyond the `x-amz-` / `x-amzn-` prefixes. Mirrors
 /// Python's `_filter_headers_for_aws_signature` allowlist.

litellm-rust/crates/auth-aws/src/error.rs

@@ -22,7 +22,7 @@ pub enum Error {
     AwsMissingWebIdentityCredentials,
 }
 
-impl From<Error> for litellm_auth::Error {
+impl From<Error> for litellm_auth_types::Error {
     fn from(error: Error) -> Self {
         Self::ProviderAuthentication(error.to_string())
     }
@@ -34,11 +34,11 @@ mod tests {
 
     #[test]
     fn converts_to_shared_auth_error_without_losing_context() {
-        let error = litellm_auth::Error::from(Error::AwsProfile("profile not found".into()));
+        let error = litellm_auth_types::Error::from(Error::AwsProfile("profile not found".into()));
 
         assert_eq!(
             error,
-            litellm_auth::Error::ProviderAuthentication(
+            litellm_auth_types::Error::ProviderAuthentication(
                 "AWS profile credentials failed: profile not found".into()
             )
         );

litellm-rust/crates/auth-azure/Cargo.toml

@@ -6,7 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
-litellm-auth.workspace = true
+litellm-auth-types.workspace = true
 
 moka.workspace = true
 serde_json.workspace = true

litellm-rust/crates/auth-azure/src/credential_provider_cache.rs

@@ -4,7 +4,7 @@ use std::sync::Arc;
 use azure_core::credentials::TokenCredential;
 use moka::future::Cache;
 
-use litellm_auth::Error;
+use litellm_auth_types::Error;
 
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub(crate) struct AzureCredentialProviderCacheKey {

litellm-rust/crates/auth-azure/src/lib.rs

@@ -3,5 +3,5 @@ mod native;
 mod resolve;
 mod types;
 
-pub use resolve::AzureAuthService;
-pub use types::AzureAuthInputs;
+pub use resolve::{AzureAuthService, SECRET_NAMES};
+pub use types::{AzureAuthInputs, ConfigValue};

litellm-rust/crates/auth-azure/src/native.rs

@@ -12,8 +12,8 @@ use azure_identity::{
 };
 use sha2::{Digest, Sha256};
 
-use litellm_auth::Error;
-use litellm_auth::{InputSource, ResolvedCredential, SecretValue, Sourced};
+use litellm_auth_types::Error;
+use litellm_auth_types::{InputSource, ResolvedCredential, SecretValue, Sourced};
 
 use super::credential_provider_cache::{
     AzureCredentialProviderCache, AzureCredentialProviderCacheKey,
@@ -484,7 +484,7 @@ mod tests {
     use azure_core::{Bytes, Result};
 
     use super::{NativeAzureRequest, NativeAzureTokenAcquirer, ValidatedAzureRequest};
-    use litellm_auth::{InputSource, SecretValue, Sourced};
+    use litellm_auth_types::{InputSource, SecretValue, Sourced};
 
     fn deployment<T>(value: T) -> Sourced<T> {
         Sourced::new(value, InputSource::Deployment)
@@ -649,7 +649,7 @@ mod tests {
 
         assert!(matches!(
             error,
-            litellm_auth::Error::MixedAzureCredentialSources
+            litellm_auth_types::Error::MixedAzureCredentialSources
         ));
     }
 
@@ -679,7 +679,10 @@ mod tests {
                 authority,
             ))
             .unwrap_err();
-            assert!(matches!(error, litellm_auth::Error::InvalidAzureAuthority));
+            assert!(matches!(
+                error,
+                litellm_auth_types::Error::InvalidAzureAuthority
+            ));
         }
     }
 }

litellm-rust/crates/auth-azure/src/resolve.rs

@@ -1,5 +1,5 @@
-use litellm_auth::Error;
-use litellm_auth::{
+use litellm_auth_types::Error;
+use litellm_auth_types::{
     CredentialFileRef, CredentialLookup, CredentialRef, InputSource, ResolvedCredential,
     SecretValue, Sourced, TokenProviderHandle,
 };
@@ -19,6 +19,17 @@ const AZURE_AUTHORITY_HOST_ENV: &str = "AZURE_AUTHORITY_HOST";
 const AZURE_CREDENTIAL_ENV: &str = "AZURE_CREDENTIAL";
 const AZURE_FEDERATED_TOKEN_FILE_ENV: &str = "AZURE_FEDERATED_TOKEN_FILE";
 
+pub const SECRET_NAMES: &[&str] = &[
+    AZURE_AD_TOKEN_ENV,
+    AZURE_TENANT_ID_ENV,
+    AZURE_CLIENT_ID_ENV,
+    AZURE_CLIENT_SECRET_ENV,
+    AZURE_SCOPE_ENV,
+    AZURE_AUTHORITY_HOST_ENV,
+    AZURE_CREDENTIAL_ENV,
+    AZURE_FEDERATED_TOKEN_FILE_ENV,
+];
+
 #[derive(Clone, Debug)]
 pub(crate) enum AzureCredentialPlan {
     Supplied(Sourced<ResolvedCredential>),
@@ -440,20 +451,21 @@ fn non_empty_reference(value: &str, kind: &str) -> Result<String, Error> {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::BTreeSet;
     use std::future::Future;
     use std::sync::{Arc, Mutex};
 
     use serde_json::json;
 
     use super::{
-        AzureAuthService, AzureCredentialPlan, AzureTokenAcquirer, oidc_reference,
+        AzureAuthService, AzureCredentialPlan, AzureTokenAcquirer, SECRET_NAMES, oidc_reference,
         resolve_reference, select_auth_plan,
     };
     use crate::native::ValidatedAzureRequest;
     use crate::types::AzureAuthInputs;
-    use litellm_auth::Error;
-    use litellm_auth::ResolvedCredential;
-    use litellm_auth::{
+    use litellm_auth_types::Error;
+    use litellm_auth_types::ResolvedCredential;
+    use litellm_auth_types::{
         CredentialFileRef, CredentialLookup, CredentialLookupFuture, CredentialRef,
         CredentialResolver, CredentialResolverHandle, InputSource, SecretValue, Sourced,
     };
@@ -517,6 +529,24 @@ mod tests {
         assert!(matches!(plan, AzureCredentialPlan::Native(_)));
     }
 
+    #[test]
+    fn secret_names_cover_environment_reads() {
+        let seen = std::sync::Arc::new(std::sync::Mutex::new(BTreeSet::<String>::new()));
+        let recorded = seen.clone();
+        let inputs = AzureAuthInputs::default();
+        select_auth_plan(&inputs, &|name| {
+            recorded.lock().unwrap().insert(name.to_string());
+            None
+        })
+        .unwrap();
+        assert!(
+            seen.lock()
+                .unwrap()
+                .iter()
+                .all(|name| SECRET_NAMES.contains(&name.as_str()))
+        );
+    }
+
     #[test]
     fn supplied_token_does_not_require_refresh() {
         let params = json!({"azure_ad_token": "token"});
@@ -661,8 +691,8 @@ mod tests {
     #[derive(Debug)]
     struct CallerToken(&'static str);
 
-    impl litellm_auth::TokenProvider for CallerToken {
-        fn acquire(&self) -> litellm_auth::TokenFuture<'_> {
+    impl litellm_auth_types::TokenProvider for CallerToken {
+        fn acquire(&self) -> litellm_auth_types::TokenFuture<'_> {
             Box::pin(async move {
                 Ok(ResolvedCredential::AccessToken {
                     token: SecretValue::new(self.0),
@@ -675,7 +705,7 @@ mod tests {
     fn caller_inputs(token: &'static str) -> AzureAuthInputs {
         let params = json!({"azure_ad_token": "static-token"});
         AzureAuthInputs {
-            azure_ad_token_provider: Some(litellm_auth::TokenProviderHandle::new(Arc::new(
+            azure_ad_token_provider: Some(litellm_auth_types::TokenProviderHandle::new(Arc::new(
                 CallerToken(token),
             ))),
             ..AzureAuthInputs::from_optional_params(params.as_object().unwrap()).unwrap()

litellm-rust/crates/auth-azure/src/types.rs

@@ -1,6 +1,6 @@
 use std::collections::BTreeMap;
 
-use litellm_auth::{
+use litellm_auth_types::{
     CredentialResolverHandle, Error, InputSource, SecretValue, Sourced, TokenProviderHandle,
 };
 use serde_json::{Map, Value};
@@ -51,6 +51,21 @@ pub struct AzureAuthInputs {
 }
 
 impl AzureAuthInputs {
+    pub fn default_credential_for_scope(scope: &str) -> Self {
+        Self {
+            azure_scope: ConfigValue::Value(Sourced::new(
+                scope.to_string(),
+                InputSource::Deployment,
+            )),
+            azure_credential: ConfigValue::Value(Sourced::new(
+                "DefaultAzureCredential".to_string(),
+                InputSource::Deployment,
+            )),
+            enable_azure_ad_token_refresh: Sourced::new(true, InputSource::Deployment),
+            ..Self::default()
+        }
+    }
+
     pub fn or_configured_token_refresh(self, enabled: bool) -> Self {
         if *self.enable_azure_ad_token_refresh.value() || !enabled {
             return self;
@@ -126,7 +141,7 @@ fn source_for(sources: &BTreeMap<String, InputSource>, name: &str) -> InputSourc
 mod tests {
     use std::collections::BTreeMap;
 
-    use litellm_auth::{InputSource, Sourced};
+    use litellm_auth_types::{InputSource, Sourced};
     use serde_json::json;
 
     use super::{AzureAuthInputs, AzureCredentialType, ConfigValue};

litellm-rust/crates/auth-gcp/Cargo.toml

@@ -5,8 +5,11 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 
+[features]
+google-sdk = ["dep:google-cloud-auth", "dep:http"]
+
 [dependencies]
-litellm-auth.workspace = true
+litellm-auth-types.workspace = true
 
 moka.workspace = true
 serde_json.workspace = true
@@ -14,3 +17,5 @@ sha2.workspace = true
 tokio.workspace = true
 
 gcp_auth = "0.12.7"
+google-cloud-auth = { workspace = true, optional = true }
+http = { workspace = true, optional = true }

litellm-rust/crates/auth-gcp/src/lib.rs

@@ -1,13 +1,18 @@
 use std::{collections::BTreeMap, future::Future, path::Path, pin::Pin, sync::Arc};
 
 use gcp_auth::{CustomServiceAccount, TokenProvider};
-use litellm_auth::{
+use litellm_auth_types::{
     CredentialPlacement, Error, InputSource, SecretValue, Sourced, http::apply_credential,
 };
 use moka::future::Cache;
 use serde_json::{Map, Value};
 use sha2::{Digest, Sha256};
 
+#[cfg(feature = "google-sdk")]
+mod sdk;
+#[cfg(feature = "google-sdk")]
+pub use sdk::GoogleCredentials;
+
 const CLOUD_PLATFORM_SCOPE: &str = "https://www.googleapis.com/auth/cloud-platform";
 const GOOGLE_OAUTH_TOKEN_ENDPOINT: &str = "https://oauth2.googleapis.com/token";
 const GOOGLE_APPLICATION_CREDENTIALS_ENV: &str = "GOOGLE_APPLICATION_CREDENTIALS";
@@ -18,6 +23,16 @@ const VERTEXAI_PROJECT_ENV: &str = "VERTEXAI_PROJECT";
 const VERTEXAI_LOCATION_ENV: &str = "VERTEXAI_LOCATION";
 const VERTEX_LOCATION_ENV: &str = "VERTEX_LOCATION";
 
+pub const SECRET_NAMES: &[&str] = &[
+    VERTEX_AI_API_KEY_ENV,
+    VERTEXAI_API_KEY_ENV,
+    VERTEXAI_CREDENTIALS_ENV,
+    GOOGLE_APPLICATION_CREDENTIALS_ENV,
+    VERTEXAI_PROJECT_ENV,
+    VERTEXAI_LOCATION_ENV,
+    VERTEX_LOCATION_ENV,
+];
+
 #[derive(Clone, Debug, Default)]
 pub struct VertexConfig {
     credentials: Option<Sourced<SecretValue>>,
@@ -26,19 +41,31 @@ pub struct VertexConfig {
 }
 
 impl VertexConfig {
+    pub fn new(
+        credentials: Option<Sourced<SecretValue>>,
+        project_id: Option<String>,
+        location: Option<String>,
+    ) -> Self {
+        Self {
+            credentials: credentials.filter(|value| !value.value().expose().trim().is_empty()),
+            project_id: project_id.filter(|value| !value.trim().is_empty()),
+            location: location.filter(|value| !value.trim().is_empty()),
+        }
+    }
+
     pub fn from_sourced_optional_params(
         params: &Map<String, Value>,
         sources: &BTreeMap<String, InputSource>,
     ) -> Result<Self, Error> {
-        Ok(Self {
-            credentials: optional_credentials(
+        Ok(Self::new(
+            optional_credentials(
                 params,
                 sources,
                 &["vertex_credentials", "vertex_ai_credentials"],
             )?,
-            project_id: optional_string(params, &["vertex_project", "vertex_ai_project"])?,
-            location: optional_string(params, &["vertex_location", "vertex_ai_location"])?,
-        })
+            optional_string(params, &["vertex_project", "vertex_ai_project"])?,
+            optional_string(params, &["vertex_location", "vertex_ai_location"])?,
+        ))
     }
 
     pub fn or_configured(self, project_id: Option<&str>, location: Option<&str>) -> Self {
@@ -111,6 +138,14 @@ impl VertexAuth {
         }
     }
 
+    pub async fn access_token(
+        &self,
+        config: &VertexConfig,
+        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
+    ) -> Result<String, Error> {
+        self.load_provider(config, env_lookup).await?.token().await
+    }
+
     pub async fn validate_environment(
         &self,
         headers: Vec<(String, String)>,
@@ -381,6 +416,7 @@ fn auth_acquisition_error(error: gcp_auth::Error) -> Error {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::BTreeSet;
     use std::sync::atomic::{AtomicUsize, Ordering};
 
     use serde_json::json;
@@ -451,6 +487,27 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn secret_names_cover_environment_reads() {
+        let seen = Arc::new(std::sync::Mutex::new(BTreeSet::<String>::new()));
+        let recorded = seen.clone();
+        let env = |name: &str| {
+            recorded.lock().unwrap().insert(name.to_string());
+            None
+        };
+        let auth = auth(Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
+        auth.validate_environment(Vec::new(), None, &VertexConfig::default(), &env)
+            .await
+            .unwrap();
+        get_vertex_ai_location(&VertexConfig::default(), &env);
+        assert!(
+            seen.lock()
+                .unwrap()
+                .iter()
+                .all(|name| SECRET_NAMES.contains(&name.as_str()))
+        );
+    }
+
     #[test]
     fn empty_primary_values_fall_back_to_python_aliases() {
         let config = config(json!({
@@ -469,6 +526,39 @@ mod tests {
         assert_eq!(config.location(), Some("alias-location"));
     }
 
+    #[test]
+    fn typed_config_preserves_source_and_empty_value_fallback() {
+        let configured = VertexConfig::new(
+            Some(Sourced::new(
+                SecretValue::new("inline-json"),
+                InputSource::Request,
+            )),
+            Some("project".into()),
+            Some("location".into()),
+        );
+        assert!(matches!(
+            credential_source(&configured, &|_| Some("environment-json".into())),
+            CredentialSource::Inline(value) if value.expose() == "inline-json"
+        ));
+        let empty = VertexConfig::new(
+            Some(Sourced::new(SecretValue::new(" "), InputSource::Request)),
+            Some(" ".into()),
+            Some(" ".into()),
+        );
+        assert!(matches!(
+            credential_source(&empty, &|_| None),
+            CredentialSource::Adc
+        ));
+        assert_eq!(
+            get_vertex_ai_project(&empty, &|_| Some("env-project".into())).as_deref(),
+            Some("env-project")
+        );
+        assert_eq!(
+            get_vertex_ai_location(&empty, &|_| Some("env-location".into())).as_deref(),
+            Some("env-location")
+        );
+    }
+
     #[test]
     fn project_and_location_prefer_input_then_environment() {
         let configured =

litellm-rust/crates/auth-gcp/src/sdk.rs

@@ -0,0 +1,106 @@
+use std::sync::Arc;
+
+use google_cloud_auth::credentials::{CacheableResource, CredentialsProvider, EntityTag};
+use google_cloud_auth::errors::CredentialsError;
+use http::{Extensions, HeaderMap, HeaderName, HeaderValue};
+use litellm_auth_types::Error;
+
+use crate::{VertexAuth, VertexConfig};
+
+type EnvironmentLookup = dyn Fn(&str) -> Option<String> + Send + Sync;
+
+pub struct GoogleCredentials {
+    auth: VertexAuth,
+    config: VertexConfig,
+    environment: Arc<EnvironmentLookup>,
+}
+
+impl GoogleCredentials {
+    pub fn new(config: VertexConfig, environment: Arc<EnvironmentLookup>) -> Self {
+        Self {
+            auth: VertexAuth::default(),
+            config,
+            environment,
+        }
+    }
+
+    pub async fn request_headers(&self) -> Result<HeaderMap, Error> {
+        let response = self
+            .auth
+            .validate_environment(Vec::new(), None, &self.config, &|name| {
+                (self.environment)(name)
+            })
+            .await?;
+        response
+            .headers
+            .into_iter()
+            .map(|(key, value)| {
+                let name =
+                    HeaderName::from_bytes(key.as_bytes()).map_err(|_| Error::InvalidHeader)?;
+                let value = HeaderValue::from_str(&value).map_err(|_| Error::InvalidHeader)?;
+                Ok((name, value))
+            })
+            .collect()
+    }
+}
+
+impl CredentialsProvider for GoogleCredentials {
+    async fn headers(
+        &self,
+        _: Extensions,
+    ) -> Result<CacheableResource<HeaderMap>, CredentialsError> {
+        self.request_headers()
+            .await
+            .map(|data| CacheableResource::New {
+                entity_tag: EntityTag::new(),
+                data,
+            })
+            .map_err(|_| CredentialsError::from_msg(false, "Google authentication failed"))
+    }
+
+    async fn universe_domain(&self) -> Option<String> {
+        None
+    }
+}
+
+impl std::fmt::Debug for GoogleCredentials {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("GoogleCredentials").finish_non_exhaustive()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn sdk_and_http_credentials_share_token_resolution_and_redaction() {
+        let credentials = GoogleCredentials::new(
+            VertexConfig::new(None, Some("project".into()), None),
+            Arc::new(|name| (name == "VERTEX_AI_API_KEY").then(|| "private-token".into())),
+        );
+        let direct = credentials.request_headers().await.unwrap();
+        let CacheableResource::New { data, .. } =
+            credentials.headers(Extensions::new()).await.unwrap()
+        else {
+            panic!("first request did not return headers");
+        };
+        assert_eq!(direct, data);
+        assert_eq!(data[http::header::AUTHORIZATION], "Bearer private-token");
+        assert!(!format!("{credentials:?}").contains("private-token"));
+    }
+
+    #[tokio::test]
+    async fn invalid_token_headers_return_a_redacted_sdk_error() {
+        let credentials = GoogleCredentials::new(
+            VertexConfig::new(None, Some("project".into()), None),
+            Arc::new(|name| (name == "VERTEX_AI_API_KEY").then(|| "private\nvalue".into())),
+        );
+        assert_eq!(
+            credentials.request_headers().await.unwrap_err(),
+            Error::InvalidHeader
+        );
+        let error = credentials.headers(Extensions::new()).await.unwrap_err();
+        assert!(!format!("{error:?}").contains("private"));
+    }
+}

litellm-rust/crates/auth-types/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "litellm-auth-types"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+serde.workspace = true
+subtle.workspace = true
+thiserror.workspace = true
+veil.workspace = true
+
+[dev-dependencies]
+tokio.workspace = true

litellm-rust/crates/auth-types/src/credential.rs

@@ -5,9 +5,7 @@ use std::sync::Arc;
 
 use veil::Redact;
 
-use crate::Error;
-
-use super::{ResolvedCredential, SecretValue, TokenProviderHandle};
+use crate::{Error, ResolvedCredential, SecretValue, TokenProviderHandle};
 
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub enum CredentialFileRef {

litellm-rust/crates/auth-types/src/http.rs

@@ -40,9 +40,6 @@ pub fn apply_credential(
     )
 }
 
-/// How the upstream call is authenticated. API-key strategies become headers
-/// in `prepare`; SigV4 covers the serialized body, so it is applied where the
-/// outbound request is built.
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub enum RequestAuth {
     Header {

litellm-rust/crates/auth-types/src/lib.rs

@@ -0,0 +1,57 @@
+#![forbid(unsafe_code)]
+
+mod credential;
+mod error;
+pub mod http;
+mod policy;
+mod secret;
+mod token;
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum InputSource {
+    Request,
+    #[default]
+    Deployment,
+    Environment,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
+pub struct Sourced<T> {
+    value: T,
+    source: InputSource,
+}
+
+impl<T> Sourced<T> {
+    pub fn new(value: T, source: InputSource) -> Self {
+        Self { value, source }
+    }
+
+    pub fn value(&self) -> &T {
+        &self.value
+    }
+
+    pub fn source(&self) -> InputSource {
+        self.source
+    }
+
+    pub fn into_value(self) -> T {
+        self.value
+    }
+
+    pub fn map<U>(self, map: impl FnOnce(T) -> U) -> Sourced<U> {
+        Sourced::new(map(self.value), self.source)
+    }
+}
+
+pub use credential::{
+    CredentialFileRef, CredentialLookup, CredentialLookupFuture, CredentialPlan,
+    CredentialPlanResolution, CredentialRef, CredentialResolver, CredentialResolverHandle,
+};
+pub use error::Error;
+pub use http::{CredentialPlacement, RequestAuth};
+pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy};
+pub use secret::SecretValue;
+pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle};

litellm-rust/crates/auth-types/src/policy.rs

@@ -1,7 +1,5 @@
-use crate::Error;
-
-use super::http::apply_credential;
-use super::{CredentialPlacement, ResolvedCredential};
+use crate::http::apply_credential;
+use crate::{CredentialPlacement, Error, ResolvedCredential};
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum CredentialPlanKind {

litellm-rust/crates/auth-types/src/secret.rs

@@ -1,4 +1,5 @@
 use serde::Deserialize;
+use std::hash::{Hash, Hasher};
 use veil::Redact;
 
 #[derive(Redact, Clone, Deserialize)]
@@ -23,6 +24,12 @@ impl PartialEq for SecretValue {
 
 impl Eq for SecretValue {}
 
+impl Hash for SecretValue {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.0.hash(state);
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::SecretValue;

litellm-rust/crates/auth-types/src/token.rs

@@ -5,9 +5,7 @@ use std::time::SystemTime;
 
 use veil::Redact;
 
-use crate::Error;
-
-use super::secret::SecretValue;
+use crate::{Error, SecretValue};
 
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub enum ResolvedCredential {

litellm-rust/crates/auth/Cargo.toml

@@ -5,11 +5,14 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 
-[dependencies]
-serde.workspace = true
-subtle.workspace = true
-thiserror.workspace = true
-veil.workspace = true
+[features]
+default = []
+aws = ["dep:litellm-auth-aws"]
+azure = ["dep:litellm-auth-azure"]
+gcp = ["dep:litellm-auth-gcp"]
 
-[dev-dependencies]
-tokio.workspace = true
+[dependencies]
+litellm-auth-types.workspace = true
+litellm-auth-aws = { workspace = true, optional = true }
+litellm-auth-azure = { workspace = true, optional = true }
+litellm-auth-gcp = { workspace = true, optional = true }

litellm-rust/crates/auth/src/lib.rs

@@ -1,55 +1,10 @@
-mod credential;
-mod error;
-pub mod http;
-mod policy;
-mod secret;
-mod token;
+#![forbid(unsafe_code)]
 
-use serde::{Deserialize, Serialize};
+pub use litellm_auth_types::*;
 
-#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)]
-#[serde(rename_all = "snake_case")]
-pub enum InputSource {
-    Request,
-    #[default]
-    Deployment,
-    Environment,
-}
-
-#[derive(Clone, Debug, Default, Eq, PartialEq)]
-pub struct Sourced<T> {
-    value: T,
-    source: InputSource,
-}
-
-impl<T> Sourced<T> {
-    pub fn new(value: T, source: InputSource) -> Self {
-        Self { value, source }
-    }
-
-    pub fn value(&self) -> &T {
-        &self.value
-    }
-
-    pub fn source(&self) -> InputSource {
-        self.source
-    }
-
-    pub fn into_value(self) -> T {
-        self.value
-    }
-
-    pub fn map<U>(self, map: impl FnOnce(T) -> U) -> Sourced<U> {
-        Sourced::new(map(self.value), self.source)
-    }
-}
-
-pub use credential::{
-    CredentialFileRef, CredentialLookup, CredentialLookupFuture, CredentialPlan,
-    CredentialPlanResolution, CredentialRef, CredentialResolver, CredentialResolverHandle,
-};
-pub use error::Error;
-pub use http::{CredentialPlacement, RequestAuth};
-pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy};
-pub use secret::SecretValue;
-pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle};
+#[cfg(feature = "aws")]
+pub use litellm_auth_aws as aws;
+#[cfg(feature = "azure")]
+pub use litellm_auth_azure as azure;
+#[cfg(feature = "gcp")]
+pub use litellm_auth_gcp as gcp;

litellm-rust/crates/auth/tests/facade.rs

@@ -0,0 +1,33 @@
+use litellm_auth::{
+    CredentialPlacement, CredentialPlanKind, CredentialRule, ExistingHeaderBehavior,
+    ProviderAuthPolicy, ResolvedCredential, SecretValue,
+};
+
+const RULES: &[CredentialRule] = &[CredentialRule {
+    kind: CredentialPlanKind::Static,
+    placement: CredentialPlacement::Header("x-api-key"),
+}];
+
+#[test]
+fn facade_applies_shared_auth_policy() {
+    let policy = ProviderAuthPolicy {
+        rules: RULES,
+        accepted_existing_headers: &["x-api-key"],
+        existing_header_behavior: ExistingHeaderBehavior::Preserve,
+        scope: None,
+        audience: None,
+    };
+
+    let headers = policy
+        .apply(
+            Vec::new(),
+            CredentialPlanKind::Static,
+            &ResolvedCredential::Static(SecretValue::new("secret")),
+        )
+        .expect("facade policy applies");
+
+    assert_eq!(
+        headers,
+        vec![("x-api-key".to_string(), "secret".to_string())]
+    );
+}

litellm-rust/crates/cache-azure-blob/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "litellm-cache-azure-blob"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-auth-azure.workspace = true
+litellm-auth-types.workspace = true
+litellm-cache.workspace = true
+
+async-trait = "0.1"
+azure_core = "1.1.0"
+azure_storage_blob = "1.1.0"
+futures-util.workspace = true
+tokio.workspace = true
+url.workspace = true
+
+[dev-dependencies]
+litellm-cache-response.workspace = true
+serde_json.workspace = true

litellm-rust/crates/cache-azure-blob/src/cache.rs

@@ -0,0 +1,254 @@
+use std::{sync::Arc, time::Duration};
+
+use azure_core::{
+    credentials::TokenCredential,
+    error::ErrorKind,
+    http::{ClientOptions, RequestContent},
+};
+use azure_storage_blob::{
+    BlobContainerClient, BlobContainerClientOptions,
+    models::{BlobClientUploadOptions, StorageErrorCode},
+};
+use futures_util::{TryStreamExt, future::try_join_all};
+use litellm_cache::{
+    BaseCache, BatchCache, CacheCodec, CacheConnectionResult, CacheConnectionStatus, Error,
+    ExactCacheContext, FlushCache,
+};
+use tokio::runtime::Handle;
+use url::Url;
+
+use crate::credential::AzureBlobCredential;
+
+pub struct AzureBlobCache<C> {
+    container: BlobContainerClient,
+    codec: C,
+    runtime: Handle,
+    account_url: String,
+    container_name: String,
+}
+
+impl<C: CacheCodec> AzureBlobCache<C> {
+    pub async fn connect(
+        account_url: &str,
+        container: &str,
+        codec: C,
+        runtime: Handle,
+    ) -> Result<Self, Error> {
+        Self::connect_with_options(
+            account_url,
+            container,
+            Some(Arc::new(AzureBlobCredential::default())),
+            ClientOptions::default(),
+            codec,
+            runtime,
+        )
+        .await
+    }
+
+    pub async fn connect_with_options(
+        account_url: &str,
+        container: &str,
+        credential: Option<Arc<dyn TokenCredential>>,
+        client_options: ClientOptions,
+        codec: C,
+        runtime: Handle,
+    ) -> Result<Self, Error> {
+        let parsed = Url::parse(account_url).map_err(|_| Error::Unavailable)?;
+        let account_url = parsed.as_str().trim_end_matches('/').to_string();
+        let container_url = {
+            let mut url = parsed;
+            url.path_segments_mut()
+                .map_err(|()| Error::Unavailable)?
+                .pop_if_empty()
+                .push(container);
+            url
+        };
+        let client = BlobContainerClient::new(
+            container_url,
+            credential,
+            Some(BlobContainerClientOptions {
+                client_options,
+                ..BlobContainerClientOptions::default()
+            }),
+        )
+        .map_err(|_| Error::Unavailable)?;
+        let cache = Self {
+            container: client,
+            codec,
+            runtime,
+            account_url,
+            container_name: container.to_string(),
+        };
+        cache.create_container().await?;
+        Ok(cache)
+    }
+
+    pub fn account_url(&self) -> &str {
+        &self.account_url
+    }
+
+    pub fn container_name(&self) -> &str {
+        &self.container_name
+    }
+
+    async fn create_container(&self) -> Result<(), Error> {
+        match self.container.create(None).await {
+            Ok(_) => Ok(()),
+            Err(error) if is_storage_error(&error, StorageErrorCode::ContainerAlreadyExists) => {
+                Ok(())
+            }
+            Err(_) => Err(Error::Unavailable),
+        }
+    }
+
+    async fn upload(&self, key: &str, value: &C::Value, overwrite: bool) -> Result<(), Error> {
+        let payload = self.codec.encode(value)?;
+        let options = (!overwrite).then(|| BlobClientUploadOptions::default().if_not_exists());
+        match self
+            .container
+            .blob_client(key)
+            .upload(RequestContent::from(payload), options)
+            .await
+        {
+            Ok(_) => Ok(()),
+            Err(error) if !overwrite && is_already_present(&error) => Ok(()),
+            Err(_) => Err(Error::Unavailable),
+        }
+    }
+
+    async fn download(&self, key: &str) -> Result<Option<C::Value>, Error> {
+        let response = match self.container.blob_client(key).download(None).await {
+            Ok(response) => response,
+            Err(error) if is_storage_error(&error, StorageErrorCode::BlobNotFound) => {
+                return Ok(None);
+            }
+            Err(_) => return Err(Error::Unavailable),
+        };
+        let bytes = response
+            .body
+            .collect()
+            .await
+            .map_err(|_| Error::Unavailable)?;
+        self.codec.decode(&bytes).map(Some)
+    }
+
+    async fn delete_all_blobs(&self) -> Result<(), Error> {
+        let mut pages = self
+            .container
+            .list_blobs(None)
+            .map_err(|_| Error::Unavailable)?
+            .into_pages();
+        while let Some(page) = pages.try_next().await.map_err(|_| Error::Unavailable)? {
+            let page = page.into_model().map_err(|_| Error::Unavailable)?;
+            for name in page.blob_items.into_iter().filter_map(|item| item.name) {
+                self.container
+                    .blob_client(&name)
+                    .delete(None)
+                    .await
+                    .map_err(|_| Error::Unavailable)?;
+            }
+        }
+        Ok(())
+    }
+
+    fn block_on<T>(&self, future: impl Future<Output = T>) -> T {
+        self.runtime.block_on(future)
+    }
+}
+
+fn is_already_present(error: &azure_core::Error) -> bool {
+    is_storage_error(error, StorageErrorCode::BlobAlreadyExists)
+        || is_storage_error(error, StorageErrorCode::ConditionNotMet)
+}
+
+fn is_storage_error(error: &azure_core::Error, code: StorageErrorCode) -> bool {
+    matches!(
+        error.kind(),
+        ErrorKind::HttpResponse {
+            error_code: Some(error_code),
+            ..
+        } if error_code == code.as_ref()
+    )
+}
+
+impl<C: CacheCodec> BaseCache for AzureBlobCache<C> {
+    type Value = C::Value;
+    type Context = ExactCacheContext;
+
+    fn get_ttl(&self, _: &ExactCacheContext) -> Option<Duration> {
+        None
+    }
+
+    fn set_cache(&self, key: &str, value: C::Value, _: &ExactCacheContext) -> Result<(), Error> {
+        self.block_on(self.upload(key, &value, false))
+    }
+
+    fn get_cache(&self, key: &str, _: &ExactCacheContext) -> Result<Option<C::Value>, Error> {
+        self.block_on(self.download(key))
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: C::Value,
+        _: ExactCacheContext,
+    ) -> Result<(), Error> {
+        self.upload(key, &value, true).await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        _: &ExactCacheContext,
+    ) -> Result<Option<C::Value>, Error> {
+        self.download(key).await
+    }
+
+    async fn async_set_cache_pipeline(
+        &self,
+        entries: Vec<(String, C::Value)>,
+        _: ExactCacheContext,
+    ) -> Result<(), Error> {
+        try_join_all(
+            entries
+                .iter()
+                .map(|(key, value)| self.upload(key, value, true)),
+        )
+        .await
+        .map(drop)
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        Ok(match self.container.get_properties(None).await {
+            Ok(_) => CacheConnectionResult {
+                status: CacheConnectionStatus::Success,
+                message: "Azure Blob cache connection test successful".into(),
+                error: None,
+            },
+            Err(error) => CacheConnectionResult {
+                status: CacheConnectionStatus::Failed,
+                message: format!("Azure Blob connection failed: {error}"),
+                error: Some(error.to_string()),
+            },
+        })
+    }
+}
+
+impl<C: CacheCodec> BatchCache for AzureBlobCache<C> {}
+
+impl<C: CacheCodec> FlushCache for AzureBlobCache<C> {
+    fn flush_cache(&self) -> Result<(), Error> {
+        self.block_on(self.delete_all_blobs())
+    }
+
+    async fn async_flush_cache(&self) -> Result<(), Error> {
+        self.delete_all_blobs().await
+    }
+}
+
+#[cfg(test)]
+mod tests;

litellm-rust/crates/cache-azure-blob/src/cache/tests.rs

@@ -0,0 +1,746 @@
+use std::{
+    collections::BTreeMap,
+    sync::{Arc, Mutex},
+    time::Duration,
+};
+
+use azure_core::http::{
+    AsyncRawResponse, Body, ClientOptions, HttpClient, Method, Request, StatusCode, Transport,
+    headers::{HeaderName, Headers},
+};
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheConnectionStatus, Error, ExactCacheContext, FlushCache,
+};
+use litellm_cache_response::{
+    CacheEntry, CacheKeyField, CacheKeyInput, ResponseCache, ResponseCacheCodec,
+    ResponseCacheRequest, cache_key,
+};
+use serde_json::json;
+use tokio::runtime::Runtime;
+
+use super::AzureBlobCache;
+
+const ACCOUNT_URL: &str = "https://example.blob.core.windows.net";
+const CONTAINER: &str = "litellm-cache";
+const IF_NONE_MATCH: HeaderName = HeaderName::from_static("if-none-match");
+const ERROR_CODE: HeaderName = HeaderName::from_static("x-ms-error-code");
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct RecordedRequest {
+    method: Method,
+    path: String,
+    query: String,
+    if_none_match: Option<String>,
+}
+
+#[derive(Default)]
+struct FakeState {
+    container_exists: bool,
+    blobs: BTreeMap<String, Vec<u8>>,
+    requests: Vec<RecordedRequest>,
+    failing: bool,
+    precondition_conflicts: bool,
+}
+
+#[derive(Clone, Default)]
+struct FakeBlobService {
+    state: Arc<Mutex<FakeState>>,
+}
+
+impl std::fmt::Debug for FakeBlobService {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("FakeBlobService")
+    }
+}
+
+impl FakeBlobService {
+    fn with_existing_container() -> Self {
+        let service = Self::default();
+        service.state.lock().unwrap().container_exists = true;
+        service
+    }
+
+    fn blob(&self, name: &str) -> Option<Vec<u8>> {
+        self.state.lock().unwrap().blobs.get(name).cloned()
+    }
+
+    fn blob_names(&self) -> Vec<String> {
+        self.state.lock().unwrap().blobs.keys().cloned().collect()
+    }
+
+    fn seed_blob(&self, name: &str, bytes: &[u8]) {
+        self.state
+            .lock()
+            .unwrap()
+            .blobs
+            .insert(name.to_string(), bytes.to_vec());
+    }
+
+    fn set_failing(&self, failing: bool) {
+        self.state.lock().unwrap().failing = failing;
+    }
+
+    fn set_precondition_conflicts(&self, enabled: bool) {
+        self.state.lock().unwrap().precondition_conflicts = enabled;
+    }
+
+    fn requests(&self) -> Vec<RecordedRequest> {
+        self.state.lock().unwrap().requests.clone()
+    }
+
+    fn container_exists(&self) -> bool {
+        self.state.lock().unwrap().container_exists
+    }
+
+    fn respond(status: StatusCode, error_code: Option<&str>, body: Vec<u8>) -> AsyncRawResponse {
+        let mut headers = Headers::new();
+        if let Some(code) = error_code {
+            headers.insert(ERROR_CODE, code.to_string());
+        }
+        AsyncRawResponse::from_bytes(status, headers, body)
+    }
+
+    fn list_body(state: &FakeState) -> Vec<u8> {
+        let mut xml = String::from(
+            r#"<?xml version="1.0" encoding="utf-8"?><EnumerationResults ServiceEndpoint="https://example.blob.core.windows.net/" ContainerName="litellm-cache"><Blobs>"#,
+        );
+        for name in state.blobs.keys() {
+            xml.push_str(&format!(
+                "<Blob><Name>{name}</Name><Properties><BlobType>BlockBlob</BlobType></Properties></Blob>"
+            ));
+        }
+        xml.push_str("</Blobs><NextMarker /></EnumerationResults>");
+        xml.into_bytes()
+    }
+}
+
+#[async_trait::async_trait]
+impl HttpClient for FakeBlobService {
+    async fn execute_request(&self, request: &Request) -> azure_core::Result<AsyncRawResponse> {
+        let mut state = self.state.lock().unwrap();
+        let path = request.url().path().to_string();
+        let query = request.url().query().unwrap_or_default().to_string();
+        let if_none_match = request
+            .headers()
+            .get_optional_str(&IF_NONE_MATCH)
+            .map(str::to_owned);
+        state.requests.push(RecordedRequest {
+            method: request.method(),
+            path: path.clone(),
+            query: query.clone(),
+            if_none_match: if_none_match.clone(),
+        });
+        if state.failing {
+            return Ok(Self::respond(
+                StatusCode::Forbidden,
+                Some("AuthorizationFailure"),
+                Vec::new(),
+            ));
+        }
+        let container_path = format!("/{CONTAINER}");
+        let blob_name = path
+            .strip_prefix(&format!("{container_path}/"))
+            .map(str::to_owned);
+        let is_container = path == container_path && query.contains("restype=container");
+        let response = match (request.method(), is_container, blob_name) {
+            (Method::Put, true, None) if state.container_exists => Self::respond(
+                StatusCode::Conflict,
+                Some("ContainerAlreadyExists"),
+                Vec::new(),
+            ),
+            (Method::Put, true, None) => {
+                state.container_exists = true;
+                Self::respond(StatusCode::Created, None, Vec::new())
+            }
+            (Method::Get, true, None) if query.contains("comp=list") => {
+                Self::respond(StatusCode::Ok, None, Self::list_body(&state))
+            }
+            (Method::Get, true, None) if state.container_exists => {
+                Self::respond(StatusCode::Ok, None, Vec::new())
+            }
+            (Method::Get, true, None) => {
+                Self::respond(StatusCode::NotFound, Some("ContainerNotFound"), Vec::new())
+            }
+            (Method::Put, false, Some(name)) => {
+                if if_none_match.as_deref() == Some("*") && state.blobs.contains_key(&name) {
+                    if state.precondition_conflicts {
+                        Self::respond(
+                            StatusCode::PreconditionFailed,
+                            Some("ConditionNotMet"),
+                            Vec::new(),
+                        )
+                    } else {
+                        Self::respond(StatusCode::Conflict, Some("BlobAlreadyExists"), Vec::new())
+                    }
+                } else {
+                    let bytes = match request.body() {
+                        Body::Bytes(bytes) => bytes.to_vec(),
+                        Body::SeekableStream(_) => panic!("unexpected streaming upload"),
+                    };
+                    state.blobs.insert(name, bytes);
+                    Self::respond(StatusCode::Created, None, Vec::new())
+                }
+            }
+            (Method::Get, false, Some(name)) => match state.blobs.get(&name) {
+                Some(bytes) => Self::respond(StatusCode::Ok, None, bytes.clone()),
+                None => Self::respond(StatusCode::NotFound, Some("BlobNotFound"), Vec::new()),
+            },
+            (Method::Delete, false, Some(name)) => match state.blobs.remove(&name) {
+                Some(_) => Self::respond(StatusCode::Accepted, None, Vec::new()),
+                None => Self::respond(StatusCode::NotFound, Some("BlobNotFound"), Vec::new()),
+            },
+            (method, _, _) => panic!("unexpected request {method:?} {path}?{query}"),
+        };
+        Ok(response)
+    }
+}
+
+struct Fixture {
+    runtime: Runtime,
+    service: FakeBlobService,
+    cache: Arc<AzureBlobCache<ResponseCacheCodec>>,
+}
+
+impl Fixture {
+    fn new(service: FakeBlobService) -> Self {
+        let runtime = Runtime::new().unwrap();
+        let cache = runtime
+            .block_on(Self::connect(&service, runtime.handle().clone()))
+            .unwrap();
+        Self {
+            runtime,
+            service,
+            cache: Arc::new(cache),
+        }
+    }
+
+    async fn connect(
+        service: &FakeBlobService,
+        handle: tokio::runtime::Handle,
+    ) -> Result<AzureBlobCache<ResponseCacheCodec>, Error> {
+        AzureBlobCache::connect_with_options(
+            ACCOUNT_URL,
+            CONTAINER,
+            None,
+            ClientOptions {
+                transport: Some(Transport::new(Arc::new(service.clone()))),
+                ..ClientOptions::default()
+            },
+            ResponseCacheCodec,
+            handle,
+        )
+        .await
+    }
+
+    fn response_cache(&self) -> ResponseCache<AzureBlobCache<ResponseCacheCodec>> {
+        ResponseCache::new(self.cache.clone())
+    }
+
+    fn stored_json(&self, key: &str) -> serde_json::Value {
+        serde_json::from_slice(&self.service.blob(key).expect("blob should exist")).unwrap()
+    }
+}
+
+fn request(model: &str) -> ResponseCacheRequest {
+    ResponseCacheRequest::new(CacheKeyInput {
+        fields: vec![CacheKeyField {
+            name: "model".into(),
+            value: Some(model.into()),
+            api_parameter: true,
+            internal_parameter: false,
+        }],
+        preset: None,
+        namespace: None,
+        include_provider_parameters: false,
+    })
+}
+
+fn now() -> Duration {
+    Duration::from_secs(1_700_000_000)
+}
+
+fn entry(value: serde_json::Value) -> CacheEntry {
+    CacheEntry {
+        timestamp: Some(1_700_000_000.5),
+        response: value,
+    }
+}
+
+fn no_ttl() -> ExactCacheContext {
+    ExactCacheContext::default()
+}
+
+fn with_ttl(seconds: u64) -> ExactCacheContext {
+    ExactCacheContext {
+        ttl: Some(Duration::from_secs(seconds)),
+    }
+}
+
+#[test]
+fn connect_creates_the_container_once() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    assert!(fixture.service.container_exists());
+    assert_eq!(
+        fixture.service.requests(),
+        vec![RecordedRequest {
+            method: Method::Put,
+            path: format!("/{CONTAINER}"),
+            query: "restype=container".into(),
+            if_none_match: None,
+        }]
+    );
+    assert_eq!(fixture.cache.account_url(), ACCOUNT_URL);
+    assert_eq!(fixture.cache.container_name(), CONTAINER);
+}
+
+#[test]
+fn connect_accepts_an_existing_container() {
+    let fixture = Fixture::new(FakeBlobService::with_existing_container());
+    assert!(fixture.service.container_exists());
+    assert_eq!(fixture.service.requests().len(), 1);
+}
+
+#[test]
+fn connect_accepts_account_urls_with_trailing_slash() {
+    let runtime = Runtime::new().unwrap();
+    let service = FakeBlobService::default();
+    let cache = runtime
+        .block_on(AzureBlobCache::connect_with_options(
+            "https://example.blob.core.windows.net/",
+            CONTAINER,
+            None,
+            ClientOptions {
+                transport: Some(Transport::new(Arc::new(service.clone()))),
+                ..ClientOptions::default()
+            },
+            ResponseCacheCodec,
+            runtime.handle().clone(),
+        ))
+        .unwrap();
+    assert_eq!(service.requests()[0].path, format!("/{CONTAINER}"));
+    assert_eq!(cache.account_url(), "https://example.blob.core.windows.net");
+}
+
+#[test]
+fn connect_keeps_account_url_query_parameters_on_the_container_path() {
+    let runtime = Runtime::new().unwrap();
+    let service = FakeBlobService::default();
+    runtime
+        .block_on(AzureBlobCache::connect_with_options(
+            "https://example.blob.core.windows.net/?sv=2024-01-01&sig=abc",
+            CONTAINER,
+            None,
+            ClientOptions {
+                transport: Some(Transport::new(Arc::new(service.clone()))),
+                ..ClientOptions::default()
+            },
+            ResponseCacheCodec,
+            runtime.handle().clone(),
+        ))
+        .unwrap();
+    let create = &service.requests()[0];
+    assert_eq!(create.path, format!("/{CONTAINER}"));
+    assert!(create.query.contains("sig=abc"));
+}
+
+#[test]
+fn connect_surfaces_service_failures() {
+    let runtime = Runtime::new().unwrap();
+    let service = FakeBlobService::default();
+    service.set_failing(true);
+    let result = runtime.block_on(Fixture::connect(&service, runtime.handle().clone()));
+    assert!(matches!(result, Err(Error::Unavailable)));
+}
+
+#[test]
+fn sync_set_and_get_round_trip_python_json_shape() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    let value = entry(json!({"choices": [{"message": {"content": "héllo 🌍"}}]}));
+    fixture
+        .cache
+        .set_cache("key-1", value.clone(), &no_ttl())
+        .unwrap();
+
+    assert_eq!(
+        fixture.stored_json("key-1"),
+        json!({
+            "timestamp": 1_700_000_000.5,
+            "response": {"choices": [{"message": {"content": "héllo 🌍"}}]}
+        })
+    );
+    assert_eq!(
+        fixture.cache.get_cache("key-1", &no_ttl()).unwrap(),
+        Some(value)
+    );
+}
+
+#[test]
+fn sync_set_does_not_overwrite_an_existing_blob() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture
+        .cache
+        .set_cache("key", entry(json!({"v": "first"})), &no_ttl())
+        .unwrap();
+    fixture
+        .cache
+        .set_cache("key", entry(json!({"v": "second"})), &no_ttl())
+        .unwrap();
+
+    assert_eq!(
+        fixture.stored_json("key")["response"],
+        json!({"v": "first"})
+    );
+    let uploads: Vec<_> = fixture
+        .service
+        .requests()
+        .into_iter()
+        .filter(|request| request.method == Method::Put && request.path.ends_with("/key"))
+        .collect();
+    assert_eq!(uploads.len(), 2);
+    assert!(
+        uploads
+            .iter()
+            .all(|request| request.if_none_match.as_deref() == Some("*"))
+    );
+}
+
+#[test]
+fn sync_set_treats_a_precondition_conflict_as_an_existing_blob() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture.service.set_precondition_conflicts(true);
+    fixture
+        .cache
+        .set_cache("key", entry(json!({"v": "first"})), &no_ttl())
+        .unwrap();
+    fixture
+        .cache
+        .set_cache("key", entry(json!({"v": "second"})), &no_ttl())
+        .unwrap();
+
+    assert_eq!(
+        fixture.stored_json("key")["response"],
+        json!({"v": "first"})
+    );
+}
+
+#[test]
+fn async_set_overwrites_an_existing_blob() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture.runtime.block_on(async {
+        fixture
+            .cache
+            .async_set_cache("key", entry(json!({"v": "first"})), no_ttl())
+            .await
+            .unwrap();
+        fixture
+            .cache
+            .async_set_cache("key", entry(json!({"v": "second"})), no_ttl())
+            .await
+            .unwrap();
+        assert_eq!(
+            fixture
+                .cache
+                .async_get_cache("key", &no_ttl())
+                .await
+                .unwrap(),
+            Some(entry(json!({"v": "second"})))
+        );
+    });
+    assert_eq!(
+        fixture.stored_json("key")["response"],
+        json!({"v": "second"})
+    );
+    assert!(
+        fixture
+            .service
+            .requests()
+            .iter()
+            .filter(|request| request.method == Method::Put && request.path.ends_with("/key"))
+            .all(|request| request.if_none_match.is_none())
+    );
+}
+
+#[test]
+fn missing_blobs_are_misses() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    assert_eq!(fixture.cache.get_cache("absent", &no_ttl()).unwrap(), None);
+    assert_eq!(
+        fixture
+            .runtime
+            .block_on(fixture.cache.async_get_cache("absent", &no_ttl()))
+            .unwrap(),
+        None
+    );
+}
+
+#[test]
+fn ttl_is_ignored_and_entries_never_expire() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    assert_eq!(fixture.cache.get_ttl(&with_ttl(1)), None);
+    assert_eq!(fixture.cache.get_ttl(&no_ttl()), None);
+
+    fixture
+        .cache
+        .set_cache("key", entry(json!("value")), &with_ttl(1))
+        .unwrap();
+    std::thread::sleep(Duration::from_millis(1100));
+    assert_eq!(
+        fixture.cache.get_cache("key", &with_ttl(1)).unwrap(),
+        Some(entry(json!("value")))
+    );
+    assert!(
+        fixture
+            .service
+            .requests()
+            .iter()
+            .all(|request| !request.query.contains("expiry"))
+    );
+}
+
+#[test]
+fn malformed_blobs_are_invalid_entries_and_response_cache_misses() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture.service.seed_blob("broken-json", b"{not json");
+    fixture
+        .service
+        .seed_blob("broken-utf8", &[0xff, 0xfe, 0x22]);
+    fixture
+        .service
+        .seed_blob("wrong-shape", br#"{"timestamp": "yesterday"}"#);
+
+    for key in ["broken-json", "broken-utf8", "wrong-shape"] {
+        assert!(matches!(
+            fixture.cache.get_cache(key, &no_ttl()),
+            Err(Error::InvalidEntry)
+        ));
+    }
+
+    let response_cache = fixture.response_cache();
+    let broken = request("broken");
+    fixture
+        .service
+        .seed_blob(&cache_key(&broken.key), b"{not json");
+    assert_eq!(response_cache.lookup(&broken, now()).unwrap(), None);
+    assert_eq!(
+        fixture
+            .runtime
+            .block_on(response_cache.async_lookup(&broken, now()))
+            .unwrap(),
+        None
+    );
+}
+
+#[test]
+fn batch_get_preserves_order_and_marks_misses_and_invalid_entries() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture
+        .cache
+        .set_cache("a", entry(json!("A")), &no_ttl())
+        .unwrap();
+    fixture
+        .cache
+        .set_cache("c", entry(json!("C")), &no_ttl())
+        .unwrap();
+    fixture.service.seed_blob("bad", b"nope");
+    let keys = ["c", "missing", "a", "bad"].map(String::from);
+
+    let sync = fixture.cache.batch_get_cache(&keys, &no_ttl()).unwrap();
+    assert_eq!(
+        sync,
+        vec![
+            BatchEntry::Hit(entry(json!("C"))),
+            BatchEntry::Miss,
+            BatchEntry::Hit(entry(json!("A"))),
+            BatchEntry::Invalid,
+        ]
+    );
+
+    let asynchronous = fixture
+        .runtime
+        .block_on(fixture.cache.async_batch_get_cache(keys.to_vec(), no_ttl()))
+        .unwrap();
+    assert_eq!(asynchronous, sync);
+
+    let response_cache = fixture.response_cache();
+    let requests = [request("hit"), request("missing"), request("bad")];
+    response_cache
+        .store(&requests[0], json!("HIT"), now())
+        .unwrap();
+    fixture
+        .service
+        .seed_blob(&cache_key(&requests[2].key), b"nope");
+    let hits = response_cache.lookup_batch(&requests, now()).unwrap();
+    assert_eq!(hits.values, vec![Some(json!("HIT")), None, None]);
+    assert_eq!(hits.missing_indices, vec![1, 2]);
+    let async_hits = fixture
+        .runtime
+        .block_on(response_cache.async_lookup_batch(&requests, now()))
+        .unwrap();
+    assert_eq!(async_hits.values, hits.values);
+}
+
+#[test]
+fn async_pipeline_writes_every_entry_with_overwrite() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture.service.seed_blob("k2", b"stale");
+    fixture
+        .runtime
+        .block_on(fixture.cache.async_set_cache_pipeline(
+            vec![
+                ("k1".into(), entry(json!({"n": 1}))),
+                ("k2".into(), entry(json!({"n": 2}))),
+                ("k3".into(), entry(json!({"n": 3}))),
+            ],
+            with_ttl(30),
+        ))
+        .unwrap();
+    assert_eq!(fixture.service.blob_names(), ["k1", "k2", "k3"]);
+    assert_eq!(fixture.stored_json("k2")["response"], json!({"n": 2}));
+}
+
+#[test]
+fn flush_deletes_every_blob_in_the_container() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    for key in ["x", "y", "z"] {
+        fixture
+            .cache
+            .set_cache(key, entry(json!(key)), &no_ttl())
+            .unwrap();
+    }
+    fixture.cache.flush_cache().unwrap();
+    assert!(fixture.service.blob_names().is_empty());
+    assert!(fixture.service.container_exists());
+
+    fixture
+        .cache
+        .set_cache("again", entry(json!(1)), &no_ttl())
+        .unwrap();
+    fixture
+        .runtime
+        .block_on(fixture.cache.async_flush_cache())
+        .unwrap();
+    assert!(fixture.service.blob_names().is_empty());
+}
+
+#[test]
+fn service_failures_map_to_unavailable() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture.service.set_failing(true);
+    assert!(matches!(
+        fixture.cache.get_cache("key", &no_ttl()),
+        Err(Error::Unavailable)
+    ));
+    assert!(matches!(
+        fixture.cache.set_cache("key", entry(json!(1)), &no_ttl()),
+        Err(Error::Unavailable)
+    ));
+    assert!(matches!(
+        fixture.cache.flush_cache(),
+        Err(Error::Unavailable)
+    ));
+    assert!(matches!(
+        fixture.runtime.block_on(
+            fixture
+                .cache
+                .async_set_cache_pipeline(vec![("k".into(), entry(json!(1)))], no_ttl())
+        ),
+        Err(Error::Unavailable)
+    ));
+}
+
+#[test]
+fn test_connection_reports_container_reachability() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    let ok = fixture
+        .runtime
+        .block_on(fixture.cache.test_connection())
+        .unwrap();
+    assert_eq!(ok.status, CacheConnectionStatus::Success);
+    assert!(ok.error.is_none());
+
+    fixture.service.set_failing(true);
+    let failed = fixture
+        .runtime
+        .block_on(fixture.cache.test_connection())
+        .unwrap();
+    assert_eq!(failed.status, CacheConnectionStatus::Failed);
+    assert!(failed.error.is_some());
+}
+
+#[test]
+fn disconnect_is_idempotent_and_keeps_data() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture
+        .cache
+        .set_cache("key", entry(json!(1)), &no_ttl())
+        .unwrap();
+    fixture.runtime.block_on(async {
+        fixture.cache.disconnect().await.unwrap();
+        fixture.cache.disconnect().await.unwrap();
+    });
+    assert_eq!(
+        fixture.cache.get_cache("key", &no_ttl()).unwrap(),
+        Some(entry(json!(1)))
+    );
+}
+
+#[test]
+fn response_cache_stores_and_reads_through_the_backend() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    let response_cache = fixture.response_cache();
+    let mut request = request("gpt");
+    request.context = with_ttl(60);
+    let response = json!({"id": "chatcmpl-1"});
+    response_cache
+        .store(&request, response.clone(), now())
+        .unwrap();
+    assert_eq!(
+        fixture.stored_json(&cache_key(&request.key)),
+        json!({"timestamp": 1_700_000_000.0, "response": {"id": "chatcmpl-1"}})
+    );
+    assert_eq!(
+        response_cache
+            .lookup(&request, now() + Duration::from_secs(3600))
+            .unwrap(),
+        Some(response.clone())
+    );
+    assert_eq!(
+        fixture
+            .runtime
+            .block_on(response_cache.async_lookup(&request, now() + Duration::from_secs(3600)))
+            .unwrap(),
+        Some(response.clone())
+    );
+    fixture.runtime.block_on(async {
+        response_cache
+            .async_store(&request, json!("replaced"), now())
+            .await
+            .unwrap();
+        assert_eq!(
+            response_cache.async_lookup(&request, now()).await.unwrap(),
+            Some(json!("replaced"))
+        );
+        response_cache.async_flush().await.unwrap();
+        assert_eq!(
+            response_cache.async_lookup(&request, now()).await.unwrap(),
+            None
+        );
+    });
+}
+
+#[test]
+fn non_object_responses_are_written_serialized_like_python() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture
+        .cache
+        .set_cache("s", entry(json!("plain")), &no_ttl())
+        .unwrap();
+    assert_eq!(
+        fixture.stored_json("s"),
+        json!({"timestamp": 1_700_000_000.5, "response": "\"plain\""})
+    );
+    assert_eq!(
+        fixture.cache.get_cache("s", &no_ttl()).unwrap(),
+        Some(entry(json!("plain")))
+    );
+}

litellm-rust/crates/cache-azure-blob/src/credential.rs

@@ -0,0 +1,84 @@
+use std::{
+    fmt,
+    sync::Arc,
+    time::{Duration, SystemTime},
+};
+
+use azure_core::{
+    credentials::{AccessToken, TokenCredential, TokenRequestOptions},
+    error::ErrorKind,
+    time::OffsetDateTime,
+};
+use litellm_auth_azure::{AzureAuthInputs, AzureAuthService};
+use litellm_auth_types::ResolvedCredential;
+
+const STATIC_TOKEN_LIFETIME: Duration = Duration::from_secs(300);
+const LLM_TOKEN_ENV: &str = "AZURE_AD_TOKEN";
+
+type EnvLookup = Arc<dyn Fn(&str) -> Option<String> + Send + Sync>;
+
+pub struct AzureBlobCredential {
+    service: AzureAuthService,
+    env_lookup: EnvLookup,
+}
+
+impl fmt::Debug for AzureBlobCredential {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str("AzureBlobCredential")
+    }
+}
+
+impl Default for AzureBlobCredential {
+    fn default() -> Self {
+        Self::new(
+            AzureAuthService::default(),
+            Arc::new(|name| std::env::var(name).ok()),
+        )
+    }
+}
+
+impl AzureBlobCredential {
+    pub fn new(service: AzureAuthService, env_lookup: EnvLookup) -> Self {
+        Self {
+            service,
+            env_lookup,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl TokenCredential for AzureBlobCredential {
+    async fn get_token(
+        &self,
+        scopes: &[&str],
+        _options: Option<TokenRequestOptions<'_>>,
+    ) -> azure_core::Result<AccessToken> {
+        let env_lookup = &self.env_lookup;
+        let lookup = move |name: &str| (name != LLM_TOKEN_ENV).then(|| env_lookup(name)).flatten();
+        let credential = self
+            .service
+            .get_azure_ad_token(
+                &AzureAuthInputs::default_credential_for_scope(&scopes.join(" ")),
+                &lookup,
+            )
+            .await
+            .map_err(|error| {
+                azure_core::Error::with_message(ErrorKind::Credential, error.to_string())
+            })?
+            .ok_or_else(|| {
+                azure_core::Error::with_message(
+                    ErrorKind::Credential,
+                    "no Azure credential is available for blob storage",
+                )
+            })?;
+        let (token, expires_on) = match credential.into_value() {
+            ResolvedCredential::AccessToken { token, expires_on } => (token, expires_on),
+            ResolvedCredential::Static(token) => (token, None),
+        };
+        let expires_on = expires_on.unwrap_or_else(|| SystemTime::now() + STATIC_TOKEN_LIFETIME);
+        Ok(AccessToken::new(
+            token.expose().to_string(),
+            OffsetDateTime::from(expires_on),
+        ))
+    }
+}

litellm-rust/crates/cache-azure-blob/src/lib.rs

@@ -0,0 +1,5 @@
+mod cache;
+mod credential;
+
+pub use cache::AzureBlobCache;
+pub use credential::AzureBlobCredential;

litellm-rust/crates/cache-disk/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "litellm-cache-disk"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-cache.workspace = true
+py_literal = "0.4.0"
+rand.workspace = true
+rusqlite = { version = "0.40", features = ["bundled"] }
+serde-pickle = "1.2"
+serde_json.workspace = true
+tokio.workspace = true
+
+[dev-dependencies]
+rstest.workspace = true
+tempfile = "3.27.0"

litellm-rust/crates/cache-disk/src/adapter.rs

@@ -0,0 +1,10 @@
+use litellm_cache::Error;
+
+use crate::StoredValue;
+
+pub trait ValueAdapter: Send + Sync + 'static {
+    fn read(&self, value: StoredValue) -> Result<Option<Vec<u8>>, Error>;
+    fn write(&self, payload: Vec<u8>) -> StoredValue;
+    fn counter_seed(&self, value: Option<StoredValue>) -> Result<f64, Error>;
+    fn counter_value(&self, value: f64) -> StoredValue;
+}

litellm-rust/crates/cache-disk/src/cache.rs

@@ -0,0 +1,301 @@
+use std::{
+    path::Path,
+    sync::Arc,
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, CacheConnectionStatus,
+    CounterCache, DeleteCache, Error, ExactCacheContext, FlushCache,
+};
+
+use crate::{DiskStore, DiskcacheSqliteStore, PythonDiskCacheAdapter, StoredValue, ValueAdapter};
+
+pub struct DiskCache<S, D = DiskcacheSqliteStore, A = PythonDiskCacheAdapter> {
+    store: Arc<D>,
+    adapter: Arc<A>,
+    codec: S,
+}
+
+impl<S: CacheCodec> DiskCache<S> {
+    pub fn open(directory: impl AsRef<Path>, codec: S) -> Result<Self, Error> {
+        Ok(Self {
+            store: Arc::new(DiskcacheSqliteStore::open(directory)?),
+            adapter: Arc::new(PythonDiskCacheAdapter),
+            codec,
+        })
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore> DiskCache<S, D, PythonDiskCacheAdapter> {
+    pub fn with_store(store: D, codec: S) -> Self {
+        Self {
+            store: Arc::new(store),
+            adapter: Arc::new(PythonDiskCacheAdapter),
+            codec,
+        }
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> DiskCache<S, D, A> {
+    pub fn with_adapter(store: D, adapter: A, codec: S) -> Self {
+        Self {
+            store: Arc::new(store),
+            adapter: Arc::new(adapter),
+            codec,
+        }
+    }
+
+    pub fn directory(&self) -> &Path {
+        self.store.directory()
+    }
+
+    fn decode_stored(&self, value: StoredValue) -> Result<Option<S::Value>, Error> {
+        let Some(bytes) = self.adapter.read(value)? else {
+            return Ok(None);
+        };
+        self.codec.decode(&bytes).map(Some)
+    }
+
+    async fn run_blocking<T, F>(store: Arc<D>, operation: F) -> Result<T, Error>
+    where
+        T: Send + 'static,
+        F: FnOnce(&D) -> Result<T, Error> + Send + 'static,
+    {
+        tokio::task::spawn_blocking(move || operation(&store))
+            .await
+            .map_err(|_| Error::Unavailable)?
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> BaseCache for DiskCache<S, D, A> {
+    type Value = S::Value;
+    type Context = ExactCacheContext;
+
+    fn get_ttl(&self, context: &Self::Context) -> Option<Duration> {
+        context.ttl
+    }
+
+    fn set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: &Self::Context,
+    ) -> Result<(), Error> {
+        let value = self.adapter.write(self.codec.encode(&value)?);
+        let expire_time = context.ttl.map(|ttl| unix_now() + ttl.as_secs_f64());
+        self.store.set(key, value, expire_time, unix_now())
+    }
+
+    fn get_cache(&self, key: &str, _: &Self::Context) -> Result<Option<Self::Value>, Error> {
+        self.store
+            .get(key, unix_now())?
+            .map(|value| self.decode_stored(value))
+            .transpose()
+            .map(|value| value.flatten())
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: ExactCacheContext,
+    ) -> Result<(), Error> {
+        let value = self.adapter.write(self.codec.encode(&value)?);
+        let ttl = context.ttl;
+        let key = key.to_string();
+        Self::run_blocking(Arc::clone(&self.store), move |store| {
+            let expire_time = ttl.map(|ttl| unix_now() + ttl.as_secs_f64());
+            store.set(&key, value, expire_time, unix_now())
+        })
+        .await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        _: &ExactCacheContext,
+    ) -> Result<Option<Self::Value>, Error> {
+        let key = key.to_string();
+        let value = Self::run_blocking(Arc::clone(&self.store), move |store| {
+            store.get(&key, unix_now())
+        })
+        .await?;
+        value
+            .map(|value| self.decode_stored(value))
+            .transpose()
+            .map(|value| value.flatten())
+    }
+
+    async fn async_set_cache_pipeline(
+        &self,
+        entries: Vec<(String, Self::Value)>,
+        context: ExactCacheContext,
+    ) -> Result<(), Error> {
+        let entries = entries
+            .into_iter()
+            .map(|(key, value)| {
+                self.codec
+                    .encode(&value)
+                    .map(|value| (key, self.adapter.write(value)))
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+        let expire_after = context.ttl;
+        Self::run_blocking(Arc::clone(&self.store), move |store| {
+            for (key, value) in entries {
+                let expire_time = expire_after.map(|ttl| unix_now() + ttl.as_secs_f64());
+                store.set(&key, value, expire_time, unix_now())?;
+            }
+            Ok(())
+        })
+        .await
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        let result = Self::run_blocking(Arc::clone(&self.store), |store| {
+            store.probe().map(|_| CacheConnectionResult {
+                status: CacheConnectionStatus::Success,
+                message: "Disk cache connection test successful".into(),
+                error: None,
+            })
+        })
+        .await;
+        Ok(match result {
+            Ok(result) => result,
+            Err(error) => CacheConnectionResult {
+                status: CacheConnectionStatus::Failed,
+                message: format!("Disk cache connection failed: {error}"),
+                error: Some(error.to_string()),
+            },
+        })
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> BatchCache for DiskCache<S, D, A> {
+    fn batch_get_cache(
+        &self,
+        keys: &[String],
+        context: &ExactCacheContext,
+    ) -> Result<Vec<BatchEntry<Self::Value>>, Error> {
+        keys.iter()
+            .map(|key| match self.get_cache(key, context) {
+                Ok(Some(value)) => Ok(BatchEntry::Hit(value)),
+                Ok(None) => Ok(BatchEntry::Miss),
+                Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid),
+                Err(error) => Err(error),
+            })
+            .collect()
+    }
+
+    async fn async_batch_get_cache(
+        &self,
+        keys: Vec<String>,
+        _: ExactCacheContext,
+    ) -> Result<Vec<BatchEntry<Self::Value>>, Error> {
+        let values = Self::run_blocking(Arc::clone(&self.store), move |store| {
+            keys.into_iter()
+                .map(|key| store.get(&key, unix_now()).map(|value| (key, value)))
+                .collect::<Result<Vec<_>, _>>()
+        })
+        .await?;
+        values
+            .into_iter()
+            .map(|(_, value)| match value {
+                None => Ok(BatchEntry::Miss),
+                Some(value) => match self.decode_stored(value) {
+                    Ok(Some(value)) => Ok(BatchEntry::Hit(value)),
+                    Ok(None) => Ok(BatchEntry::Miss),
+                    Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid),
+                    Err(error) => Err(error),
+                },
+            })
+            .collect()
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> DeleteCache for DiskCache<S, D, A> {
+    fn delete_cache(&self, key: &str) -> Result<(), Error> {
+        self.store.pop(key, unix_now()).map(|_| ())
+    }
+
+    async fn async_delete_cache(&self, key: &str) -> Result<(), Error> {
+        let key = key.to_string();
+        Self::run_blocking(Arc::clone(&self.store), move |store| {
+            store.pop(&key, unix_now()).map(|_| ())
+        })
+        .await
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> FlushCache for DiskCache<S, D, A> {
+    fn flush_cache(&self) -> Result<(), Error> {
+        self.store.clear()
+    }
+
+    async fn async_flush_cache(&self) -> Result<(), Error> {
+        Self::run_blocking(Arc::clone(&self.store), |store| store.clear()).await
+    }
+}
+
+impl<S: CacheCodec<Value = f64>, D: DiskStore, A: ValueAdapter> CounterCache
+    for DiskCache<S, D, A>
+{
+    fn increment_cache(
+        &self,
+        key: &str,
+        amount: f64,
+        context: ExactCacheContext,
+    ) -> Result<f64, Error> {
+        increment(
+            self.adapter.as_ref(),
+            self.store.as_ref(),
+            key,
+            amount,
+            context.ttl,
+        )
+    }
+
+    async fn async_increment(
+        &self,
+        key: &str,
+        amount: f64,
+        context: ExactCacheContext,
+    ) -> Result<f64, Error> {
+        let key = key.to_string();
+        let adapter = Arc::clone(&self.adapter);
+        Self::run_blocking(Arc::clone(&self.store), move |store| {
+            increment(adapter.as_ref(), store, &key, amount, context.ttl)
+        })
+        .await
+    }
+}
+
+fn increment<A: ValueAdapter, D: DiskStore>(
+    adapter: &A,
+    store: &D,
+    key: &str,
+    amount: f64,
+    ttl: Option<Duration>,
+) -> Result<f64, Error> {
+    let mut result = None;
+    let mut apply = |current: Option<StoredValue>| {
+        let initial = adapter.counter_seed(current)?;
+        let value = initial + amount;
+        let stored = adapter.counter_value(value);
+        result = Some(value);
+        Ok((stored, ttl.map(|ttl| unix_now() + ttl.as_secs_f64())))
+    };
+    store.update(key, unix_now(), &mut apply)?;
+    result.ok_or(Error::InvalidEntry)
+}
+
+fn unix_now() -> f64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs_f64()
+}

litellm-rust/crates/cache-disk/src/lib.rs

@@ -0,0 +1,11 @@
+mod adapter;
+mod cache;
+mod python;
+mod sqlite;
+mod store;
+
+pub use adapter::ValueAdapter;
+pub use cache::DiskCache;
+pub use python::PythonDiskCacheAdapter;
+pub use sqlite::DiskcacheSqliteStore;
+pub use store::{DiskStore, StoredValue};

litellm-rust/crates/cache-disk/src/python/mod.rs

@@ -0,0 +1,77 @@
+mod value;
+
+use litellm_cache::Error;
+use py_literal::Value;
+
+use crate::{StoredValue, ValueAdapter};
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct PythonDiskCacheAdapter;
+
+impl PythonDiskCacheAdapter {
+    fn python_get_cache(value: StoredValue) -> Result<Option<Value>, Error> {
+        let value = match value {
+            StoredValue::Bytes(value) => Value::Bytes(value),
+            StoredValue::Text(value) => Value::String(value),
+            StoredValue::Integer(value) => Value::Integer(value.into()),
+            StoredValue::Float(value) => Value::Float(value),
+            StoredValue::Pickle(value) => value::from_pickle(&value)?,
+        };
+        if !value::is_truthy(&value) {
+            return Ok(None);
+        }
+        match value {
+            Value::String(text) => Ok(Some(
+                value::from_json_text(&text).unwrap_or(Value::String(text)),
+            )),
+            Value::Bytes(bytes) => match std::str::from_utf8(&bytes) {
+                Ok(text) => Ok(Some(
+                    value::from_json_text(text).unwrap_or(Value::Bytes(bytes)),
+                )),
+                Err(_) => Ok(Some(Value::Bytes(bytes))),
+            },
+            value => Ok(Some(value)),
+        }
+    }
+}
+
+impl ValueAdapter for PythonDiskCacheAdapter {
+    fn read(&self, value: StoredValue) -> Result<Option<Vec<u8>>, Error> {
+        match value {
+            StoredValue::Text(value) => Ok((!value.is_empty()).then(|| value.into_bytes())),
+            StoredValue::Bytes(value) => Ok((!value.is_empty()).then_some(value)),
+            value => {
+                let Some(value) = Self::python_get_cache(value)? else {
+                    return Ok(None);
+                };
+                value::to_json(&value).map(Some)
+            }
+        }
+    }
+
+    fn write(&self, payload: Vec<u8>) -> StoredValue {
+        StoredValue::Bytes(payload)
+    }
+
+    fn counter_seed(&self, value: Option<StoredValue>) -> Result<f64, Error> {
+        let Some(value) = value else {
+            return Ok(0.0);
+        };
+        let Some(value) = Self::python_get_cache(value)? else {
+            return Ok(0.0);
+        };
+        Ok(if value::is_int(&value) {
+            value::to_f64(&value).unwrap_or(0.0)
+        } else {
+            0.0
+        })
+    }
+
+    fn counter_value(&self, value: f64) -> StoredValue {
+        if value.fract() == 0.0 && value >= i64::MIN as f64 && value <= i64::MAX as f64 {
+            StoredValue::Integer(value as i64)
+        } else {
+            StoredValue::Float(value)
+        }
+    }
+}

litellm-rust/crates/cache-disk/src/python/value.rs

@@ -0,0 +1,173 @@
+use litellm_cache::Error;
+use py_literal::Value;
+use serde_json::{Map, Number};
+
+pub(crate) fn from_pickle(bytes: &[u8]) -> Result<Value, Error> {
+    let value = serde_pickle::value_from_slice(bytes, Default::default())
+        .map_err(|_| Error::InvalidEntry)?;
+    from_pickle_value(value)
+}
+
+fn from_pickle_value(value: serde_pickle::Value) -> Result<Value, Error> {
+    match value {
+        serde_pickle::Value::None => Ok(Value::None),
+        serde_pickle::Value::Bool(value) => Ok(Value::Boolean(value)),
+        serde_pickle::Value::I64(value) => integer(value.to_string()),
+        serde_pickle::Value::Int(value) => integer(value.to_string()),
+        serde_pickle::Value::F64(value) => Ok(Value::Float(value)),
+        serde_pickle::Value::String(value) => Ok(Value::String(value)),
+        serde_pickle::Value::Bytes(value) => Ok(Value::Bytes(value)),
+        serde_pickle::Value::List(values) => values
+            .into_iter()
+            .map(from_pickle_value)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::List),
+        serde_pickle::Value::Tuple(values) => values
+            .into_iter()
+            .map(from_pickle_value)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::Tuple),
+        serde_pickle::Value::Set(values) => values
+            .into_iter()
+            .map(from_pickle_hashable)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::Set),
+        serde_pickle::Value::FrozenSet(values) => values
+            .into_iter()
+            .map(from_pickle_hashable)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::Set),
+        serde_pickle::Value::Dict(values) => values
+            .into_iter()
+            .map(|(key, value)| Ok((from_pickle_hashable(key)?, from_pickle_value(value)?)))
+            .collect::<Result<Vec<_>, Error>>()
+            .map(Value::Dict),
+    }
+}
+
+fn from_pickle_hashable(value: serde_pickle::HashableValue) -> Result<Value, Error> {
+    Ok(match value {
+        serde_pickle::HashableValue::None => Value::None,
+        serde_pickle::HashableValue::Bool(value) => Value::Boolean(value),
+        serde_pickle::HashableValue::I64(value) => integer(value.to_string())?,
+        serde_pickle::HashableValue::Int(value) => integer(value.to_string())?,
+        serde_pickle::HashableValue::F64(value) => Value::Float(value),
+        serde_pickle::HashableValue::Bytes(value) => Value::Bytes(value),
+        serde_pickle::HashableValue::String(value) => Value::String(value),
+        serde_pickle::HashableValue::Tuple(values) => Value::Tuple(
+            values
+                .into_iter()
+                .map(from_pickle_hashable)
+                .collect::<Result<Vec<_>, _>>()?,
+        ),
+        serde_pickle::HashableValue::FrozenSet(values) => Value::Set(
+            values
+                .into_iter()
+                .map(from_pickle_hashable)
+                .collect::<Result<Vec<_>, _>>()?,
+        ),
+    })
+}
+
+fn integer(value: String) -> Result<Value, Error> {
+    value.parse().map_err(|_| Error::InvalidEntry)
+}
+
+pub(crate) fn from_json(value: serde_json::Value) -> Value {
+    match value {
+        serde_json::Value::Null => Value::None,
+        serde_json::Value::Bool(value) => Value::Boolean(value),
+        serde_json::Value::Number(value) => {
+            if value.is_i64() || value.is_u64() {
+                integer(value.to_string())
+                    .unwrap_or(Value::Float(value.as_f64().unwrap_or(f64::NAN)))
+            } else {
+                Value::Float(value.as_f64().unwrap_or(f64::NAN))
+            }
+        }
+        serde_json::Value::String(value) => Value::String(value),
+        serde_json::Value::Array(values) => {
+            Value::List(values.into_iter().map(from_json).collect())
+        }
+        serde_json::Value::Object(values) => Value::Dict(
+            values
+                .into_iter()
+                .map(|(key, value)| (Value::String(key), from_json(value)))
+                .collect(),
+        ),
+    }
+}
+
+pub(crate) fn from_json_text(value: &str) -> Result<Value, Error> {
+    serde_json::from_str(value)
+        .map(from_json)
+        .map_err(|_| Error::InvalidEntry)
+}
+
+pub(crate) fn is_truthy(value: &Value) -> bool {
+    match value {
+        Value::None => false,
+        Value::Boolean(value) => *value,
+        Value::Integer(value) => value.to_string() != "0",
+        Value::Float(value) => *value != 0.0,
+        Value::Complex(value) => value.re != 0.0 || value.im != 0.0,
+        Value::String(value) => !value.is_empty(),
+        Value::Bytes(value) => !value.is_empty(),
+        Value::Tuple(value) | Value::List(value) | Value::Set(value) => !value.is_empty(),
+        Value::Dict(value) => !value.is_empty(),
+    }
+}
+
+pub(crate) fn is_int(value: &Value) -> bool {
+    matches!(value, Value::Integer(_) | Value::Boolean(_))
+}
+
+pub(crate) fn to_f64(value: &Value) -> Option<f64> {
+    match value {
+        Value::Integer(value) => value.to_string().parse().ok(),
+        Value::Boolean(value) => Some(if *value { 1.0 } else { 0.0 }),
+        _ => None,
+    }
+}
+
+pub(crate) fn to_json(value: &Value) -> Result<Vec<u8>, Error> {
+    serde_json::to_vec(&to_json_value(value)?).map_err(|_| Error::InvalidEntry)
+}
+
+fn to_json_value(value: &Value) -> Result<serde_json::Value, Error> {
+    Ok(match value {
+        Value::None => serde_json::Value::Null,
+        Value::Boolean(value) => serde_json::Value::Bool(*value),
+        Value::Integer(value) => serde_json::Value::Number(
+            value
+                .to_string()
+                .parse::<Number>()
+                .map_err(|_| Error::InvalidEntry)?,
+        ),
+        Value::Float(value) => {
+            serde_json::Value::Number(Number::from_f64(*value).ok_or(Error::InvalidEntry)?)
+        }
+        Value::Complex(_) | Value::Bytes(_) => return Err(Error::InvalidEntry),
+        Value::String(value) => serde_json::Value::String(value.clone()),
+        Value::Tuple(values) | Value::List(values) | Value::Set(values) => {
+            serde_json::Value::Array(
+                values
+                    .iter()
+                    .map(to_json_value)
+                    .collect::<Result<Vec<_>, _>>()?,
+            )
+        }
+        Value::Dict(values) => {
+            let values = values
+                .iter()
+                .map(|(key, value)| {
+                    let Value::String(key) = key else {
+                        return Err(Error::InvalidEntry);
+                    };
+                    Ok((key.clone(), to_json_value(value)?))
+                })
+                .collect::<Result<Map<String, serde_json::Value>, _>>()?;
+            serde_json::Value::Object(values)
+        }
+    })
+}

litellm-rust/crates/cache-disk/src/sqlite.rs

@@ -0,0 +1,817 @@
+use std::{
+    collections::HashMap,
+    fs::{self, OpenOptions},
+    io::Write,
+    path::{Path, PathBuf},
+    sync::Mutex,
+};
+
+use litellm_cache::Error;
+use rand::RngCore;
+use rusqlite::{Connection, OptionalExtension, params, types::Value};
+
+use crate::{DiskStore, StoredValue};
+
+const MODE_RAW: i64 = 1;
+const MODE_BINARY: i64 = 2;
+const MODE_TEXT: i64 = 3;
+const MODE_PICKLE: i64 = 4;
+
+const DEFAULT_DISK_MIN_FILE_SIZE: i64 = 2_i64.pow(15);
+const DEFAULT_SIZE_LIMIT: i64 = 2_i64.pow(30);
+const DEFAULT_CULL_LIMIT: i64 = 10;
+
+pub struct DiskcacheSqliteStore {
+    directory: PathBuf,
+    connection: Mutex<Connection>,
+    min_file_size: usize,
+    eviction_policy: String,
+    size_limit: i64,
+    cull_limit: i64,
+    statistics: bool,
+}
+
+struct StoredColumns {
+    size: i64,
+    mode: i64,
+    filename: Option<String>,
+    value: Option<Value>,
+}
+
+struct Row {
+    rowid: i64,
+    mode: i64,
+    filename: Option<String>,
+    value: Value,
+}
+
+impl DiskcacheSqliteStore {
+    pub fn open(directory: impl AsRef<Path>) -> Result<Self, Error> {
+        let directory = directory.as_ref().to_path_buf();
+        fs::create_dir_all(&directory).map_err(|_| Error::Unavailable)?;
+        let directory = std::path::absolute(&directory).map_err(|_| Error::Unavailable)?;
+        let database = directory.join("cache.db");
+        let connection = Connection::open(database).map_err(|_| Error::Unavailable)?;
+        connection
+            .busy_timeout(std::time::Duration::from_secs(60))
+            .map_err(|_| Error::Unavailable)?;
+
+        let mut settings = read_settings(&connection)?;
+        for (key, value) in default_settings() {
+            settings.entry(key).or_insert(value);
+        }
+        for (key, value) in settings
+            .iter()
+            .filter(|(key, _)| key.starts_with("sqlite_"))
+        {
+            apply_pragma(&connection, key, value)?;
+        }
+
+        connection
+            .execute_batch(
+                "CREATE TABLE IF NOT EXISTS Settings (
+                    key TEXT NOT NULL UNIQUE,
+                    value
+                )",
+            )
+            .map_err(|_| Error::Unavailable)?;
+        for (key, value) in &settings {
+            if !matches!(key.as_str(), "count" | "size" | "hits" | "misses") {
+                connection
+                    .execute(
+                        "INSERT OR REPLACE INTO Settings VALUES (?, ?)",
+                        params![key, value],
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+        }
+        for (key, value) in [
+            ("count", Value::Integer(0)),
+            ("size", Value::Integer(0)),
+            ("hits", Value::Integer(0)),
+            ("misses", Value::Integer(0)),
+        ] {
+            connection
+                .execute(
+                    "INSERT OR IGNORE INTO Settings VALUES (?, ?)",
+                    params![key, value],
+                )
+                .map_err(|_| Error::Unavailable)?;
+        }
+        connection
+            .execute_batch(
+                "CREATE TABLE IF NOT EXISTS Cache (
+                    rowid INTEGER PRIMARY KEY,
+                    key BLOB,
+                    raw INTEGER,
+                    store_time REAL,
+                    expire_time REAL,
+                    access_time REAL,
+                    access_count INTEGER DEFAULT 0,
+                    tag BLOB,
+                    size INTEGER DEFAULT 0,
+                    mode INTEGER DEFAULT 0,
+                    filename TEXT,
+                    value BLOB
+                );
+                CREATE UNIQUE INDEX IF NOT EXISTS Cache_key_raw ON Cache(key, raw);
+                CREATE INDEX IF NOT EXISTS Cache_expire_time ON Cache(expire_time);",
+            )
+            .map_err(|_| Error::Unavailable)?;
+
+        let eviction_policy = setting_string(&settings, "eviction_policy")
+            .unwrap_or_else(|| "least-recently-stored".to_string());
+        match eviction_policy.as_str() {
+            "none" => {}
+            "least-recently-stored" => {
+                connection
+                    .execute_batch(
+                        "CREATE INDEX IF NOT EXISTS Cache_store_time ON Cache(store_time)",
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            "least-recently-used" => {
+                connection
+                    .execute_batch(
+                        "CREATE INDEX IF NOT EXISTS Cache_access_time ON Cache(access_time)",
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            "least-frequently-used" => {
+                connection
+                    .execute_batch(
+                        "CREATE INDEX IF NOT EXISTS Cache_access_count ON Cache(access_count)",
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            _ => return Err(Error::Unavailable),
+        }
+        connection
+            .execute_batch(
+                "CREATE TRIGGER IF NOT EXISTS Settings_count_insert
+                    AFTER INSERT ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings SET value = value + 1
+                    WHERE key = \"count\"; END;
+                 CREATE TRIGGER IF NOT EXISTS Settings_count_delete
+                    AFTER DELETE ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings SET value = value - 1
+                    WHERE key = \"count\"; END;
+                 CREATE TRIGGER IF NOT EXISTS Settings_size_insert
+                    AFTER INSERT ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings SET value = value + NEW.size
+                    WHERE key = \"size\"; END;
+                 CREATE TRIGGER IF NOT EXISTS Settings_size_update
+                    AFTER UPDATE ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings
+                    SET value = value + NEW.size - OLD.size
+                    WHERE key = \"size\"; END;
+                 CREATE TRIGGER IF NOT EXISTS Settings_size_delete
+                    AFTER DELETE ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings SET value = value - OLD.size
+                    WHERE key = \"size\"; END;",
+            )
+            .map_err(|_| Error::Unavailable)?;
+
+        let min_file_size = setting_i64(&settings, "disk_min_file_size")
+            .unwrap_or(DEFAULT_DISK_MIN_FILE_SIZE)
+            .try_into()
+            .map_err(|_| Error::Unavailable)?;
+        let size_limit = setting_i64(&settings, "size_limit").unwrap_or(DEFAULT_SIZE_LIMIT);
+        let cull_limit = setting_i64(&settings, "cull_limit").unwrap_or(DEFAULT_CULL_LIMIT);
+        let statistics = setting_i64(&settings, "statistics").unwrap_or_default() != 0;
+
+        Ok(Self {
+            directory,
+            connection: Mutex::new(connection),
+            min_file_size,
+            eviction_policy,
+            size_limit,
+            cull_limit,
+            statistics,
+        })
+    }
+
+    fn set_locked(
+        &self,
+        connection: &Connection,
+        key: &str,
+        columns: StoredColumns,
+        expire_time: Option<f64>,
+        now: f64,
+    ) -> Result<Vec<String>, Error> {
+        let mut cleanup = Vec::new();
+        if let Some(old_filename) = connection
+            .query_row(
+                "SELECT filename FROM Cache WHERE key = ? AND raw = 1",
+                params![key],
+                |row| row.get::<_, Option<String>>(0),
+            )
+            .optional()
+            .map_err(|_| Error::Unavailable)?
+            .flatten()
+        {
+            cleanup.push(old_filename);
+        }
+        let (size, mode, filename, value) =
+            (columns.size, columns.mode, columns.filename, columns.value);
+        let rowid = connection
+            .query_row(
+                "SELECT rowid FROM Cache WHERE key = ? AND raw = 1",
+                params![key],
+                |row| row.get::<_, i64>(0),
+            )
+            .optional()
+            .map_err(|_| Error::Unavailable)?;
+        if let Some(rowid) = rowid {
+            connection
+                .execute(
+                    "UPDATE Cache SET store_time = ?, expire_time = ?, access_time = ?,
+                        access_count = 0, tag = NULL, size = ?, mode = ?, filename = ?, value = ?
+                        WHERE rowid = ?",
+                    params![now, expire_time, now, size, mode, filename, value, rowid],
+                )
+                .map_err(|_| Error::Unavailable)?;
+        } else {
+            connection
+                .execute(
+                    "INSERT INTO Cache(
+                        key, raw, store_time, expire_time, access_time, access_count,
+                        tag, size, mode, filename, value
+                    ) VALUES (?, 1, ?, ?, ?, 0, NULL, ?, ?, ?, ?)",
+                    params![key, now, expire_time, now, size, mode, filename, value],
+                )
+                .map_err(|_| Error::Unavailable)?;
+        }
+        cleanup.extend(self.cull(connection, now)?);
+        Ok(cleanup)
+    }
+
+    fn cull(&self, connection: &Connection, now: f64) -> Result<Vec<String>, Error> {
+        if self.cull_limit <= 0 {
+            return Ok(Vec::new());
+        }
+        let mut cleanup = Vec::new();
+        let expired = connection
+            .prepare(
+                "SELECT rowid, filename FROM Cache
+                 WHERE expire_time IS NOT NULL AND expire_time < ?
+                 ORDER BY expire_time LIMIT ?",
+            )
+            .map_err(|_| Error::Unavailable)?
+            .query_map(params![now, self.cull_limit], |row| {
+                Ok((row.get::<_, i64>(0)?, row.get::<_, Option<String>>(1)?))
+            })
+            .map_err(|_| Error::Unavailable)?
+            .collect::<Result<Vec<_>, _>>()
+            .map_err(|_| Error::Unavailable)?;
+        for (_, filename) in &expired {
+            if let Some(filename) = filename {
+                cleanup.push(filename.clone());
+            }
+        }
+        for (rowid, _) in &expired {
+            connection
+                .execute("DELETE FROM Cache WHERE rowid = ?", params![rowid])
+                .map_err(|_| Error::Unavailable)?;
+        }
+        let remaining = self.cull_limit - i64::try_from(expired.len()).unwrap_or(self.cull_limit);
+        if remaining <= 0 || self.volume(connection)? < self.size_limit {
+            return Ok(cleanup);
+        }
+        let order = match self.eviction_policy.as_str() {
+            "none" => return Ok(cleanup),
+            "least-recently-stored" => "store_time",
+            "least-recently-used" => "access_time",
+            "least-frequently-used" => "access_count",
+            _ => return Err(Error::Unavailable),
+        };
+        let rows = connection
+            .prepare(&format!(
+                "SELECT rowid, filename FROM Cache ORDER BY {order} LIMIT ?"
+            ))
+            .map_err(|_| Error::Unavailable)?
+            .query_map(params![remaining], |row| {
+                Ok((row.get::<_, i64>(0)?, row.get::<_, Option<String>>(1)?))
+            })
+            .map_err(|_| Error::Unavailable)?
+            .collect::<Result<Vec<_>, _>>()
+            .map_err(|_| Error::Unavailable)?;
+        for (_, filename) in &rows {
+            if let Some(filename) = filename {
+                cleanup.push(filename.clone());
+            }
+        }
+        for (rowid, _) in rows {
+            connection
+                .execute("DELETE FROM Cache WHERE rowid = ?", params![rowid])
+                .map_err(|_| Error::Unavailable)?;
+        }
+        Ok(cleanup)
+    }
+
+    fn volume(&self, connection: &Connection) -> Result<i64, Error> {
+        let page_count: i64 = connection
+            .query_row("PRAGMA page_count", [], |row| row.get(0))
+            .map_err(|_| Error::Unavailable)?;
+        let page_size: i64 = connection
+            .query_row("PRAGMA page_size", [], |row| row.get(0))
+            .map_err(|_| Error::Unavailable)?;
+        let size: i64 = connection
+            .query_row("SELECT value FROM Settings WHERE key = 'size'", [], |row| {
+                row.get(0)
+            })
+            .map_err(|_| Error::Unavailable)?;
+        Ok(page_count.saturating_mul(page_size).saturating_add(size))
+    }
+}
+
+impl DiskStore for DiskcacheSqliteStore {
+    fn directory(&self) -> &Path {
+        &self.directory
+    }
+
+    fn get(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let select = "SELECT rowid, expire_time, mode, filename, value FROM Cache
+            WHERE key = ? AND raw = 1 AND (expire_time IS NULL OR expire_time > ?)";
+        let row = connection
+            .query_row(select, params![key, now], row_from_query)
+            .optional()
+            .map_err(|_| Error::Unavailable)?;
+        if !self.statistics && !has_get_update(&self.eviction_policy) {
+            return row
+                .map(|row| fetch_row(&self.directory, row))
+                .transpose()
+                .map(|value| value.flatten());
+        }
+        transactional(&connection, |connection| {
+            let row = connection
+                .query_row(select, params![key, now], row_from_query)
+                .optional()
+                .map_err(|_| Error::Unavailable)?;
+            let Some(row) = row else {
+                if self.statistics {
+                    connection
+                        .execute(
+                            "UPDATE Settings SET value = value + 1 WHERE key = 'misses'",
+                            [],
+                        )
+                        .map_err(|_| Error::Unavailable)?;
+                }
+                return Ok(None);
+            };
+            let rowid = row.rowid;
+            let value = fetch_row(&self.directory, row);
+            let hit = value.as_ref().is_ok_and(Option::is_some);
+            if hit && self.statistics {
+                connection
+                    .execute(
+                        "UPDATE Settings SET value = value + 1 WHERE key = 'hits'",
+                        [],
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            } else if !hit && self.statistics {
+                connection
+                    .execute(
+                        "UPDATE Settings SET value = value + 1 WHERE key = 'misses'",
+                        [],
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            if has_get_update(&self.eviction_policy) && hit {
+                let update = match self.eviction_policy.as_str() {
+                    "least-recently-used" => "UPDATE Cache SET access_time = ? WHERE rowid = ?",
+                    "least-frequently-used" => {
+                        "UPDATE Cache SET access_count = access_count + 1 WHERE rowid = ?"
+                    }
+                    _ => return Err(Error::Unavailable),
+                };
+                if self.eviction_policy == "least-recently-used" {
+                    connection
+                        .execute(update, params![now, rowid])
+                        .map_err(|_| Error::Unavailable)?;
+                } else {
+                    connection
+                        .execute(update, params![rowid])
+                        .map_err(|_| Error::Unavailable)?;
+                }
+            }
+            value
+        })
+    }
+
+    fn set(
+        &self,
+        key: &str,
+        value: StoredValue,
+        expire_time: Option<f64>,
+        now: f64,
+    ) -> Result<(), Error> {
+        let columns = store_value(&self.directory, self.min_file_size, value)?;
+        let new_filename = columns.filename.clone();
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let result = transactional(&connection, |connection| {
+            self.set_locked(connection, key, columns, expire_time, now)
+        });
+        match result {
+            Ok(cleanup) => {
+                cleanup_files(&self.directory, cleanup);
+                Ok(())
+            }
+            Err(error) => {
+                if let Some(filename) = new_filename {
+                    remove_file(&self.directory, &filename);
+                }
+                Err(error)
+            }
+        }
+    }
+
+    fn pop(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let selected = transactional(&connection, |connection| {
+            let row = connection
+                .query_row(
+                    "SELECT rowid, expire_time, mode, filename, value FROM Cache
+                     WHERE key = ? AND raw = 1
+                     AND (expire_time IS NULL OR expire_time > ?)",
+                    params![key, now],
+                    row_from_query,
+                )
+                .optional()
+                .map_err(|_| Error::Unavailable)?;
+            let Some(row) = row else {
+                return Ok(None);
+            };
+            connection
+                .execute("DELETE FROM Cache WHERE rowid = ?", params![row.rowid])
+                .map_err(|_| Error::Unavailable)?;
+            Ok(Some(row))
+        })?;
+        let Some(row) = selected else {
+            return Ok(None);
+        };
+        let filename = row.filename.clone();
+        let result = fetch_row(&self.directory, row)?;
+        if let Some(filename) = filename {
+            remove_file(&self.directory, &filename);
+        }
+        Ok(result)
+    }
+
+    fn clear(&self) -> Result<(), Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let mut last_rowid = 0_i64;
+        loop {
+            let batch = transactional(&connection, |connection| {
+                let rows = connection
+                    .prepare(
+                        "SELECT rowid, filename FROM Cache
+                         WHERE rowid > ? ORDER BY rowid LIMIT 100",
+                    )
+                    .map_err(|_| Error::Unavailable)?
+                    .query_map(params![last_rowid], |row| {
+                        Ok((row.get::<_, i64>(0)?, row.get::<_, Option<String>>(1)?))
+                    })
+                    .map_err(|_| Error::Unavailable)?
+                    .collect::<Result<Vec<_>, _>>()
+                    .map_err(|_| Error::Unavailable)?;
+                if rows.is_empty() {
+                    return Ok(rows);
+                }
+                let ids = rows
+                    .iter()
+                    .map(|(rowid, _)| rowid.to_string())
+                    .collect::<Vec<_>>()
+                    .join(",");
+                connection
+                    .execute(&format!("DELETE FROM Cache WHERE rowid IN ({ids})"), [])
+                    .map_err(|_| Error::Unavailable)?;
+                Ok(rows)
+            })?;
+            if batch.is_empty() {
+                return Ok(());
+            }
+            last_rowid = batch.last().map(|(rowid, _)| *rowid).unwrap_or(last_rowid);
+            cleanup_files(
+                &self.directory,
+                batch
+                    .into_iter()
+                    .filter_map(|(_, filename)| filename)
+                    .collect(),
+            );
+        }
+    }
+
+    fn update(
+        &self,
+        key: &str,
+        now: f64,
+        apply: &mut dyn FnMut(Option<StoredValue>) -> Result<(StoredValue, Option<f64>), Error>,
+    ) -> Result<(), Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let mut created_filename = None;
+        let result = transactional(&connection, |connection| {
+            let current = connection
+                .query_row(
+                    "SELECT rowid, expire_time, mode, filename, value FROM Cache
+                     WHERE key = ? AND raw = 1
+                     AND (expire_time IS NULL OR expire_time > ?)",
+                    params![key, now],
+                    row_from_query,
+                )
+                .optional()
+                .map_err(|_| Error::Unavailable)?
+                .map(|row| fetch_row(&self.directory, row))
+                .transpose()?
+                .flatten();
+            let (value, expire_time) = apply(current)?;
+            let columns = store_value(&self.directory, self.min_file_size, value)?;
+            created_filename = columns.filename.clone();
+            let cleanup = self.set_locked(connection, key, columns, expire_time, now)?;
+            Ok(cleanup)
+        });
+        match result {
+            Ok(cleanup) => {
+                cleanup_files(&self.directory, cleanup);
+                Ok(())
+            }
+            Err(error) => {
+                if let Some(filename) = created_filename {
+                    remove_file(&self.directory, &filename);
+                }
+                Err(error)
+            }
+        }
+    }
+
+    fn probe(&self) -> Result<(), Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        connection
+            .query_row(
+                "SELECT value FROM Settings WHERE key = 'count'",
+                [],
+                |row| row.get::<_, i64>(0),
+            )
+            .map(|_| ())
+            .map_err(|_| Error::Unavailable)
+    }
+}
+
+fn default_settings() -> HashMap<String, Value> {
+    HashMap::from([
+        ("statistics".to_string(), Value::Integer(0)),
+        ("tag_index".to_string(), Value::Integer(0)),
+        (
+            "eviction_policy".to_string(),
+            Value::Text("least-recently-stored".to_string()),
+        ),
+        ("size_limit".to_string(), Value::Integer(DEFAULT_SIZE_LIMIT)),
+        ("cull_limit".to_string(), Value::Integer(DEFAULT_CULL_LIMIT)),
+        ("sqlite_auto_vacuum".to_string(), Value::Integer(1)),
+        ("sqlite_cache_size".to_string(), Value::Integer(8192)),
+        (
+            "sqlite_journal_mode".to_string(),
+            Value::Text("wal".to_string()),
+        ),
+        (
+            "sqlite_mmap_size".to_string(),
+            Value::Integer(2_i64.pow(26)),
+        ),
+        ("sqlite_synchronous".to_string(), Value::Integer(1)),
+        (
+            "disk_min_file_size".to_string(),
+            Value::Integer(DEFAULT_DISK_MIN_FILE_SIZE),
+        ),
+        ("disk_pickle_protocol".to_string(), Value::Integer(5)),
+    ])
+}
+
+fn read_settings(connection: &Connection) -> Result<HashMap<String, Value>, Error> {
+    let mut statement = match connection.prepare("SELECT key, value FROM Settings") {
+        Ok(statement) => statement,
+        Err(_) => return Ok(HashMap::new()),
+    };
+    statement
+        .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
+        .map_err(|_| Error::Unavailable)?
+        .collect::<Result<HashMap<_, _>, _>>()
+        .map_err(|_| Error::Unavailable)
+}
+
+fn apply_pragma(connection: &Connection, key: &str, value: &Value) -> Result<(), Error> {
+    let pragma = key.strip_prefix("sqlite_").ok_or(Error::Unavailable)?;
+    match value {
+        Value::Integer(value) => connection
+            .pragma_update(None, pragma, value)
+            .map_err(|_| Error::Unavailable),
+        Value::Text(value) => connection
+            .pragma_update(None, pragma, value)
+            .map_err(|_| Error::Unavailable),
+        _ => Err(Error::Unavailable),
+    }
+}
+
+fn setting_i64(settings: &HashMap<String, Value>, key: &str) -> Option<i64> {
+    match settings.get(key) {
+        Some(Value::Integer(value)) => Some(*value),
+        _ => None,
+    }
+}
+
+fn setting_string(settings: &HashMap<String, Value>, key: &str) -> Option<String> {
+    match settings.get(key) {
+        Some(Value::Text(value)) => Some(value.clone()),
+        _ => None,
+    }
+}
+
+fn has_get_update(policy: &str) -> bool {
+    matches!(policy, "least-recently-used" | "least-frequently-used")
+}
+
+fn row_from_query(row: &rusqlite::Row<'_>) -> rusqlite::Result<Row> {
+    Ok(Row {
+        rowid: row.get(0)?,
+        mode: row.get(2)?,
+        filename: row.get(3)?,
+        value: row.get(4)?,
+    })
+}
+
+fn fetch_row(directory: &Path, row: Row) -> Result<Option<StoredValue>, Error> {
+    match row.mode {
+        MODE_RAW => match row.value {
+            Value::Blob(value) => Ok(Some(StoredValue::Bytes(value))),
+            Value::Text(value) => Ok(Some(StoredValue::Text(value))),
+            Value::Integer(value) => Ok(Some(StoredValue::Integer(value))),
+            Value::Real(value) => Ok(Some(StoredValue::Float(value))),
+            Value::Null => Err(Error::InvalidEntry),
+        },
+        MODE_BINARY | MODE_PICKLE => {
+            let bytes = match row.value {
+                Value::Blob(value) => value,
+                Value::Null => {
+                    let Some(value) = read_file(directory, row.filename.as_deref())? else {
+                        return Ok(None);
+                    };
+                    value
+                }
+                _ => return Err(Error::InvalidEntry),
+            };
+            Ok(Some(if row.mode == MODE_BINARY {
+                StoredValue::Bytes(bytes)
+            } else {
+                StoredValue::Pickle(bytes)
+            }))
+        }
+        MODE_TEXT => {
+            let bytes = match row.value {
+                Value::Null => {
+                    let Some(value) = read_file(directory, row.filename.as_deref())? else {
+                        return Ok(None);
+                    };
+                    value
+                }
+                Value::Blob(value) => value,
+                Value::Text(value) => value.into_bytes(),
+                _ => return Err(Error::InvalidEntry),
+            };
+            Ok(Some(StoredValue::Text(
+                String::from_utf8(bytes).map_err(|_| Error::InvalidEntry)?,
+            )))
+        }
+        _ => Err(Error::InvalidEntry),
+    }
+}
+
+fn read_file(directory: &Path, filename: Option<&str>) -> Result<Option<Vec<u8>>, Error> {
+    let Some(filename) = filename else {
+        return Err(Error::InvalidEntry);
+    };
+    match fs::read(directory.join(filename)) {
+        Ok(value) => Ok(Some(value)),
+        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
+        Err(_) => Err(Error::Unavailable),
+    }
+}
+
+fn store_value(
+    directory: &Path,
+    min_file_size: usize,
+    value: StoredValue,
+) -> Result<StoredColumns, Error> {
+    match value {
+        StoredValue::Integer(value) => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_RAW,
+            filename: None,
+            value: Some(Value::Integer(value)),
+        }),
+        StoredValue::Float(value) => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_RAW,
+            filename: None,
+            value: Some(Value::Real(value)),
+        }),
+        StoredValue::Text(value) if value.chars().count() < min_file_size => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_RAW,
+            filename: None,
+            value: Some(Value::Text(value)),
+        }),
+        StoredValue::Text(value) => {
+            let bytes = value.into_bytes();
+            let filename = write_file(directory, &bytes)?;
+            Ok(StoredColumns {
+                size: i64::try_from(bytes.len()).map_err(|_| Error::Unavailable)?,
+                mode: MODE_TEXT,
+                filename: Some(filename),
+                value: None,
+            })
+        }
+        StoredValue::Bytes(value) if value.len() < min_file_size => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_RAW,
+            filename: None,
+            value: Some(Value::Blob(value)),
+        }),
+        StoredValue::Bytes(value) => {
+            let filename = write_file(directory, &value)?;
+            Ok(StoredColumns {
+                size: i64::try_from(value.len()).map_err(|_| Error::Unavailable)?,
+                mode: MODE_BINARY,
+                filename: Some(filename),
+                value: None,
+            })
+        }
+        StoredValue::Pickle(value) if value.len() < min_file_size => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_PICKLE,
+            filename: None,
+            value: Some(Value::Blob(value)),
+        }),
+        StoredValue::Pickle(value) => {
+            let filename = write_file(directory, &value)?;
+            Ok(StoredColumns {
+                size: i64::try_from(value.len()).map_err(|_| Error::Unavailable)?,
+                mode: MODE_PICKLE,
+                filename: Some(filename),
+                value: None,
+            })
+        }
+    }
+}
+
+fn write_file(directory: &Path, bytes: &[u8]) -> Result<String, Error> {
+    let mut random = [0_u8; 16];
+    rand::rngs::OsRng.fill_bytes(&mut random);
+    let hex = random
+        .iter()
+        .map(|byte| format!("{byte:02x}"))
+        .collect::<String>();
+    let filename = format!("{}/{}/{}.val", &hex[..2], &hex[2..4], &hex[4..]);
+    let path = directory.join(&filename);
+    if let Some(parent) = path.parent() {
+        fs::create_dir_all(parent).map_err(|_| Error::Unavailable)?;
+    }
+    let mut file = OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(path)
+        .map_err(|_| Error::Unavailable)?;
+    file.write_all(bytes).map_err(|_| Error::Unavailable)?;
+    Ok(filename)
+}
+
+fn cleanup_files(directory: &Path, filenames: Vec<String>) {
+    for filename in filenames {
+        remove_file(directory, &filename);
+    }
+}
+
+fn remove_file(directory: &Path, filename: &str) {
+    let path = directory.join(filename);
+    let _ = fs::remove_file(&path);
+}
+
+fn transactional<T>(
+    connection: &Connection,
+    operation: impl FnOnce(&Connection) -> Result<T, Error>,
+) -> Result<T, Error> {
+    connection
+        .execute_batch("BEGIN IMMEDIATE")
+        .map_err(|_| Error::Unavailable)?;
+    match operation(connection) {
+        Ok(value) => {
+            connection
+                .execute_batch("COMMIT")
+                .map_err(|_| Error::Unavailable)?;
+            Ok(value)
+        }
+        Err(error) => {
+            let _ = connection.execute_batch("ROLLBACK");
+            Err(error)
+        }
+    }
+}

litellm-rust/crates/cache-disk/src/store.rs

@@ -0,0 +1,33 @@
+use std::path::Path;
+
+use litellm_cache::Error;
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum StoredValue {
+    Bytes(Vec<u8>),
+    Text(String),
+    Integer(i64),
+    Float(f64),
+    Pickle(Vec<u8>),
+}
+
+pub trait DiskStore: Send + Sync + 'static {
+    fn directory(&self) -> &Path;
+    fn get(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error>;
+    fn set(
+        &self,
+        key: &str,
+        value: StoredValue,
+        expire_time: Option<f64>,
+        now: f64,
+    ) -> Result<(), Error>;
+    fn pop(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error>;
+    fn clear(&self) -> Result<(), Error>;
+    fn update(
+        &self,
+        key: &str,
+        now: f64,
+        apply: &mut dyn FnMut(Option<StoredValue>) -> Result<(StoredValue, Option<f64>), Error>,
+    ) -> Result<(), Error>;
+    fn probe(&self) -> Result<(), Error>;
+}

litellm-rust/crates/cache-disk/tests/cache.rs

@@ -0,0 +1,431 @@
+use std::{
+    fs,
+    path::{Path, PathBuf},
+    sync::Arc,
+    thread,
+    time::Duration,
+};
+
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheCodec, CounterCache, DeleteCache, ExactCacheContext,
+    FlushCache, JsonCodec,
+};
+use litellm_cache_disk::{DiskCache, DiskStore, DiskcacheSqliteStore, StoredValue, ValueAdapter};
+use rstest::{fixture, rstest};
+use rusqlite::Connection;
+use serde_json::{Value, json};
+use tempfile::TempDir;
+
+struct Sandbox {
+    directory: TempDir,
+}
+
+#[fixture]
+fn sandbox() -> Sandbox {
+    Sandbox {
+        directory: tempfile::tempdir().unwrap(),
+    }
+}
+
+impl Sandbox {
+    fn store(&self) -> DiskcacheSqliteStore {
+        DiskcacheSqliteStore::open(self.directory.path()).unwrap()
+    }
+
+    fn cache<V>(&self) -> DiskCache<JsonCodec<V>>
+    where
+        JsonCodec<V>: CacheCodec,
+    {
+        DiskCache::open(self.directory.path(), JsonCodec::new()).unwrap()
+    }
+
+    fn db(&self) -> Connection {
+        Connection::open(self.directory.path().join("cache.db")).unwrap()
+    }
+
+    fn value_files(&self) -> Vec<PathBuf> {
+        fn visit(directory: &Path, files: &mut Vec<PathBuf>) {
+            for entry in fs::read_dir(directory).unwrap() {
+                let path = entry.unwrap().path();
+                if path.is_dir() {
+                    visit(&path, files);
+                } else if path.extension().is_some_and(|extension| extension == "val") {
+                    files.push(path);
+                }
+            }
+        }
+
+        let mut files = Vec::new();
+        visit(self.directory.path(), &mut files);
+        files
+    }
+}
+
+#[rstest]
+fn relative_store_directory_is_absolutized(sandbox: Sandbox) {
+    let relative = PathBuf::from(format!(
+        ".litellm-cache-disk-{}",
+        sandbox
+            .directory
+            .path()
+            .file_name()
+            .unwrap()
+            .to_string_lossy()
+    ));
+    let store = DiskcacheSqliteStore::open(&relative).unwrap();
+    assert!(store.directory().is_absolute());
+    assert!(store.directory().ends_with(&relative));
+    let directory = store.directory().to_path_buf();
+    drop(store);
+    fs::remove_dir_all(directory).unwrap();
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+struct TextAdapter;
+
+impl ValueAdapter for TextAdapter {
+    fn read(&self, value: StoredValue) -> Result<Option<Vec<u8>>, litellm_cache::Error> {
+        match value {
+            StoredValue::Text(value) => Ok(Some(value.into_bytes())),
+            _ => Ok(None),
+        }
+    }
+
+    fn write(&self, payload: Vec<u8>) -> StoredValue {
+        StoredValue::Text(String::from_utf8(payload).unwrap())
+    }
+
+    fn counter_seed(&self, _: Option<StoredValue>) -> Result<f64, litellm_cache::Error> {
+        Ok(0.0)
+    }
+
+    fn counter_value(&self, value: f64) -> StoredValue {
+        if value.fract() == 0.0 {
+            StoredValue::Integer(value as i64)
+        } else {
+            StoredValue::Float(value)
+        }
+    }
+}
+
+#[rstest]
+fn roundtrip_persists_and_reopens(sandbox: Sandbox) {
+    let context = ExactCacheContext::default();
+    let opened = sandbox.cache::<Value>();
+    opened
+        .set_cache("key", json!({"answer": 42}), &context)
+        .unwrap();
+    assert_eq!(
+        opened.get_cache("key", &context).unwrap(),
+        Some(json!({"answer": 42}))
+    );
+    drop(opened);
+    let reopened = sandbox.cache::<Value>();
+    assert_eq!(
+        reopened.get_cache("key", &context).unwrap(),
+        Some(json!({"answer": 42}))
+    );
+}
+
+#[rstest]
+fn ttl_and_expired_culling_match_cache_contract(sandbox: Sandbox) {
+    let store = sandbox.store();
+    store
+        .set(
+            "expired",
+            StoredValue::Bytes(b"old".to_vec()),
+            Some(10.0),
+            0.0,
+        )
+        .unwrap();
+    assert_eq!(store.get("expired", 10.0).unwrap(), None);
+    store
+        .set("new", StoredValue::Bytes(b"new".to_vec()), None, 11.0)
+        .unwrap();
+    assert_eq!(
+        sandbox
+            .db()
+            .query_row("SELECT COUNT(*) FROM Cache", [], |row| row.get::<_, i64>(0))
+            .unwrap(),
+        1
+    );
+    assert_eq!(
+        sandbox
+            .db()
+            .query_row(
+                "SELECT value FROM Settings WHERE key = 'count'",
+                [],
+                |row| row.get::<_, i64>(0)
+            )
+            .unwrap(),
+        1
+    );
+}
+
+#[rstest]
+fn batch_preserves_order_and_classifies_misses_and_invalid_values(sandbox: Sandbox) {
+    let store = sandbox.store();
+    store
+        .set(
+            "hit",
+            StoredValue::Bytes(br#"{"ok":true}"#.to_vec()),
+            None,
+            0.0,
+        )
+        .unwrap();
+    store
+        .set(
+            "invalid",
+            StoredValue::Pickle(vec![0x80, 0x05, 0x2e]),
+            None,
+            0.0,
+        )
+        .unwrap();
+    let entries = sandbox
+        .cache::<Value>()
+        .batch_get_cache(
+            &["hit".into(), "missing".into(), "invalid".into()],
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    assert_eq!(
+        entries,
+        vec![
+            BatchEntry::Hit(json!({"ok": true})),
+            BatchEntry::Miss,
+            BatchEntry::Invalid
+        ]
+    );
+}
+
+#[rstest]
+#[case(StoredValue::Bytes(Vec::new()))]
+#[case(StoredValue::Text(String::new()))]
+#[case(StoredValue::Integer(0))]
+#[case(StoredValue::Float(0.0))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x89, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x00, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x94, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x5d, 0x94, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x7d, 0x94, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x29, 0x2e]))]
+fn falsy_values_are_misses(sandbox: Sandbox, #[case] value: StoredValue) {
+    sandbox.store().set("key", value, None, 0.0).unwrap();
+    assert_eq!(
+        sandbox
+            .cache::<Value>()
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap(),
+        None
+    );
+}
+
+#[rstest]
+#[case(Some(StoredValue::Integer(2)), 1.5, 3.5, "real")]
+#[case(Some(StoredValue::Integer(2)), 1.0, 3.0, "integer")]
+#[case(Some(StoredValue::Float(3.5)), 1.0, 1.0, "integer")]
+#[case(Some(StoredValue::Text("not a number".into())), 2.0, 2.0, "integer")]
+#[case(Some(StoredValue::Text("5".into())), 2.0, 7.0, "integer")]
+#[case(Some(StoredValue::Text("3.5".into())), 2.0, 2.0, "integer")]
+#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e])), 1.0, 2.0, "integer")]
+#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x02, 0x2e])), 1.0, 3.0, "integer")]
+#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e])), 1.0, 1.0, "integer")]
+#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e])), 4.0, 4.0, "integer")]
+fn counters_follow_python_initialization(
+    sandbox: Sandbox,
+    #[case] initial: Option<StoredValue>,
+    #[case] amount: f64,
+    #[case] expected: f64,
+    #[case] sqlite_type: &str,
+) {
+    if let Some(initial) = initial {
+        sandbox.store().set("counter", initial, None, 0.0).unwrap();
+    }
+    let cache = sandbox.cache::<f64>();
+    assert_eq!(
+        cache
+            .increment_cache("counter", amount, ExactCacheContext::default())
+            .unwrap(),
+        expected
+    );
+    assert_eq!(
+        sandbox
+            .db()
+            .query_row(
+                "SELECT typeof(value) FROM Cache WHERE key = 'counter'",
+                [],
+                |row| row.get::<_, String>(0)
+            )
+            .unwrap(),
+        sqlite_type
+    );
+}
+
+#[rstest]
+fn counters_are_atomic_across_concurrent_callers(sandbox: Sandbox) {
+    let cache = Arc::new(sandbox.cache::<f64>());
+    let workers = (0..8)
+        .map(|_| {
+            let cache = Arc::clone(&cache);
+            thread::spawn(move || {
+                for _ in 0..25 {
+                    cache
+                        .increment_cache("counter", 1.0, ExactCacheContext::default())
+                        .unwrap();
+                }
+            })
+        })
+        .collect::<Vec<_>>();
+    for worker in workers {
+        worker.join().unwrap();
+    }
+    assert_eq!(
+        cache
+            .increment_cache("counter", 0.0, ExactCacheContext::default())
+            .unwrap(),
+        200.0
+    );
+}
+
+#[rstest]
+fn fractional_then_integer_increment_follows_python_behavior(sandbox: Sandbox) {
+    let cache = sandbox.cache::<f64>();
+    assert_eq!(
+        cache
+            .increment_cache("counter", 3.5, ExactCacheContext::default())
+            .unwrap(),
+        3.5
+    );
+    assert_eq!(
+        cache
+            .increment_cache("counter", 1.0, ExactCacheContext::default())
+            .unwrap(),
+        1.0
+    );
+}
+
+#[rstest]
+fn increment_ttl_replacement_clears_expiry_without_ttl(sandbox: Sandbox) {
+    let cache = sandbox.cache::<f64>();
+    cache
+        .increment_cache(
+            "counter",
+            1.0,
+            ExactCacheContext {
+                ttl: Some(Duration::from_secs(60)),
+            },
+        )
+        .unwrap();
+    assert!(
+        sandbox
+            .db()
+            .query_row(
+                "SELECT expire_time IS NOT NULL FROM Cache WHERE key = 'counter'",
+                [],
+                |row| row.get::<_, bool>(0)
+            )
+            .unwrap()
+    );
+    cache
+        .increment_cache("counter", 1.0, ExactCacheContext::default())
+        .unwrap();
+    assert!(
+        !sandbox
+            .db()
+            .query_row(
+                "SELECT expire_time IS NOT NULL FROM Cache WHERE key = 'counter'",
+                [],
+                |row| row.get::<_, bool>(0)
+            )
+            .unwrap()
+    );
+}
+
+#[rstest]
+fn custom_adapter_controls_storage_and_reads(sandbox: Sandbox) {
+    let cache = DiskCache::with_adapter(sandbox.store(), TextAdapter, JsonCodec::<Value>::new());
+    cache
+        .set_cache("key", json!({"answer": 42}), &ExactCacheContext::default())
+        .unwrap();
+    assert!(matches!(
+        sandbox.store().get("key", 0.0).unwrap(),
+        Some(StoredValue::Text(_))
+    ));
+    assert_eq!(
+        cache
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap(),
+        Some(json!({"answer": 42}))
+    );
+}
+
+#[rstest]
+fn delete_flush_and_spilled_file_replacement_clean_up_storage(sandbox: Sandbox) {
+    let large = vec![b'x'; 32 * 1024];
+    sandbox
+        .store()
+        .set("large", StoredValue::Bytes(large.clone()), None, 0.0)
+        .unwrap();
+    assert_eq!(sandbox.value_files().len(), 1);
+    sandbox
+        .store()
+        .set(
+            "large",
+            StoredValue::Bytes(vec![b'y'; 32 * 1024]),
+            None,
+            0.0,
+        )
+        .unwrap();
+    assert_eq!(sandbox.value_files().len(), 1);
+    sandbox.store().pop("large", 0.0).unwrap();
+    assert!(sandbox.value_files().is_empty());
+    sandbox
+        .store()
+        .set("a", StoredValue::Bytes(large.clone()), None, 0.0)
+        .unwrap();
+    sandbox
+        .store()
+        .set("b", StoredValue::Bytes(large), None, 0.0)
+        .unwrap();
+    sandbox.store().clear().unwrap();
+    assert!(sandbox.value_files().is_empty());
+}
+
+#[rstest]
+#[tokio::test]
+async fn async_operations_connection_and_delete_match_sync_operations(sandbox: Sandbox) {
+    let cache = sandbox.cache::<Value>();
+    let context = ExactCacheContext {
+        ttl: Some(Duration::from_secs(60)),
+    };
+    cache
+        .async_set_cache("a", json!(1), context.clone())
+        .await
+        .unwrap();
+    cache
+        .async_set_cache_pipeline(
+            vec![("b".into(), json!(2)), ("c".into(), json!(3))],
+            context.clone(),
+        )
+        .await
+        .unwrap();
+    assert_eq!(
+        cache.async_get_cache("a", &context).await.unwrap(),
+        Some(json!(1))
+    );
+    assert_eq!(
+        cache
+            .async_batch_get_cache(vec!["c".into(), "missing".into()], context.clone())
+            .await
+            .unwrap(),
+        vec![BatchEntry::Hit(json!(3)), BatchEntry::Miss]
+    );
+    cache.async_delete_cache("a").await.unwrap();
+    cache.async_flush_cache().await.unwrap();
+    assert_eq!(
+        cache.test_connection().await.unwrap().status,
+        litellm_cache::CacheConnectionStatus::Success
+    );
+}

litellm-rust/crates/cache-disk/tests/python_compat.rs

@@ -0,0 +1,113 @@
+use litellm_cache::Error;
+use litellm_cache_disk::{PythonDiskCacheAdapter, StoredValue, ValueAdapter};
+use rstest::rstest;
+
+enum ReadExpectation {
+    Bytes(&'static [u8]),
+    Miss,
+    Invalid,
+}
+
+#[rstest]
+#[case::pickled_dictionary_with_string_keys(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e]),
+    ReadExpectation::Bytes(br#"{"a":1}"#)
+)]
+#[case::pickled_list_of_integers(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x94, 0x28, 0x4b, 0x01, 0x4b, 0x02, 0x65, 0x2e]),
+    ReadExpectation::Bytes(br#"[1,2]"#)
+)]
+#[case::pickled_tuple_of_integers(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x01, 0x4b, 0x02, 0x86, 0x94, 0x2e]),
+    ReadExpectation::Bytes(br#"[1,2]"#)
+)]
+#[case::pickled_set_of_integers(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x94, 0x28, 0x4b, 0x01, 0x4b, 0x02, 0x90, 0x2e]),
+    ReadExpectation::Bytes(br#"[1,2]"#)
+)]
+#[case::pickled_response_envelope(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x28, 0x8c, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x94, 0x47, 0x3f, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x94, 0x8c, 0x08, 0x7b, 0x22, 0x61, 0x22, 0x3a, 0x20, 0x31, 0x7d, 0x94, 0x75, 0x2e]),
+    ReadExpectation::Bytes(br#"{"response":"{\"a\": 1}","timestamp":1.5}"#)
+)]
+#[case::non_json_text(
+    StoredValue::Text("not json".into()),
+    ReadExpectation::Bytes(b"not json")
+)]
+#[case::json_text(
+    StoredValue::Text("{\"a\": 1}".into()),
+    ReadExpectation::Bytes(br#"{"a": 1}"#)
+)]
+#[case::non_utf8_bytes(
+    StoredValue::Bytes(vec![0xff, 0xfe]),
+    ReadExpectation::Bytes(&[0xff, 0xfe])
+)]
+#[case::integer_seven(StoredValue::Integer(7), ReadExpectation::Bytes(b"7"))]
+#[case::float_one_point_five(StoredValue::Float(1.5), ReadExpectation::Bytes(b"1.5"))]
+#[case::pickled_true(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e]),
+    ReadExpectation::Bytes(b"true")
+)]
+#[case::pickled_negative_integer(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0xfd, 0xff, 0xff, 0xff, 0x2e]),
+    ReadExpectation::Bytes(b"-3")
+)]
+#[case::pickled_bytes(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x05, 0x62, 0x79, 0x74, 0x65, 0x73, 0x94, 0x2e]),
+    ReadExpectation::Invalid
+)]
+#[case::pickled_dictionary_with_integer_key(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x4b, 0x01, 0x8c, 0x01, 0x61, 0x94, 0x73, 0x2e]),
+    ReadExpectation::Invalid
+)]
+#[case::pickled_complex(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x2e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x08, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x73, 0x94, 0x8c, 0x07, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x94, 0x93, 0x94, 0x47, 0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0x94, 0x52, 0x94, 0x2e]),
+    ReadExpectation::Invalid
+)]
+#[case::truncated_pickle(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x2e]),
+    ReadExpectation::Invalid
+)]
+#[case::empty_bytes(StoredValue::Bytes(Vec::new()), ReadExpectation::Miss)]
+#[case::empty_text(StoredValue::Text(String::new()), ReadExpectation::Miss)]
+#[case::zero_integer(StoredValue::Integer(0), ReadExpectation::Miss)]
+#[case::zero_float(StoredValue::Float(0.0), ReadExpectation::Miss)]
+#[case::pickled_none(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_false(StoredValue::Pickle(vec![0x80, 0x05, 0x89, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_zero(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x00, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_zero_float(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_empty_string(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x94, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_empty_list(StoredValue::Pickle(vec![0x80, 0x05, 0x5d, 0x94, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_empty_dictionary(StoredValue::Pickle(vec![0x80, 0x05, 0x7d, 0x94, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_empty_tuple(StoredValue::Pickle(vec![0x80, 0x05, 0x29, 0x2e]), ReadExpectation::Miss)]
+fn python_read_cases(#[case] row: StoredValue, #[case] expected: ReadExpectation) {
+    let result = PythonDiskCacheAdapter.read(row);
+    match expected {
+        ReadExpectation::Bytes(expected) => assert_eq!(result.unwrap().unwrap(), expected),
+        ReadExpectation::Miss => assert_eq!(result.unwrap(), None),
+        ReadExpectation::Invalid => assert!(matches!(result, Err(Error::InvalidEntry))),
+    }
+}
+
+#[rstest]
+#[case::integer_two(Some(StoredValue::Integer(2)), 2.0)]
+#[case::float_three_point_five(Some(StoredValue::Float(3.5)), 0.0)]
+#[case::text_not_a_number(Some(StoredValue::Text("not a number".into())), 0.0)]
+#[case::text_five(Some(StoredValue::Text("5".into())), 5.0)]
+#[case::text_three_point_five(Some(StoredValue::Text("3.5".into())), 0.0)]
+#[case::pickled_true(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e])), 1.0)]
+#[case::pickled_two(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x02, 0x2e])), 2.0)]
+#[case::pickled_dictionary(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e])), 0.0)]
+#[case::missing(None, 0.0)]
+#[case::pickled_none(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e])), 0.0)]
+fn python_counter_seed_cases(#[case] row: Option<StoredValue>, #[case] expected: f64) {
+    assert_eq!(PythonDiskCacheAdapter.counter_seed(row).unwrap(), expected);
+}
+
+#[rstest]
+#[case::integer_three(3.0, StoredValue::Integer(3))]
+#[case::fractional_three_point_five(3.5, StoredValue::Float(3.5))]
+#[case::negative_zero(-0.0, StoredValue::Integer(0))]
+#[case::large_float(1e300, StoredValue::Float(1e300))]
+fn python_counter_value_cases(#[case] value: f64, #[case] expected: StoredValue) {
+    assert_eq!(PythonDiskCacheAdapter.counter_value(value), expected);
+}

litellm-rust/crates/cache-gcs/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "litellm-cache-gcs"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+futures-util.workspace = true
+litellm-auth-gcp.workspace = true
+litellm-auth-types.workspace = true
+litellm-cache.workspace = true
+percent-encoding.workspace = true
+reqwest.workspace = true
+tokio.workspace = true
+
+[dev-dependencies]
+serde_json.workspace = true
+tokio.workspace = true
+wiremock = "0.6.5"

litellm-rust/crates/cache-gcs/src/cache.rs

@@ -0,0 +1,260 @@
+use std::{future::Future, sync::Arc, time::Duration};
+
+use futures_util::future::try_join_all;
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, Error, ExactCacheContext,
+    FlushCache,
+};
+use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode};
+use reqwest::Client;
+
+use crate::{GcpTokenSource, TokenSource};
+
+pub const DEFAULT_ENDPOINT: &str = "https://storage.googleapis.com";
+
+const OBJECT_NAME_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC
+    .remove(b'-')
+    .remove(b'_')
+    .remove(b'.')
+    .remove(b'~');
+
+pub fn key_prefix(gcs_path: Option<&str>) -> String {
+    match gcs_path {
+        Some(path) if !path.is_empty() => format!("{}/", path.trim_end_matches('/')),
+        _ => String::new(),
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct GcsConfig {
+    pub bucket_name: String,
+    pub gcs_path: Option<String>,
+    pub path_service_account: Option<String>,
+    pub endpoint: String,
+}
+
+impl GcsConfig {
+    pub fn new(bucket_name: impl Into<String>) -> Self {
+        Self {
+            bucket_name: bucket_name.into(),
+            gcs_path: None,
+            path_service_account: None,
+            endpoint: DEFAULT_ENDPOINT.to_string(),
+        }
+    }
+}
+
+pub struct GcsCache<S: CacheCodec> {
+    config: GcsConfig,
+    key_prefix: String,
+    client: Client,
+    token: Arc<dyn TokenSource>,
+    codec: S,
+}
+
+impl<S: CacheCodec> GcsCache<S> {
+    pub fn new(config: GcsConfig, codec: S) -> Result<Self, Error> {
+        let token = Arc::new(GcpTokenSource::new(config.path_service_account.clone()));
+        Self::with_token_source(config, codec, token)
+    }
+
+    pub fn with_token_source(
+        config: GcsConfig,
+        codec: S,
+        token: Arc<dyn TokenSource>,
+    ) -> Result<Self, Error> {
+        let client = Client::builder().build().map_err(|_| Error::Unavailable)?;
+        let key_prefix = key_prefix(config.gcs_path.as_deref());
+        Ok(Self {
+            config,
+            key_prefix,
+            client,
+            token,
+            codec,
+        })
+    }
+
+    pub fn bucket_name(&self) -> &str {
+        &self.config.bucket_name
+    }
+
+    pub fn key_prefix(&self) -> &str {
+        &self.key_prefix
+    }
+
+    pub fn path_service_account(&self) -> Option<&str> {
+        self.config.path_service_account.as_deref()
+    }
+
+    pub fn object_name(&self, key: &str) -> String {
+        format!("{}{}", self.key_prefix, key)
+    }
+
+    fn encoded_object_name(&self, key: &str) -> String {
+        percent_encode(self.object_name(key).as_bytes(), OBJECT_NAME_ENCODE_SET).to_string()
+    }
+
+    fn endpoint(&self, path: &str) -> String {
+        format!("{}{}", self.config.endpoint.trim_end_matches('/'), path)
+    }
+
+    async fn async_set(&self, key: &str, value: S::Value) -> Result<(), Error> {
+        let token = self.token.bearer_token().await?;
+        let payload = self.codec.encode(&value)?;
+        let url = self.endpoint(&format!(
+            "/upload/storage/v1/b/{}/o?uploadType=media&name={}",
+            self.config.bucket_name,
+            self.encoded_object_name(key)
+        ));
+        let response = self
+            .client
+            .post(url)
+            .bearer_auth(token)
+            .header(reqwest::header::CONTENT_TYPE, "application/json")
+            .body(payload)
+            .send()
+            .await
+            .map_err(|_| Error::Unavailable)?;
+        if !response.status().is_success() {
+            return Err(Error::Unavailable);
+        }
+        Ok(())
+    }
+
+    async fn async_get(&self, key: &str) -> Result<Option<S::Value>, Error> {
+        let token = self.token.bearer_token().await?;
+        let url = self.endpoint(&format!(
+            "/storage/v1/b/{}/o/{}?alt=media",
+            self.config.bucket_name,
+            self.encoded_object_name(key)
+        ));
+        let response = self
+            .client
+            .get(url)
+            .bearer_auth(token)
+            .header(reqwest::header::CONTENT_TYPE, "application/json")
+            .send()
+            .await
+            .map_err(|_| Error::Unavailable)?;
+        if response.status() == reqwest::StatusCode::NOT_FOUND {
+            return Ok(None);
+        }
+        if !response.status().is_success() {
+            return Err(Error::Unavailable);
+        }
+        let body = response.bytes().await.map_err(|_| Error::Unavailable)?;
+        self.codec
+            .decode(&body)
+            .map(Some)
+            .map_err(|_| Error::InvalidEntry)
+    }
+
+    fn run_sync<T, F>(future: F) -> Result<T, Error>
+    where
+        F: Future<Output = Result<T, Error>> + Send,
+        T: Send,
+    {
+        let run = || {
+            tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .map_err(|_| Error::Unavailable)
+                .and_then(|runtime| runtime.block_on(future))
+        };
+        if let Ok(handle) = tokio::runtime::Handle::try_current() {
+            if handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread {
+                return tokio::task::block_in_place(run);
+            }
+            return std::thread::scope(|scope| {
+                scope
+                    .spawn(run)
+                    .join()
+                    .map_err(|_| Error::Unavailable)
+                    .and_then(|result| result)
+            });
+        }
+        run()
+    }
+}
+
+impl<S: CacheCodec> BaseCache for GcsCache<S> {
+    type Value = S::Value;
+    type Context = ExactCacheContext;
+
+    fn get_ttl(&self, _: &Self::Context) -> Option<Duration> {
+        None
+    }
+
+    fn set_cache(&self, key: &str, value: Self::Value, _: &Self::Context) -> Result<(), Error> {
+        Self::run_sync(self.async_set(key, value))
+    }
+
+    fn get_cache(&self, key: &str, _: &Self::Context) -> Result<Option<Self::Value>, Error> {
+        Self::run_sync(self.async_get(key))
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        _: Self::Context,
+    ) -> Result<(), Error> {
+        self.async_set(key, value).await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        _: &Self::Context,
+    ) -> Result<Option<Self::Value>, Error> {
+        self.async_get(key).await
+    }
+
+    async fn async_set_cache_pipeline(
+        &self,
+        entries: Vec<(String, Self::Value)>,
+        context: Self::Context,
+    ) -> Result<(), Error> {
+        try_join_all(entries.into_iter().map(|(key, value)| {
+            let context = context.clone();
+            async move { self.async_set_cache(&key, value, context).await }
+        }))
+        .await
+        .map(|_| ())
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        Err(Error::UnsupportedOperation)
+    }
+}
+
+impl<S: CacheCodec> BatchCache for GcsCache<S> {
+    async fn async_batch_get_cache(
+        &self,
+        keys: Vec<String>,
+        context: Self::Context,
+    ) -> Result<Vec<BatchEntry<Self::Value>>, Error> {
+        try_join_all(keys.into_iter().map(|key| {
+            let context = context.clone();
+            async move {
+                match self.async_get_cache(&key, &context).await {
+                    Ok(Some(value)) => Ok(BatchEntry::Hit(value)),
+                    Ok(None) => Ok(BatchEntry::Miss),
+                    Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid),
+                    Err(error) => Err(error),
+                }
+            }
+        }))
+        .await
+    }
+}
+
+impl<S: CacheCodec> FlushCache for GcsCache<S> {
+    fn flush_cache(&self) -> Result<(), Error> {
+        Ok(())
+    }
+}

litellm-rust/crates/cache-gcs/src/lib.rs

@@ -0,0 +1,5 @@
+mod cache;
+mod token;
+
+pub use cache::{DEFAULT_ENDPOINT, GcsCache, GcsConfig, key_prefix};
+pub use token::{GcpTokenSource, StaticTokenSource, TokenSource};

litellm-rust/crates/cache-gcs/src/token.rs

@@ -0,0 +1,44 @@
+use std::{future::Future, pin::Pin};
+
+use litellm_auth_gcp::{VertexAuth, VertexConfig};
+use litellm_auth_types::{InputSource, SecretValue, Sourced};
+use litellm_cache::Error;
+
+pub trait TokenSource: Send + Sync + 'static {
+    fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>>;
+}
+
+pub struct GcpTokenSource {
+    auth: VertexAuth,
+    config: VertexConfig,
+}
+
+impl GcpTokenSource {
+    pub fn new(path_service_account: Option<String>) -> Self {
+        let credentials = path_service_account
+            .map(|path| Sourced::new(SecretValue::new(path), InputSource::Deployment));
+        Self {
+            auth: VertexAuth::default(),
+            config: VertexConfig::new(credentials, None, None),
+        }
+    }
+}
+
+impl TokenSource for GcpTokenSource {
+    fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>> {
+        Box::pin(async move {
+            self.auth
+                .access_token(&self.config, &|name| std::env::var(name).ok())
+                .await
+                .map_err(|_| Error::Unavailable)
+        })
+    }
+}
+
+pub struct StaticTokenSource(pub String);
+
+impl TokenSource for StaticTokenSource {
+    fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>> {
+        Box::pin(async move { Ok(self.0.clone()) })
+    }
+}

litellm-rust/crates/cache-gcs/tests/cache.rs

@@ -0,0 +1,324 @@
+use std::{sync::Arc, time::Duration};
+
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheContext, Error, ExactCacheContext, FlushCache,
+    JsonCodec,
+};
+use litellm_cache_gcs::{GcsCache, GcsConfig, StaticTokenSource, TokenSource, key_prefix};
+use serde_json::json;
+use wiremock::{
+    Mock, MockServer, ResponseTemplate,
+    matchers::{body_bytes, header, method, path, query_param},
+};
+
+fn config(server: &MockServer, gcs_path: Option<&str>) -> GcsConfig {
+    GcsConfig {
+        bucket_name: "bucket".into(),
+        gcs_path: gcs_path.map(str::to_string),
+        path_service_account: None,
+        endpoint: server.uri(),
+    }
+}
+
+fn cache(server: &MockServer, gcs_path: Option<&str>) -> GcsCache<JsonCodec<serde_json::Value>> {
+    GcsCache::with_token_source(
+        config(server, gcs_path),
+        JsonCodec::new(),
+        Arc::new(StaticTokenSource("tok".into())),
+    )
+    .unwrap()
+}
+
+#[tokio::test]
+async fn set_writes_encoded_object_and_headers() {
+    let server = MockServer::start().await;
+    Mock::given(method("POST"))
+        .and(path("/upload/storage/v1/b/bucket/o"))
+        .and(query_param("uploadType", "media"))
+        .and(header("authorization", "Bearer tok"))
+        .and(header("content-type", "application/json"))
+        .and(body_bytes(br#"{"value":"entry"}"#))
+        .respond_with(ResponseTemplate::new(200))
+        .expect(1)
+        .mount(&server)
+        .await;
+    cache(&server, Some("cache/"))
+        .set_cache(
+            "team:a b/c",
+            json!({"value": "entry"}),
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    let requests = server.received_requests().await.unwrap();
+    assert_eq!(requests.len(), 1);
+    assert_eq!(
+        requests[0].url.query(),
+        Some("uploadType=media&name=cache%2Fteam%3Aa%20b%2Fc")
+    );
+}
+
+#[tokio::test]
+async fn get_maps_statuses_and_decode_failures() {
+    let server = MockServer::start().await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/hit"))
+        .and(query_param("alt", "media"))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/missing"))
+        .respond_with(ResponseTemplate::new(404))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/server-error"))
+        .respond_with(ResponseTemplate::new(500))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/invalid"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("not json"))
+        .mount(&server)
+        .await;
+
+    let cache = cache(&server, None);
+    assert_eq!(
+        cache
+            .get_cache("hit", &ExactCacheContext::default())
+            .unwrap(),
+        Some(json!({"value": "entry"}))
+    );
+    assert_eq!(
+        cache
+            .get_cache("missing", &ExactCacheContext::default())
+            .unwrap(),
+        None
+    );
+    assert_eq!(
+        cache
+            .get_cache("server-error", &ExactCacheContext::default())
+            .unwrap_err(),
+        Error::Unavailable
+    );
+    assert_eq!(
+        cache
+            .get_cache("invalid", &ExactCacheContext::default())
+            .unwrap_err(),
+        Error::InvalidEntry
+    );
+}
+
+#[test]
+fn key_prefix_normalizes_paths() {
+    assert_eq!(key_prefix(None), "");
+    assert_eq!(key_prefix(Some("a/b/")), "a/b/");
+    assert_eq!(key_prefix(Some("a/b")), "a/b/");
+    assert_eq!(key_prefix(Some("")), "");
+}
+
+#[tokio::test]
+async fn object_names_use_python_quote_encoding() {
+    let server = MockServer::start().await;
+    Mock::given(method("POST"))
+        .and(path("/upload/storage/v1/b/bucket/o"))
+        .and(query_param("uploadType", "media"))
+        .respond_with(ResponseTemplate::new(200))
+        .expect(2)
+        .mount(&server)
+        .await;
+    let cache = cache(&server, Some("p/"));
+    cache
+        .set_cache(
+            "a~b-c_d.e/f g%h",
+            json!({"value": "punctuation"}),
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    cache
+        .set_cache(
+            "ключ",
+            json!({"value": "utf8"}),
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    let requests = server.received_requests().await.unwrap();
+    let queries: Vec<_> = requests
+        .iter()
+        .filter_map(|request| request.url.query())
+        .collect();
+    assert!(queries.contains(&"uploadType=media&name=p%2Fa~b-c_d.e%2Ff%20g%25h"));
+    assert!(queries.contains(&"uploadType=media&name=p%2F%D0%BA%D0%BB%D1%8E%D1%87"));
+}
+
+#[tokio::test]
+async fn ignores_ttl_and_writes_pipeline_concurrently() {
+    let server = MockServer::start().await;
+    for key in ["one", "two", "three"] {
+        Mock::given(method("POST"))
+            .and(path("/upload/storage/v1/b/bucket/o"))
+            .and(query_param("uploadType", "media"))
+            .and(query_param("name", key))
+            .respond_with(ResponseTemplate::new(200))
+            .expect(1)
+            .mount(&server)
+            .await;
+    }
+    let cache = cache(&server, None);
+    assert_eq!(cache.get_ttl(&ExactCacheContext::default()), None);
+    assert_eq!(
+        cache.get_ttl(&ExactCacheContext::default().with_ttl(Some(Duration::from_secs(5)))),
+        None
+    );
+    cache
+        .async_set_cache_pipeline(
+            vec![
+                ("one".into(), json!({"key": "one"})),
+                ("two".into(), json!({"key": "two"})),
+                ("three".into(), json!({"key": "three"})),
+            ],
+            ExactCacheContext::default().with_ttl(Some(Duration::from_secs(5))),
+        )
+        .await
+        .unwrap();
+}
+
+#[tokio::test]
+async fn async_batch_get_preserves_hits_misses_and_invalid_entries() {
+    let server = MockServer::start().await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/hit"))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/missing"))
+        .respond_with(ResponseTemplate::new(404))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/invalid"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("not json"))
+        .mount(&server)
+        .await;
+
+    assert_eq!(
+        cache(&server, None)
+            .async_batch_get_cache(
+                vec!["hit".into(), "missing".into(), "invalid".into()],
+                ExactCacheContext::default(),
+            )
+            .await
+            .unwrap(),
+        vec![
+            BatchEntry::Hit(json!({"value": "entry"})),
+            BatchEntry::Miss,
+            BatchEntry::Invalid,
+        ]
+    );
+}
+
+#[tokio::test]
+async fn lifecycle_operations_are_noops_and_connection_test_is_unsupported() {
+    let server = MockServer::start().await;
+    let cache = cache(&server, None);
+    assert_eq!(cache.flush_cache(), Ok(()));
+    assert_eq!(cache.disconnect().await, Ok(()));
+    assert_eq!(
+        cache.test_connection().await,
+        Err(Error::UnsupportedOperation)
+    );
+}
+
+#[test]
+fn sync_operations_work_without_an_active_runtime() {
+    let runtime = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+    let server = runtime.block_on(MockServer::start());
+    runtime.block_on(
+        Mock::given(method("POST"))
+            .and(path("/upload/storage/v1/b/bucket/o"))
+            .respond_with(ResponseTemplate::new(200))
+            .mount(&server),
+    );
+    runtime.block_on(
+        Mock::given(method("GET"))
+            .and(path("/storage/v1/b/bucket/o/key"))
+            .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})))
+            .mount(&server),
+    );
+    let cache = cache(&server, None);
+    cache
+        .set_cache(
+            "key",
+            json!({"value": "entry"}),
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    assert_eq!(
+        cache
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap(),
+        Some(json!({"value": "entry"}))
+    );
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn sync_operations_work_inside_a_multi_thread_runtime() {
+    let server = MockServer::start().await;
+    Mock::given(method("POST"))
+        .and(path("/upload/storage/v1/b/bucket/o"))
+        .respond_with(ResponseTemplate::new(200))
+        .mount(&server)
+        .await;
+    Mock::given(method("GET"))
+        .and(path("/storage/v1/b/bucket/o/key"))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})))
+        .mount(&server)
+        .await;
+    let cache = cache(&server, None);
+    cache
+        .set_cache(
+            "key",
+            json!({"value": "entry"}),
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    assert_eq!(
+        cache
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap(),
+        Some(json!({"value": "entry"}))
+    );
+}
+
+struct FailingTokenSource;
+
+impl TokenSource for FailingTokenSource {
+    fn bearer_token(
+        &self,
+    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<String, Error>> + Send + '_>>
+    {
+        Box::pin(async { Err(Error::Unavailable) })
+    }
+}
+
+#[tokio::test]
+async fn token_source_failure_skips_http() {
+    let server = MockServer::start().await;
+    let cache = GcsCache::with_token_source(
+        config(&server, None),
+        JsonCodec::<serde_json::Value>::new(),
+        Arc::new(FailingTokenSource),
+    )
+    .unwrap();
+    assert_eq!(
+        cache
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap_err(),
+        Error::Unavailable
+    );
+    assert_eq!(server.received_requests().await.unwrap().len(), 0);
+}

litellm-rust/crates/cache-memory/Cargo.toml

@@ -7,8 +7,8 @@ repository.workspace = true
 
 [dependencies]
 litellm-cache.workspace = true
-serde_json.workspace = true
 
 [dev-dependencies]
+serde_json.workspace = true
 rstest.workspace = true
 tokio.workspace = true

litellm-rust/crates/cache-memory/src/cache.rs

@@ -1,18 +1,20 @@
-use std::cmp::Reverse;
-use std::collections::{BinaryHeap, HashMap};
-use std::sync::{Arc, Mutex};
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use std::{
+    cmp::Reverse,
+    collections::{BinaryHeap, HashMap, HashSet},
+    hash::Hash,
+    sync::{Arc, Mutex},
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
 
 use litellm_cache::{
-    BaseCache, CacheConnectionResult, CacheConnectionStatus, CacheEntry, CacheFuture, CacheKwargs,
-    Error,
+    BaseCache, BatchCache, CacheConnectionResult, CacheConnectionStatus, ClaimCache, CounterCache,
+    DeleteCache, Error, ExactCacheContext, FlushCache, IncrementOperation, SetCache, TtlCache,
 };
 
 const DEFAULT_MAX_SIZE_IN_MEMORY: usize = 200;
 const DEFAULT_TTL: Duration = Duration::from_secs(600);
 
 type ValueMeasure<V> = Arc<dyn Fn(&V) -> Result<usize, Error> + Send + Sync>;
-type ValueValidator<V> = Arc<dyn Fn(&V) -> Result<(), Error> + Send + Sync>;
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum CacheWrite {
@@ -33,7 +35,6 @@ pub struct InMemoryCache<V: Clone> {
     default_ttl: Duration,
     max_entry_bytes: Option<usize>,
     measure_value: Option<ValueMeasure<V>>,
-    validate_value: Option<ValueValidator<V>>,
     now: Arc<dyn Fn() -> Duration + Send + Sync>,
 }
 
@@ -77,7 +78,6 @@ impl<V: Clone> InMemoryCache<V> {
             default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
             max_entry_bytes,
             measure_value,
-            validate_value: None,
             now: Arc::new(now),
         }
     }
@@ -91,9 +91,6 @@ impl<V: Clone> InMemoryCache<V> {
         if self.max_size_in_memory == 0 {
             return Ok(CacheWrite::Disabled);
         }
-        if let Some(validate) = &self.validate_value {
-            validate(&value)?;
-        }
         if let (Some(limit), Some(measure)) = (self.max_entry_bytes, &self.measure_value)
             && measure(&value)? > limit
         {
@@ -101,15 +98,13 @@ impl<V: Clone> InMemoryCache<V> {
         }
         let now = (self.now)();
         let mut state = self.state.lock().map_err(|_| Error::Unavailable)?;
-        Self::evict(&mut state, self.max_size_in_memory, now);
         let key = key.into();
-        state.values.insert(key.clone(), value);
+        Self::evict(&mut state, self.max_size_in_memory, now, &key);
         let expiration = state.expirations.get(&key).copied();
         if expiration.is_none_or(|expiration| expiration < now) {
-            let expiration = now + ttl.unwrap_or(self.default_ttl);
-            state.expirations.insert(key.clone(), expiration);
-            state.expiration_heap.push(Reverse((expiration, key)));
+            Self::set_expiration(&mut state, &key, now + ttl.unwrap_or(self.default_ttl));
         }
+        state.values.insert(key, value);
         Ok(CacheWrite::Stored)
     }
 
@@ -126,6 +121,14 @@ impl<V: Clone> InMemoryCache<V> {
         Ok(state.values.get(key).cloned())
     }
 
+    pub fn max_size_in_memory(&self) -> usize {
+        self.max_size_in_memory
+    }
+
+    pub fn max_entry_bytes(&self) -> Option<usize> {
+        self.max_entry_bytes
+    }
+
     pub fn expires_at(&self, key: &str) -> Result<Option<Duration>, Error> {
         Ok(self
             .state
@@ -136,6 +139,25 @@ impl<V: Clone> InMemoryCache<V> {
             .copied())
     }
 
+    pub async fn async_get_ttl(&self, key: &str) -> Result<Option<Duration>, Error> {
+        self.expires_at(key)
+    }
+
+    pub async fn async_get_oldest_n_keys(&self, count: usize) -> Result<Vec<String>, Error> {
+        let state = self.state.lock().map_err(|_| Error::Unavailable)?;
+        let mut expirations = state
+            .expirations
+            .iter()
+            .map(|(key, expiration)| (key.clone(), *expiration))
+            .collect::<Vec<_>>();
+        expirations.sort_unstable_by_key(|(_, expiration)| *expiration);
+        Ok(expirations
+            .into_iter()
+            .take(count)
+            .map(|(key, _)| key)
+            .collect())
+    }
+
     pub fn delete_cache(&self, key: &str) -> Result<(), Error> {
         let mut state = self.state.lock().map_err(|_| Error::Unavailable)?;
         Self::remove(&mut state, key);
@@ -150,7 +172,7 @@ impl<V: Clone> InMemoryCache<V> {
         Ok(())
     }
 
-    fn evict(state: &mut CacheState<V>, capacity: usize, now: Duration) {
+    fn evict(state: &mut CacheState<V>, capacity: usize, now: Duration, key: &str) {
         while let Some(Reverse((expiration, key))) = state.expiration_heap.peek().cloned() {
             if state.expirations.get(&key).copied() != Some(expiration) {
                 state.expiration_heap.pop();
@@ -161,6 +183,9 @@ impl<V: Clone> InMemoryCache<V> {
                 break;
             }
         }
+        if state.values.contains_key(key) {
+            return;
+        }
         while state.values.len() >= capacity {
             let Some(Reverse((expiration, key))) = state.expiration_heap.pop() else {
                 break;
@@ -171,84 +196,205 @@ impl<V: Clone> InMemoryCache<V> {
         }
     }
 
+    fn set_expiration(state: &mut CacheState<V>, key: &str, expiration: Duration) {
+        if state.expirations.get(key).copied() != Some(expiration) {
+            state.expirations.insert(key.into(), expiration);
+            state
+                .expiration_heap
+                .push(Reverse((expiration, key.into())));
+        }
+    }
+
     fn remove(state: &mut CacheState<V>, key: &str) {
         state.values.remove(key);
         state.expirations.remove(key);
     }
 }
 
-impl InMemoryCache<CacheEntry> {
-    pub fn response_cache(capacity: usize, ttl: Duration, max_entry_bytes: usize) -> Self {
-        Self::response_cache_with_clock(capacity, ttl, max_entry_bytes, || {
-            SystemTime::now()
-                .duration_since(UNIX_EPOCH)
-                .unwrap_or_default()
-        })
-    }
-
-    pub fn response_cache_with_clock(
-        capacity: usize,
-        ttl: Duration,
-        max_entry_bytes: usize,
-        now: impl Fn() -> Duration + Send + Sync + 'static,
-    ) -> Self {
-        let mut cache = Self::with_clock_and_size_measurement(
-            Some(capacity),
-            Some(ttl),
-            Some(max_entry_bytes),
-            Some(Arc::new(|entry: &CacheEntry| {
-                serde_json::to_vec(entry)
-                    .map(|bytes| bytes.len())
-                    .map_err(|_| Error::InvalidEntry)
-            })),
-            now,
+impl<V> ClaimCache for InMemoryCache<V>
+where
+    V: Clone + PartialEq + Send + Sync + 'static,
+{
+    fn claim_cache(
+        &self,
+        key: &str,
+        candidate: V,
+        eligible: &[V],
+        context: ExactCacheContext,
+    ) -> Result<V, Error> {
+        if self.max_size_in_memory == 0 {
+            return Ok(candidate);
+        }
+        let now = (self.now)();
+        let mut state = self.state.lock().map_err(|_| Error::Unavailable)?;
+        Self::evict(&mut state, self.max_size_in_memory, now, key);
+        let existing = state
+            .values
+            .get(key)
+            .filter(|existing| eligible.is_empty() || eligible.contains(existing))
+            .cloned();
+        if let Some(existing) = &existing
+            && eligible.is_empty()
+            && *existing != candidate
+        {
+            return Ok(existing.clone());
+        }
+        let winner = existing.unwrap_or(candidate);
+        Self::set_expiration(
+            &mut state,
+            key,
+            now + self.get_ttl(&context).unwrap_or(self.default_ttl),
         );
-        cache.validate_value = Some(Arc::new(|entry: &CacheEntry| {
-            entry
-                .timestamp
-                .is_finite()
-                .then_some(())
-                .ok_or(Error::InvalidEntry)
-        }));
-        cache
+        state.values.insert(key.into(), winner.clone());
+        Ok(winner)
     }
 }
 
-impl BaseCache for InMemoryCache<CacheEntry> {
-    type Value = CacheEntry;
+impl CounterCache for InMemoryCache<f64> {
+    fn increment_cache(
+        &self,
+        key: &str,
+        amount: f64,
+        context: ExactCacheContext,
+    ) -> Result<f64, Error> {
+        if self.max_size_in_memory == 0 {
+            return Ok(amount);
+        }
+        let now = (self.now)();
+        let mut state = self.state.lock().map_err(|_| Error::Unavailable)?;
+        Self::evict(&mut state, self.max_size_in_memory, now, key);
+        let value = state.values.get(key).copied().unwrap_or_default() + amount;
+        if !state.expirations.contains_key(key) {
+            Self::set_expiration(
+                &mut state,
+                key,
+                now + self.get_ttl(&context).unwrap_or(self.default_ttl),
+            );
+        }
+        state.values.insert(key.into(), value);
+        Ok(value)
+    }
+}
 
-    fn default_ttl(&self) -> Duration {
-        self.default_ttl
+impl InMemoryCache<f64> {
+    pub async fn async_increment_pipeline(
+        &self,
+        operations: Vec<IncrementOperation>,
+    ) -> Result<Vec<f64>, Error> {
+        operations
+            .into_iter()
+            .map(|operation| {
+                self.increment_cache(
+                    &operation.key,
+                    operation.amount,
+                    ExactCacheContext { ttl: operation.ttl },
+                )
+            })
+            .collect()
+    }
+}
+
+impl<V: Clone + Send + Sync + 'static> BaseCache for InMemoryCache<V> {
+    type Value = V;
+    type Context = ExactCacheContext;
+
+    fn get_ttl(&self, context: &Self::Context) -> Option<Duration> {
+        context.ttl.or(Some(self.default_ttl))
     }
 
-    fn set_cache(&self, key: &str, value: Self::Value, kwargs: CacheKwargs) -> Result<(), Error> {
-        let ttl = self.get_ttl(&kwargs);
+    fn set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: &ExactCacheContext,
+    ) -> Result<(), Error> {
+        let ttl = self.get_ttl(context).unwrap_or(self.default_ttl);
         self.set_cache(key, value, Some(ttl)).map(|_| ())
     }
 
-    fn get_cache(&self, key: &str, _: &CacheKwargs) -> Result<Option<Self::Value>, Error> {
+    fn get_cache(&self, key: &str, _: &ExactCacheContext) -> Result<Option<Self::Value>, Error> {
         self.get_cache(key)
     }
 
-    fn delete_cache(&self, key: &str) -> Result<(), Error> {
-        self.delete_cache(key)
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
     }
 
-    fn flush_cache(&self) -> Result<(), Error> {
-        self.flush_cache()
-    }
-
-    fn disconnect(&self) -> CacheFuture<'_, ()> {
-        Box::pin(async { Ok(()) })
-    }
-
-    fn test_connection(&self) -> CacheFuture<'_, CacheConnectionResult> {
-        Box::pin(async {
-            Ok(CacheConnectionResult {
-                status: CacheConnectionStatus::Success,
-                message: "In-memory cache connection test successful".into(),
-                error: None,
-            })
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        Ok(CacheConnectionResult {
+            status: CacheConnectionStatus::Success,
+            message: "In-memory cache connection test successful".into(),
+            error: None,
         })
     }
 }
+
+impl<V: Clone + Send + Sync + 'static> BatchCache for InMemoryCache<V> {}
+
+impl<V: Clone + Send + Sync + 'static> DeleteCache for InMemoryCache<V> {
+    fn delete_cache(&self, key: &str) -> Result<(), Error> {
+        InMemoryCache::delete_cache(self, key)
+    }
+}
+
+impl<V: Clone + Send + Sync + 'static> FlushCache for InMemoryCache<V> {
+    fn flush_cache(&self) -> Result<(), Error> {
+        InMemoryCache::flush_cache(self)
+    }
+}
+
+impl<V: Clone + Send + Sync + 'static> TtlCache for InMemoryCache<V> {
+    async fn async_get_ttl(&self, key: &str) -> Result<Option<Duration>, Error> {
+        InMemoryCache::async_get_ttl(self, key).await
+    }
+}
+
+impl<T> SetCache for InMemoryCache<HashSet<T>>
+where
+    T: Clone + Eq + Hash + Send + Sync + 'static,
+{
+    type SetValue = T;
+    type SetResult = Vec<T>;
+
+    async fn async_set_cache_sadd(
+        &self,
+        key: &str,
+        values: Vec<Self::SetValue>,
+        ttl: Option<Duration>,
+    ) -> Result<Self::SetResult, Error> {
+        if self.max_size_in_memory == 0 {
+            return Ok(values);
+        }
+        let now = (self.now)();
+        let mut state = self.state.lock().map_err(|_| Error::Unavailable)?;
+        Self::evict(&mut state, self.max_size_in_memory, now, key);
+        let mut stored = state.values.get(key).cloned().unwrap_or_default();
+        stored.extend(values.iter().cloned());
+        if let (Some(limit), Some(measure)) = (self.max_entry_bytes, &self.measure_value)
+            && measure(&stored)? > limit
+        {
+            return Ok(values);
+        }
+        if !state.expirations.contains_key(key) {
+            Self::set_expiration(&mut state, key, now + ttl.unwrap_or(self.default_ttl));
+        }
+        state.values.insert(key.into(), stored);
+        Ok(values)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn repeated_increments_keep_one_heap_entry_per_expiration() {
+        let cache = InMemoryCache::<f64>::new(Some(4), None);
+        for _ in 0..100 {
+            cache
+                .increment_cache("counter", 1.0, ExactCacheContext::default())
+                .unwrap();
+        }
+        assert_eq!(cache.state.lock().unwrap().expiration_heap.len(), 1);
+    }
+}

litellm-rust/crates/cache-memory/tests/cache.rs

@@ -1,8 +1,16 @@
-use std::sync::Arc;
-use std::sync::atomic::{AtomicU64, Ordering};
-use std::time::Duration;
+use std::{
+    collections::HashSet,
+    sync::{
+        Arc,
+        atomic::{AtomicU64, Ordering},
+    },
+    time::Duration,
+};
 
-use litellm_cache::{BaseCache, CacheConnectionStatus, CacheEntry, Error};
+use litellm_cache::{
+    BaseCache, CacheBackend, CacheConnectionStatus, ClaimCache, CounterCache, DeleteCache, Error,
+    ExactCacheContext, IncrementOperation, SetCache, get_cache, set_cache,
+};
 use litellm_cache_memory::{CacheWrite, InMemoryCache};
 use rstest::{fixture, rstest};
 
@@ -84,66 +92,49 @@ fn capacity_evicts_earliest_and_ignores_stale_heap_entries(clock: Arc<AtomicU64>
 }
 
 #[test]
-fn disabled_size_limited_and_synchronized_response_writes_are_observable() {
-    let disabled = InMemoryCache::<CacheEntry>::response_cache(0, Duration::from_secs(60), 80);
+fn disabled_size_limited_and_validated_writes_are_observable() {
+    let cache = |capacity| {
+        InMemoryCache::with_clock_and_size_measurement(
+            Some(capacity),
+            Some(Duration::from_secs(60)),
+            Some(4),
+            Some(Arc::new(|value: &String| {
+                if value.is_empty() {
+                    return Err(Error::InvalidEntry);
+                }
+                Ok(value.len())
+            })),
+            || Duration::from_secs(100),
+        )
+    };
+    let disabled = cache(0);
     assert_eq!(
-        disabled
-            .set_cache(
-                "a",
-                CacheEntry {
-                    timestamp: 1.0,
-                    response: serde_json::json!("x")
-                },
-                None
-            )
-            .unwrap(),
+        disabled.set_cache("a", "x".into(), None).unwrap(),
         CacheWrite::Disabled
     );
-    let cache = InMemoryCache::<CacheEntry>::response_cache(2, Duration::from_secs(60), 80);
+    let cache = cache(2);
     assert_eq!(
-        cache
-            .set_cache(
-                "large",
-                CacheEntry {
-                    timestamp: 1.0,
-                    response: serde_json::json!("x".repeat(100))
-                },
-                None
-            )
-            .unwrap(),
+        cache.set_cache("large", "oversized".into(), None).unwrap(),
         CacheWrite::TooLarge
     );
-    cache
-        .set_cache(
-            "small",
-            CacheEntry {
-                timestamp: 1.0,
-                response: serde_json::json!("ok"),
-            },
-            None,
-        )
-        .unwrap();
-    assert!(cache.get_cache("small").unwrap().is_some());
+    assert_eq!(cache.get_cache("large").unwrap(), None);
     assert_eq!(
-        cache
-            .set_cache(
-                "invalid",
-                CacheEntry {
-                    timestamp: f64::NAN,
-                    response: serde_json::json!("bad"),
-                },
-                None,
-            )
-            .unwrap_err(),
-        Error::InvalidEntry
+        cache.set_cache("small", "ok".into(), None).unwrap(),
+        CacheWrite::Stored
     );
+    assert_eq!(cache.get_cache("small").unwrap(), Some("ok".into()));
+    assert_eq!(
+        cache.set_cache("invalid", String::new(), None),
+        Err(Error::InvalidEntry)
+    );
+    assert_eq!(cache.get_cache("invalid").unwrap(), None);
     cache.delete_cache("small").unwrap();
-    cache.flush_cache().unwrap();
+    assert_eq!(cache.get_cache("small").unwrap(), None);
 }
 
 #[tokio::test]
 async fn connection_test_matches_python_result_contract() {
-    let cache = InMemoryCache::<CacheEntry>::default();
+    let cache = InMemoryCache::<String>::default();
     let result = BaseCache::test_connection(&cache).await.unwrap();
     assert_eq!(result.status, CacheConnectionStatus::Success);
     assert_eq!(result.message, "In-memory cache connection test successful");
@@ -156,3 +147,222 @@ async fn connection_test_matches_python_result_contract() {
         })
     );
 }
+
+#[tokio::test]
+async fn generic_consumers_share_typed_values_and_honor_expiration() {
+    let clock = clock();
+    let cache: CacheBackend<InMemoryCache<String>> = Arc::new(cache(clock.clone(), 4));
+    let reader = Arc::clone(&cache);
+    let context = ExactCacheContext {
+        ttl: Some(Duration::from_secs(5)),
+    };
+    set_cache(cache.as_ref(), "sync", "first".into(), &context).unwrap();
+    assert_eq!(
+        get_cache(reader.as_ref(), "sync", &context).unwrap(),
+        Some("first".into())
+    );
+    cache
+        .batch_cache_write("async", "second".into(), context.clone())
+        .await
+        .unwrap();
+    cache
+        .async_set_cache_pipeline(vec![("batch".into(), "third".into())], context.clone())
+        .await
+        .unwrap();
+    drop(cache);
+    for (key, value) in [("sync", "first"), ("async", "second"), ("batch", "third")] {
+        assert_eq!(
+            reader.async_get_cache(key, &context).await.unwrap(),
+            Some(value.into())
+        );
+    }
+    reader.async_delete_cache("async").await.unwrap();
+    assert_eq!(
+        reader.async_get_cache("async", &context).await.unwrap(),
+        None
+    );
+    clock.store(106, Ordering::SeqCst);
+    assert_eq!(get_cache(reader.as_ref(), "sync", &context).unwrap(), None);
+    assert_eq!(
+        reader.async_get_cache("batch", &context).await.unwrap(),
+        None
+    );
+}
+
+#[test]
+fn claims_are_atomic_and_refresh_eligible_winners() {
+    let clock = clock();
+    let cache = InMemoryCache::with_clock(Some(4), Some(Duration::from_secs(60)), {
+        let clock = clock.clone();
+        move || Duration::from_secs(clock.load(Ordering::SeqCst))
+    });
+    let context = ExactCacheContext {
+        ttl: Some(Duration::from_secs(10)),
+    };
+    assert_eq!(
+        cache
+            .claim_cache("affinity", "first".to_string(), &[], context.clone())
+            .unwrap(),
+        "first"
+    );
+    clock.store(103, Ordering::SeqCst);
+    assert_eq!(
+        cache
+            .claim_cache("affinity", "second".to_string(), &[], context.clone())
+            .unwrap(),
+        "first"
+    );
+    assert_eq!(
+        cache.expires_at("affinity").unwrap(),
+        Some(Duration::from_secs(110))
+    );
+    clock.store(105, Ordering::SeqCst);
+    assert_eq!(
+        cache
+            .claim_cache(
+                "affinity",
+                "second".to_string(),
+                &["first".to_string(), "second".to_string()],
+                context,
+            )
+            .unwrap(),
+        "first"
+    );
+    assert_eq!(
+        cache.expires_at("affinity").unwrap(),
+        Some(Duration::from_secs(115))
+    );
+}
+
+#[test]
+fn counters_increment_under_one_lock() {
+    let cache = InMemoryCache::<f64>::default();
+    assert_eq!(
+        CounterCache::increment_cache(&cache, "counter", 1.5, ExactCacheContext::default())
+            .unwrap(),
+        1.5
+    );
+    assert_eq!(
+        CounterCache::increment_cache(&cache, "counter", 2.0, ExactCacheContext::default())
+            .unwrap(),
+        3.5
+    );
+}
+
+#[rstest]
+fn rewriting_an_existing_key_at_capacity_keeps_other_entries(clock: Arc<AtomicU64>) {
+    let cache = cache(clock, 2);
+    cache
+        .set_cache("hot", "1".into(), Some(Duration::from_secs(10)))
+        .unwrap();
+    cache
+        .set_cache("cold", "2".into(), Some(Duration::from_secs(20)))
+        .unwrap();
+
+    cache.set_cache("cold", "3".into(), None).unwrap();
+    assert_eq!(cache.get_cache("hot").unwrap(), Some("1".into()));
+    assert_eq!(cache.get_cache("cold").unwrap(), Some("3".into()));
+
+    cache
+        .claim_cache("cold", "4".into(), &[], ExactCacheContext::default())
+        .unwrap();
+    assert_eq!(cache.get_cache("hot").unwrap(), Some("1".into()));
+
+    cache.set_cache("new", "5".into(), None).unwrap();
+    assert_eq!(cache.get_cache("hot").unwrap(), None);
+    assert_eq!(cache.get_cache("cold").unwrap(), Some("3".into()));
+    assert_eq!(cache.get_cache("new").unwrap(), Some("5".into()));
+}
+
+#[test]
+fn incrementing_an_existing_counter_at_capacity_keeps_every_counter() {
+    let cache = InMemoryCache::<f64>::new(Some(2), None);
+    for key in ["a", "b", "a", "b"] {
+        cache
+            .increment_cache(key, 1.0, ExactCacheContext::default())
+            .unwrap();
+    }
+    assert_eq!(cache.get_cache("a").unwrap(), Some(2.0));
+    assert_eq!(cache.get_cache("b").unwrap(), Some(2.0));
+}
+
+#[test]
+fn disabled_cache_does_not_retain_claims_or_counters() {
+    let claims = InMemoryCache::<String>::new(Some(0), None);
+    assert_eq!(
+        claims
+            .claim_cache("key", "first".into(), &[], ExactCacheContext::default())
+            .unwrap(),
+        "first"
+    );
+    assert_eq!(claims.get_cache("key").unwrap(), None);
+
+    let counters = InMemoryCache::<f64>::new(Some(0), None);
+    assert_eq!(
+        counters
+            .increment_cache("key", 2.0, ExactCacheContext::default())
+            .unwrap(),
+        2.0
+    );
+    assert_eq!(counters.get_cache("key").unwrap(), None);
+}
+
+#[tokio::test]
+async fn ttl_and_oldest_key_operations_use_the_stored_expirations() {
+    let clock = Arc::new(AtomicU64::new(100));
+    let cache = cache(clock, 3);
+    cache
+        .set_cache("later", "2".into(), Some(Duration::from_secs(20)))
+        .unwrap();
+    cache
+        .set_cache("first", "1".into(), Some(Duration::from_secs(10)))
+        .unwrap();
+
+    assert_eq!(
+        cache.async_get_ttl("first").await.unwrap(),
+        Some(Duration::from_secs(110))
+    );
+    assert_eq!(cache.async_get_oldest_n_keys(1).await.unwrap(), ["first"]);
+    assert_eq!(cache.async_get_ttl("missing").await.unwrap(), None);
+}
+
+#[tokio::test]
+async fn increment_pipeline_preserves_operation_order() {
+    let cache = InMemoryCache::<f64>::new(Some(3), None);
+    assert_eq!(
+        cache
+            .async_increment_pipeline(vec![
+                IncrementOperation {
+                    key: "a".into(),
+                    amount: 1.0,
+                    ttl: Some(Duration::from_secs(10)),
+                },
+                IncrementOperation {
+                    key: "a".into(),
+                    amount: 2.0,
+                    ttl: Some(Duration::from_secs(20)),
+                },
+            ])
+            .await
+            .unwrap(),
+        [1.0, 3.0]
+    );
+    assert_eq!(cache.get_cache("a").unwrap(), Some(3.0));
+}
+
+#[tokio::test]
+async fn set_capability_preserves_python_result_and_deduplicates_storage() {
+    let cache = InMemoryCache::<HashSet<String>>::new(None, None);
+    let inserted = vec!["a".into(), "a".into(), "b".into()];
+    assert_eq!(
+        cache
+            .async_set_cache_sadd("members", inserted.clone(), None)
+            .await
+            .unwrap(),
+        inserted
+    );
+    assert_eq!(
+        cache.get_cache("members").unwrap(),
+        Some(HashSet::from(["a".into(), "b".into()]))
+    );
+}

litellm-rust/crates/cache-qdrant-semantic/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "litellm-cache-qdrant-semantic"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+futures-util.workspace = true
+litellm-cache.workspace = true
+qdrant-client = { workspace = true, features = ["serde"] }
+reqwest.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+tokio.workspace = true
+uuid.workspace = true
+
+[dev-dependencies]
+litellm-cache-response.workspace = true
+rstest.workspace = true
+tonic = "0.14"
+tonic-prost = "0.14"
+tokio-stream = "0.1"

litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs

@@ -0,0 +1,75 @@
+use std::time::Duration;
+
+use litellm_cache::Error;
+use reqwest::Client;
+use serde_json::Value;
+
+use crate::Embedder;
+
+pub struct OpenAiEmbedder {
+    client: Client,
+    api_base: String,
+    api_key: String,
+    model: String,
+    timeout: Option<Duration>,
+}
+
+pub struct OpenAiEmbedderConfig {
+    pub api_base: String,
+    pub api_key: String,
+    pub model: String,
+    pub timeout: Option<Duration>,
+}
+
+impl OpenAiEmbedder {
+    pub fn new(client: Client, config: OpenAiEmbedderConfig) -> Self {
+        Self {
+            client,
+            api_base: config.api_base.trim_end_matches('/').to_owned(),
+            api_key: config.api_key,
+            model: config.model,
+            timeout: config.timeout,
+        }
+    }
+}
+
+impl Embedder for OpenAiEmbedder {
+    fn model(&self) -> &str {
+        &self.model
+    }
+
+    async fn embed(&self, input: &str) -> Result<Vec<f32>, Error> {
+        let request = self
+            .client
+            .post(format!("{}/embeddings", self.api_base))
+            .bearer_auth(&self.api_key)
+            .json(&serde_json::json!({
+                "model": self.model,
+                "input": input,
+                "encoding_format": "float",
+            }));
+        let response = if let Some(timeout) = self.timeout {
+            request.timeout(timeout)
+        } else {
+            request
+        }
+        .send()
+        .await
+        .map_err(|_| Error::Unavailable)?
+        .error_for_status()
+        .map_err(|_| Error::Unavailable)?;
+        let body: Value = response.json().await.map_err(|_| Error::Unavailable)?;
+        body.get("data")
+            .and_then(Value::as_array)
+            .and_then(|data| data.first())
+            .and_then(|item| item.get("embedding"))
+            .and_then(Value::as_array)
+            .and_then(|embedding| {
+                embedding
+                    .iter()
+                    .map(|value| value.as_f64().map(|value| value as f32))
+                    .collect::<Option<Vec<_>>>()
+            })
+            .ok_or(Error::Unavailable)
+    }
+}

litellm-rust/crates/cache-qdrant-semantic/src/lib.rs

@@ -0,0 +1,7 @@
+mod embedder;
+mod prompt;
+mod semantic;
+
+pub use embedder::{OpenAiEmbedder, OpenAiEmbedderConfig};
+pub use prompt::prompt_from_messages;
+pub use semantic::{Embedder, QdrantSemanticCache, QdrantSemanticConfig, Quantization};

litellm-rust/crates/cache-qdrant-semantic/src/prompt.rs

@@ -0,0 +1,59 @@
+use serde_json::Value;
+
+fn search_results_text(search_results: Option<&Value>) -> String {
+    let Some(Value::Array(results)) = search_results else {
+        return String::new();
+    };
+    results
+        .iter()
+        .filter_map(Value::as_object)
+        .flat_map(|result| {
+            let source = result
+                .get("source")
+                .and_then(Value::as_str)
+                .map(str::to_owned);
+            let title = result
+                .get("title")
+                .and_then(Value::as_str)
+                .map(str::to_owned);
+            let content = result
+                .get("content")
+                .and_then(Value::as_array)
+                .into_iter()
+                .flatten()
+                .filter_map(Value::as_object)
+                .filter_map(|block| block.get("text").and_then(Value::as_str).map(str::to_owned));
+            let citations = result
+                .get("citations")
+                .filter(|value| !value.is_null())
+                .map(|value| serde_json::to_string(value).unwrap_or_default());
+            source
+                .into_iter()
+                .chain(title)
+                .chain(content)
+                .chain(citations)
+        })
+        .collect()
+}
+
+pub fn prompt_from_messages(messages: &[Value]) -> String {
+    messages
+        .iter()
+        .filter_map(Value::as_object)
+        .map(|message| {
+            let content = match message.get("content") {
+                Some(Value::String(content)) => content.clone(),
+                Some(Value::Array(parts)) => parts
+                    .iter()
+                    .filter_map(Value::as_object)
+                    .filter_map(|part| part.get("text").and_then(Value::as_str))
+                    .collect(),
+                _ => String::new(),
+            };
+            format!(
+                "{content}{}",
+                search_results_text(message.get("search_results"))
+            )
+        })
+        .collect()
+}

litellm-rust/crates/cache-qdrant-semantic/src/semantic.rs

@@ -0,0 +1,262 @@
+use std::future::Future;
+
+use futures_util::future::try_join_all;
+use litellm_cache::{BaseCache, CacheCodec, CacheConnectionResult, Error, SemanticCacheContext};
+use qdrant_client::{
+    Payload, Qdrant,
+    qdrant::{
+        BinaryQuantizationBuilder, CompressionRatio, Condition, CreateCollectionBuilder,
+        CreateFieldIndexCollectionBuilder, Distance, FieldType, Filter, PointStruct,
+        ProductQuantizationBuilder, QuantizationSearchParamsBuilder, ScalarQuantizationBuilder,
+        SearchParamsBuilder, SearchPointsBuilder, UpsertPointsBuilder, VectorParamsBuilder,
+    },
+};
+use serde_json::{Map, Value, json};
+use uuid::Uuid;
+
+use crate::prompt_from_messages;
+
+pub trait Embedder: Send + Sync + 'static {
+    fn model(&self) -> &str;
+    fn embed(&self, input: &str) -> impl Future<Output = Result<Vec<f32>, Error>> + Send;
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum Quantization {
+    Binary,
+    Scalar,
+    Product,
+}
+
+pub struct QdrantSemanticConfig {
+    pub collection_name: String,
+    pub similarity_threshold: f64,
+    pub vector_size: u64,
+    pub quantization: Quantization,
+}
+
+pub struct QdrantSemanticCache<E: Embedder, C: CacheCodec> {
+    client: Qdrant,
+    embedder: E,
+    codec: C,
+    config: QdrantSemanticConfig,
+    runtime: tokio::runtime::Handle,
+}
+
+impl<E: Embedder, C: CacheCodec> QdrantSemanticCache<E, C> {
+    pub async fn connect(
+        client: Qdrant,
+        embedder: E,
+        codec: C,
+        config: QdrantSemanticConfig,
+        runtime: tokio::runtime::Handle,
+    ) -> Result<Self, Error> {
+        let exists = client
+            .collection_exists(config.collection_name.clone())
+            .await
+            .map_err(|_| Error::Unavailable)?;
+        if !exists {
+            client
+                .create_collection(
+                    CreateCollectionBuilder::new(config.collection_name.clone())
+                        .vectors_config(VectorParamsBuilder::new(
+                            config.vector_size,
+                            Distance::Cosine,
+                        ))
+                        .quantization_config(quantization(&config.quantization)),
+                )
+                .await
+                .map_err(|_| Error::Unavailable)?;
+        }
+        let _ = client
+            .create_field_index(CreateFieldIndexCollectionBuilder::new(
+                config.collection_name.clone(),
+                "litellm_cache_key".to_owned(),
+                FieldType::Keyword,
+            ))
+            .await;
+        Ok(Self {
+            client,
+            embedder,
+            codec,
+            config,
+            runtime,
+        })
+    }
+
+    pub fn collection_name(&self) -> &str {
+        &self.config.collection_name
+    }
+
+    pub fn similarity_threshold(&self) -> f64 {
+        self.config.similarity_threshold
+    }
+
+    pub fn vector_size(&self) -> u64 {
+        self.config.vector_size
+    }
+
+    pub fn embedder(&self) -> &E {
+        &self.embedder
+    }
+
+    fn prompt(context: &SemanticCacheContext) -> Result<String, Error> {
+        let Some(messages) = context.messages.as_ref().and_then(Value::as_array) else {
+            return Err(Error::MissingPrompt);
+        };
+        if messages.is_empty() {
+            return Err(Error::MissingPrompt);
+        }
+        Ok(prompt_from_messages(messages))
+    }
+
+    async fn set(
+        &self,
+        key: &str,
+        value: C::Value,
+        context: &SemanticCacheContext,
+    ) -> Result<(), Error> {
+        let prompt = Self::prompt(context)?;
+        let vector = self.embedder.embed(&prompt).await?;
+        let response =
+            String::from_utf8(self.codec.encode(&value)?).map_err(|_| Error::InvalidEntry)?;
+        let payload = Payload::try_from(json!({
+            "litellm_cache_key": key,
+            "text": prompt,
+            "response": response,
+        }))
+        .map_err(|_| Error::InvalidEntry)?;
+        self.client
+            .upsert_points(
+                UpsertPointsBuilder::new(
+                    self.collection_name(),
+                    vec![PointStruct::new(
+                        Uuid::new_v4().to_string(),
+                        vector,
+                        payload,
+                    )],
+                )
+                .wait(true),
+            )
+            .await
+            .map_err(|_| Error::Unavailable)?;
+        Ok(())
+    }
+
+    async fn get(
+        &self,
+        key: &str,
+        context: &SemanticCacheContext,
+    ) -> Result<Option<C::Value>, Error> {
+        let prompt = Self::prompt(context)?;
+        let vector = self.embedder.embed(&prompt).await?;
+        let result = self
+            .client
+            .search_points(
+                SearchPointsBuilder::new(self.collection_name(), vector, 1)
+                    .with_payload(true)
+                    .filter(Filter::must([Condition::matches(
+                        "litellm_cache_key",
+                        key.to_owned(),
+                    )]))
+                    .params(
+                        SearchParamsBuilder::default().quantization(
+                            QuantizationSearchParamsBuilder::default()
+                                .ignore(false)
+                                .rescore(true)
+                                .oversampling(3.0),
+                        ),
+                    ),
+            )
+            .await
+            .map_err(|_| Error::Unavailable)?;
+        let Some(point) = result.result.into_iter().next() else {
+            return Ok(None);
+        };
+        let payload: Map<String, Value> = Payload::from(point.payload).into();
+        if payload.get("litellm_cache_key").and_then(Value::as_str) != Some(key) {
+            return Ok(None);
+        }
+        if f64::from(point.score) < self.config.similarity_threshold {
+            return Ok(None);
+        }
+        let response = payload
+            .get("response")
+            .and_then(Value::as_str)
+            .ok_or(Error::InvalidEntry)?;
+        self.codec.decode(response.as_bytes()).map(Some)
+    }
+}
+
+fn quantization(value: &Quantization) -> qdrant_client::qdrant::quantization_config::Quantization {
+    match value {
+        Quantization::Binary => BinaryQuantizationBuilder::new(false).into(),
+        Quantization::Scalar => ScalarQuantizationBuilder::default()
+            .quantile(0.99)
+            .always_ram(false)
+            .into(),
+        Quantization::Product => ProductQuantizationBuilder::new(CompressionRatio::X16.into())
+            .always_ram(false)
+            .into(),
+    }
+}
+
+impl<E: Embedder, C: CacheCodec> BaseCache for QdrantSemanticCache<E, C> {
+    type Value = C::Value;
+    type Context = SemanticCacheContext;
+
+    fn get_ttl(&self, _: &Self::Context) -> Option<std::time::Duration> {
+        None
+    }
+
+    fn set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: &Self::Context,
+    ) -> Result<(), Error> {
+        self.runtime.block_on(self.set(key, value, context))
+    }
+
+    fn get_cache(&self, key: &str, context: &Self::Context) -> Result<Option<Self::Value>, Error> {
+        self.runtime.block_on(self.get(key, context))
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: Self::Context,
+    ) -> Result<(), Error> {
+        self.set(key, value, &context).await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        context: &Self::Context,
+    ) -> Result<Option<Self::Value>, Error> {
+        self.get(key, context).await
+    }
+
+    async fn async_set_cache_pipeline(
+        &self,
+        entries: Vec<(String, Self::Value)>,
+        context: Self::Context,
+    ) -> Result<(), Error> {
+        try_join_all(entries.into_iter().map(|(key, value)| {
+            let context = context.clone();
+            async move { self.async_set_cache(&key, value, context).await }
+        }))
+        .await
+        .map(|_| ())
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        Err(Error::UnsupportedOperation)
+    }
+}

litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs

@@ -0,0 +1,166 @@
+use std::{
+    sync::{Arc, Mutex},
+    time::Duration,
+};
+
+use litellm_cache::Error;
+use litellm_cache_qdrant_semantic::{Embedder, OpenAiEmbedder, OpenAiEmbedderConfig};
+use serde_json::Value;
+use tokio::{
+    io::{AsyncReadExt, AsyncWriteExt},
+    net::TcpListener,
+};
+
+struct TestHttpServer {
+    address: std::net::SocketAddr,
+    request: Arc<Mutex<Option<Vec<u8>>>>,
+    task: tokio::task::JoinHandle<()>,
+}
+
+impl TestHttpServer {
+    async fn response(status: &str, body: &str) -> Self {
+        Self::response_after(status, body, Duration::ZERO).await
+    }
+
+    async fn response_after(status: &str, body: &str, delay: Duration) -> Self {
+        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let address = listener.local_addr().unwrap();
+        let request = Arc::new(Mutex::new(None));
+        let captured = request.clone();
+        let status = status.to_owned();
+        let body = body.to_owned();
+        let task = tokio::spawn(async move {
+            let (mut stream, _) = listener.accept().await.unwrap();
+            let request_bytes = read_request(&mut stream).await;
+            *captured.lock().unwrap() = Some(request_bytes);
+            tokio::time::sleep(delay).await;
+            let response = format!(
+                "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+                body.len()
+            );
+            stream.write_all(response.as_bytes()).await.unwrap();
+        });
+        Self {
+            address,
+            request,
+            task,
+        }
+    }
+
+    fn base_url(&self) -> String {
+        format!("http://{}", self.address)
+    }
+}
+
+impl Drop for TestHttpServer {
+    fn drop(&mut self) {
+        self.task.abort();
+    }
+}
+
+async fn read_request(stream: &mut tokio::net::TcpStream) -> Vec<u8> {
+    let mut bytes = Vec::new();
+    let header_end = loop {
+        let mut chunk = [0_u8; 1024];
+        let count = stream.read(&mut chunk).await.unwrap();
+        assert_ne!(count, 0);
+        bytes.extend_from_slice(&chunk[..count]);
+        if let Some(end) = bytes.windows(4).position(|window| window == b"\r\n\r\n") {
+            break end + 4;
+        }
+    };
+    let headers = String::from_utf8_lossy(&bytes[..header_end]);
+    let content_length = headers
+        .lines()
+        .find_map(|line| {
+            line.split_once(':')
+                .filter(|(name, _)| name.eq_ignore_ascii_case("content-length"))
+                .map(|(_, value)| value.trim())
+        })
+        .unwrap()
+        .parse::<usize>()
+        .unwrap();
+    while bytes.len() < header_end + content_length {
+        let mut chunk = [0_u8; 1024];
+        let count = stream.read(&mut chunk).await.unwrap();
+        assert_ne!(count, 0);
+        bytes.extend_from_slice(&chunk[..count]);
+    }
+    bytes
+}
+
+fn config(base: String, timeout: Option<Duration>) -> OpenAiEmbedderConfig {
+    OpenAiEmbedderConfig {
+        api_base: base,
+        api_key: "test-key".to_owned(),
+        model: "test-model".to_owned(),
+        timeout,
+    }
+}
+
+#[tokio::test]
+async fn posts_embeddings_request_and_parses_vector() {
+    let server = TestHttpServer::response("200 OK", r#"{"data":[{"embedding":[0.1,0.2]}]}"#).await;
+    let embedder = OpenAiEmbedder::new(
+        reqwest::Client::new(),
+        config(
+            format!("{}/", server.base_url()),
+            Some(Duration::from_secs(1)),
+        ),
+    );
+    assert_eq!(embedder.embed("hello").await.unwrap(), vec![0.1, 0.2]);
+    let request = server.request.lock().unwrap().clone().unwrap();
+    let request_text = String::from_utf8(request).unwrap();
+    assert!(request_text.starts_with("POST /embeddings HTTP/1.1\r\n"));
+    assert!(request_text.contains("\r\nauthorization: Bearer test-key\r\n"));
+    let body = request_text.split("\r\n\r\n").nth(1).unwrap();
+    let body: Value = serde_json::from_str(body).unwrap();
+    assert_eq!(body["model"], "test-model");
+    assert_eq!(body["input"], "hello");
+    assert_eq!(body["encoding_format"], "float");
+}
+
+#[tokio::test]
+async fn status_and_timeout_errors_are_unavailable() {
+    let server = TestHttpServer::response("500 Internal Server Error", "{}").await;
+    let embedder = OpenAiEmbedder::new(reqwest::Client::new(), config(server.base_url(), None));
+    assert_eq!(embedder.embed("hello").await, Err(Error::Unavailable));
+
+    let server = TestHttpServer::response_after(
+        "200 OK",
+        r#"{"data":[{"embedding":[0.1,0.2]}]}"#,
+        Duration::from_millis(500),
+    )
+    .await;
+    let embedder = OpenAiEmbedder::new(
+        reqwest::Client::new(),
+        config(server.base_url(), Some(Duration::from_millis(200))),
+    );
+    assert_eq!(embedder.embed("hello").await, Err(Error::Unavailable));
+
+    let server = TestHttpServer::response_after(
+        "200 OK",
+        r#"{"data":[{"embedding":[0.1,0.2]}]}"#,
+        Duration::from_millis(100),
+    )
+    .await;
+    let embedder = OpenAiEmbedder::new(
+        reqwest::Client::new(),
+        config(server.base_url(), Some(Duration::from_secs(1))),
+    );
+    assert_eq!(embedder.embed("hello").await.unwrap(), vec![0.1, 0.2]);
+}
+
+#[tokio::test]
+async fn uses_the_injected_client() {
+    let server = TestHttpServer::response("200 OK", r#"{"data":[{"embedding":[0.1,0.2]}]}"#).await;
+    let client = reqwest::Client::builder()
+        .user_agent("litellm-embedder-test")
+        .build()
+        .unwrap();
+    let embedder = OpenAiEmbedder::new(client, config(server.base_url(), None));
+    assert_eq!(embedder.embed("hello").await.unwrap(), vec![0.1, 0.2]);
+    let request = server.request.lock().unwrap().clone().unwrap();
+    let request_text = String::from_utf8(request).unwrap();
+    assert!(request_text.contains("\r\nuser-agent: litellm-embedder-test\r\n"));
+}

litellm-rust/crates/cache-qdrant-semantic/tests/prompt.rs

@@ -0,0 +1,38 @@
+use litellm_cache_qdrant_semantic::prompt_from_messages;
+use serde_json::json;
+
+#[test]
+fn prompt_matches_python_message_content_rules() {
+    let messages = vec![
+        json!({"role": "user", "content": "hello"}),
+        json!({
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "world"},
+                {"type": "image_url", "image_url": {"url": "ignored"}},
+                {"type": "text", "text": "!"},
+            ],
+        }),
+    ];
+
+    assert_eq!(prompt_from_messages(&messages), "helloworld!");
+}
+
+#[test]
+fn prompt_includes_search_result_text_and_compact_citations() {
+    let messages = vec![json!({
+        "role": "tool",
+        "content": null,
+        "search_results": [{
+            "source": "source",
+            "title": "title",
+            "content": [{"text": "body"}],
+            "citations": {"page": 1, "section": "intro"},
+        }],
+    })];
+
+    assert_eq!(
+        prompt_from_messages(&messages),
+        r#"sourcetitlebody{"page":1,"section":"intro"}"#
+    );
+}

litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs

@@ -0,0 +1,422 @@
+#[path = "support/mod.rs"]
+mod support;
+
+use std::{collections::HashMap, sync::Arc, time::Duration};
+
+use litellm_cache::{BaseCache, CacheCodec, CacheContext, Error, SemanticCacheContext};
+use litellm_cache_qdrant_semantic::{
+    Embedder, QdrantSemanticCache, QdrantSemanticConfig, Quantization,
+};
+use litellm_cache_response::{
+    CacheEntry, CacheKeyInput, ResponseCache, ResponseCacheCodec, ResponseCacheRequest,
+};
+use qdrant_client::Payload;
+use qdrant_client::{
+    Qdrant,
+    qdrant::{self, CompressionRatio, Distance, PointId, QuantizationType, Value, VectorParams},
+};
+use serde_json::{Value as JsonValue, json};
+
+use support::{FakeQdrant, FakeState, StoredPoint};
+
+#[derive(Clone)]
+struct FixedEmbedder {
+    vectors: Arc<HashMap<String, Vec<f32>>>,
+}
+
+impl FixedEmbedder {
+    fn new(vectors: impl IntoIterator<Item = (&'static str, Vec<f32>)>) -> Self {
+        Self {
+            vectors: Arc::new(
+                vectors
+                    .into_iter()
+                    .map(|(prompt, vector)| (prompt.to_owned(), vector))
+                    .collect(),
+            ),
+        }
+    }
+}
+
+impl Embedder for FixedEmbedder {
+    fn model(&self) -> &str {
+        "fixed"
+    }
+
+    async fn embed(&self, input: &str) -> Result<Vec<f32>, Error> {
+        self.vectors.get(input).cloned().ok_or(Error::Unavailable)
+    }
+}
+
+fn config(quantization: Quantization) -> QdrantSemanticConfig {
+    QdrantSemanticConfig {
+        collection_name: "semantic".to_owned(),
+        similarity_threshold: 0.9,
+        vector_size: 2,
+        quantization,
+    }
+}
+
+fn context(prompt: &str) -> SemanticCacheContext {
+    SemanticCacheContext {
+        messages: Some(json!([{"role": "user", "content": prompt}])),
+        ..Default::default()
+    }
+}
+
+fn value(response: JsonValue) -> CacheEntry {
+    CacheEntry {
+        timestamp: Some(1.0),
+        response,
+    }
+}
+
+async fn connect(
+    server: &FakeQdrant,
+    vectors: impl IntoIterator<Item = (&'static str, Vec<f32>)>,
+) -> QdrantSemanticCache<FixedEmbedder, ResponseCacheCodec> {
+    let client = Qdrant::from_url(&server.url()).build().unwrap();
+    QdrantSemanticCache::connect(
+        client,
+        FixedEmbedder::new(vectors),
+        ResponseCacheCodec,
+        config(Quantization::Binary),
+        tokio::runtime::Handle::current(),
+    )
+    .await
+    .unwrap()
+}
+
+#[tokio::test(flavor = "multi_thread")]
+#[expect(
+    deprecated,
+    reason = "the test verifies Qdrant's legacy always_ram quantization contract"
+)]
+async fn connect_sets_collection_quantization_and_index() {
+    for (quantization, expected) in [
+        (Quantization::Binary, 0),
+        (Quantization::Scalar, 1),
+        (Quantization::Product, 2),
+    ] {
+        let server = FakeQdrant::start(FakeState::default()).await;
+        let client = Qdrant::from_url(&server.url()).build().unwrap();
+        QdrantSemanticCache::connect(
+            client,
+            FixedEmbedder::new([]),
+            ResponseCacheCodec,
+            config(quantization),
+            tokio::runtime::Handle::current(),
+        )
+        .await
+        .unwrap();
+        let state = server.state.lock().unwrap();
+        let request = &state.created_collections[0];
+        let Some(qdrant::vectors_config::Config::Params(VectorParams { size, distance, .. })) =
+            request
+                .vectors_config
+                .as_ref()
+                .and_then(|config| config.config.clone())
+        else {
+            panic!("missing vector params");
+        };
+        assert_eq!(size, 2);
+        assert_eq!(distance, Distance::Cosine as i32);
+        let quantization_config = request
+            .quantization_config
+            .as_ref()
+            .unwrap()
+            .quantization
+            .unwrap();
+        match (expected, quantization_config) {
+            (0, qdrant::quantization_config::Quantization::Binary(binary)) => {
+                assert_eq!(binary.always_ram, Some(false));
+            }
+            (1, qdrant::quantization_config::Quantization::Scalar(scalar)) => {
+                assert_eq!(scalar.r#type, QuantizationType::Int8 as i32);
+                assert_eq!(scalar.quantile, Some(0.99));
+                assert_eq!(scalar.always_ram, Some(false));
+            }
+            (2, qdrant::quantization_config::Quantization::Product(product)) => {
+                assert_eq!(product.compression, CompressionRatio::X16 as i32);
+                assert_eq!(product.always_ram, Some(false));
+            }
+            _ => panic!("unexpected quantization"),
+        }
+        assert!(state.index_creations >= 1);
+        assert_eq!(state.field_indexes[0].collection_name, "semantic");
+        assert_eq!(state.field_indexes[0].field_name, "litellm_cache_key");
+        assert_eq!(
+            state.field_indexes[0].field_type,
+            Some(qdrant::FieldType::Keyword as i32)
+        );
+        server.stop();
+    }
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn existing_collection_skips_create_and_index_failure_is_non_fatal() {
+    let server = FakeQdrant::start(FakeState {
+        collections: ["semantic".to_owned()].into_iter().collect(),
+        fail_field_index: true,
+        ..Default::default()
+    })
+    .await;
+    let _cache = connect(&server, [("hello", vec![1.0, 0.0])]).await;
+    let state = server.state.lock().unwrap();
+    assert!(state.created_collections.is_empty());
+    assert!(state.index_creations >= 1);
+    server.stop();
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn async_and_sync_set_get_store_exact_payload() {
+    let server = FakeQdrant::start(FakeState::default()).await;
+    let cache = Arc::new(connect(&server, [("hello", vec![1.0, 0.0])]).await);
+    let ctx = context("hello");
+    let entry = value(json!({"answer": 42}));
+    cache
+        .async_set_cache("key", entry.clone(), ctx.clone())
+        .await
+        .unwrap();
+    assert_eq!(
+        cache.async_get_cache("key", &ctx).await.unwrap().as_ref(),
+        Some(&entry)
+    );
+    {
+        let state = server.state.lock().unwrap();
+        let payload = &state.points[0].payload;
+        let mut payload_keys = payload.keys().cloned().collect::<Vec<_>>();
+        payload_keys.sort();
+        assert_eq!(payload_keys, ["litellm_cache_key", "response", "text"]);
+        assert_eq!(payload["litellm_cache_key"], Value::from("key"));
+        assert_eq!(
+            payload["response"],
+            Value::from(String::from_utf8(ResponseCacheCodec.encode(&entry).unwrap()).unwrap())
+        );
+    }
+    let sync_entry = entry.clone();
+    let sync_cache = cache.clone();
+    let sync_ctx = ctx.clone();
+    tokio::task::spawn_blocking(move || {
+        sync_cache
+            .set_cache("sync", sync_entry.clone(), &sync_ctx)
+            .unwrap();
+        assert_eq!(
+            sync_cache.get_cache("sync", &sync_ctx).unwrap(),
+            Some(sync_entry)
+        );
+    })
+    .await
+    .unwrap();
+    server.stop();
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn misses_and_payload_validation_are_safe() {
+    let server = FakeQdrant::start(FakeState::default()).await;
+    let cache = connect(
+        &server,
+        [("hello", vec![1.0, 0.0]), ("near", vec![0.7, 0.71414286])],
+    )
+    .await;
+    let entry = value(json!({"answer": 1}));
+    cache
+        .async_set_cache("key", entry, context("hello"))
+        .await
+        .unwrap();
+    assert_eq!(
+        cache
+            .async_get_cache("other", &context("hello"))
+            .await
+            .unwrap(),
+        None
+    );
+    assert_eq!(
+        cache
+            .async_get_cache("key", &context("near"))
+            .await
+            .unwrap(),
+        None
+    );
+    server.insert_point(StoredPoint {
+        id: Some(PointId::from(99_u64)),
+        vector: vec![1.0, 0.0],
+        payload: Payload::try_from(json!({
+            "litellm_cache_key": 99,
+            "response": "{}",
+        }))
+        .unwrap()
+        .into(),
+    });
+    assert_eq!(
+        cache
+            .async_get_cache("99", &context("hello"))
+            .await
+            .unwrap(),
+        None
+    );
+    server.stop();
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn decoding_errors_missing_prompt_pipeline_and_ttl_behave_as_required() {
+    let server = FakeQdrant::start(FakeState::default()).await;
+    let cache = connect(&server, [("one", vec![1.0, 0.0]), ("two", vec![0.0, 1.0])]).await;
+    let empty = SemanticCacheContext::default();
+    assert_eq!(
+        cache
+            .async_set_cache("key", value(json!({})), empty.clone())
+            .await,
+        Err(Error::MissingPrompt)
+    );
+    assert_eq!(
+        cache.async_get_cache("key", &empty).await,
+        Err(Error::MissingPrompt)
+    );
+    assert_eq!(
+        cache.async_get_cache("key", &context("unknown")).await,
+        Err(Error::Unavailable)
+    );
+    cache
+        .async_set_cache(
+            "ttl",
+            value(json!({"ttl": true})),
+            context("one").with_ttl(Some(Duration::from_secs(1))),
+        )
+        .await
+        .unwrap();
+    tokio::time::sleep(Duration::from_millis(1_100)).await;
+    assert!(
+        cache
+            .async_get_cache(
+                "ttl",
+                &context("one").with_ttl(Some(Duration::from_secs(1))),
+            )
+            .await
+            .unwrap()
+            .is_some()
+    );
+    cache
+        .async_set_cache_pipeline(
+            vec![
+                ("one".to_owned(), value(json!({"n": 1}))),
+                ("two".to_owned(), value(json!({"n": 2}))),
+            ],
+            context("one"),
+        )
+        .await
+        .unwrap();
+    assert!(
+        cache
+            .async_get_cache("one", &context("one"))
+            .await
+            .unwrap()
+            .is_some()
+    );
+    assert!(
+        cache
+            .async_get_cache("two", &context("one"))
+            .await
+            .unwrap()
+            .is_some()
+    );
+    assert_eq!(
+        server.state.lock().unwrap().upsert_waits,
+        vec![Some(true), Some(true), Some(true)]
+    );
+    assert_eq!(cache.get_ttl(&context("one")), None);
+    assert_eq!(
+        cache.test_connection().await,
+        Err(Error::UnsupportedOperation)
+    );
+    server.stop();
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn response_payloads_decode_and_invalid_entries_fail() {
+    let server = FakeQdrant::start(FakeState::default()).await;
+    let cache = connect(&server, [("hello", vec![1.0, 0.0])]).await;
+    for (key, response) in [
+        ("python", json!("{'timestamp': 1.0, 'response': {'a': 1}}")),
+        ("garbage", json!("not json")),
+        ("missing", json!("unused")),
+    ] {
+        let mut payload = serde_json::Map::new();
+        payload.insert("litellm_cache_key".to_owned(), json!(key));
+        if key != "missing" {
+            payload.insert("response".to_owned(), response);
+        }
+        server.insert_point(StoredPoint {
+            id: Some(PointId::from(key.len() as u64)),
+            vector: vec![1.0, 0.0],
+            payload: Payload::try_from(JsonValue::Object(payload))
+                .unwrap()
+                .into(),
+        });
+    }
+    assert_eq!(
+        cache
+            .async_get_cache("python", &context("hello"))
+            .await
+            .unwrap(),
+        Some(value(json!({"a": 1})))
+    );
+    assert_eq!(
+        cache.async_get_cache("garbage", &context("hello")).await,
+        Err(Error::InvalidEntry)
+    );
+    assert_eq!(
+        cache.async_get_cache("missing", &context("hello")).await,
+        Err(Error::InvalidEntry)
+    );
+    server.stop();
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn response_cache_facade_turns_invalid_entry_into_miss() {
+    let server = FakeQdrant::start(FakeState::default()).await;
+    let cache = Arc::new(connect(&server, [("hello", vec![1.0, 0.0])]).await);
+    let request = ResponseCacheRequest::<SemanticCacheContext>::new(CacheKeyInput {
+        preset: Some("key".to_owned()),
+        ..Default::default()
+    })
+    .with_context(context("hello"));
+    let response = json!({"answer": 42});
+    let facade = ResponseCache::new(cache.clone());
+    facade
+        .async_store(&request, response.clone(), Duration::from_secs(1))
+        .await
+        .unwrap();
+    assert_eq!(
+        facade
+            .async_lookup(&request, Duration::from_secs(1))
+            .await
+            .unwrap(),
+        Some(response)
+    );
+    {
+        let mut state = server.state.lock().unwrap();
+        state.points[0]
+            .payload
+            .insert("response".to_owned(), Value::from("not json"));
+    }
+    assert_eq!(
+        facade
+            .async_lookup(&request, Duration::from_secs(1))
+            .await
+            .unwrap(),
+        None
+    );
+    server.stop();
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn stopped_qdrant_server_maps_to_unavailable() {
+    let server = FakeQdrant::start(FakeState::default()).await;
+    let cache = connect(&server, [("hello", vec![1.0, 0.0])]).await;
+    server.stop();
+    tokio::time::sleep(Duration::from_millis(50)).await;
+    assert_eq!(
+        cache.async_get_cache("key", &context("hello")).await,
+        Err(Error::Unavailable)
+    );
+}

litellm-rust/crates/cache-qdrant-semantic/tests/support/mod.rs

@@ -0,0 +1,342 @@
+use std::{
+    collections::{HashMap, HashSet},
+    net::SocketAddr,
+    sync::{Arc, Mutex},
+};
+
+use qdrant_client::qdrant::collections_server::CollectionsServer;
+use qdrant_client::qdrant::{
+    self, CollectionExists, CollectionExistsRequest, CollectionExistsResponse,
+    CollectionOperationResponse, CreateCollection, CreateFieldIndexCollection, Filter, PointId,
+    PointsOperationResponse, ScoredPoint, SearchPoints, SearchResponse, Value, Vector, Vectors,
+    collections_server::Collections,
+    points_server::{Points, PointsServer},
+};
+use tokio::sync::oneshot;
+use tokio_stream::wrappers::TcpListenerStream;
+use tonic::{Request, Response, Status, transport::Server};
+
+#[derive(Clone, Debug)]
+pub struct StoredPoint {
+    pub id: Option<PointId>,
+    pub vector: Vec<f32>,
+    pub payload: HashMap<String, Value>,
+}
+
+#[derive(Default)]
+pub struct FakeState {
+    pub collections: HashSet<String>,
+    pub created_collections: Vec<CreateCollection>,
+    pub field_indexes: Vec<CreateFieldIndexCollection>,
+    pub points: Vec<StoredPoint>,
+    pub upsert_waits: Vec<Option<bool>>,
+    pub index_creations: usize,
+    pub fail_field_index: bool,
+}
+
+#[derive(Clone)]
+pub struct FakeQdrant {
+    pub state: Arc<Mutex<FakeState>>,
+    pub address: SocketAddr,
+    shutdown: Arc<Mutex<Option<oneshot::Sender<()>>>>,
+}
+
+impl FakeQdrant {
+    pub async fn start(state: FakeState) -> Self {
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let address = listener.local_addr().unwrap();
+        let state = Arc::new(Mutex::new(state));
+        let service = FakeService {
+            state: state.clone(),
+        };
+        let (shutdown_tx, shutdown_rx) = oneshot::channel();
+        tokio::spawn(async move {
+            Server::builder()
+                .add_service(CollectionsServer::new(service.clone()))
+                .add_service(PointsServer::new(service))
+                .serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
+                    let _ = shutdown_rx.await;
+                })
+                .await
+                .unwrap();
+        });
+        Self {
+            state,
+            address,
+            shutdown: Arc::new(Mutex::new(Some(shutdown_tx))),
+        }
+    }
+
+    pub fn url(&self) -> String {
+        format!("http://{}", self.address)
+    }
+
+    pub fn stop(&self) {
+        self.shutdown
+            .lock()
+            .unwrap()
+            .take()
+            .unwrap()
+            .send(())
+            .unwrap();
+    }
+
+    pub fn insert_point(&self, point: StoredPoint) {
+        self.state.lock().unwrap().points.push(point);
+    }
+}
+
+#[derive(Clone)]
+struct FakeService {
+    state: Arc<Mutex<FakeState>>,
+}
+
+macro_rules! unimplemented_collections {
+    ($($name:ident, $request:ty, $response:ty);* $(;)?) => {
+        $(
+            fn $name<'life0, 'async_trait>(
+                &'life0 self,
+                _: Request<$request>,
+            ) -> std::pin::Pin<
+                Box<
+                    dyn std::future::Future<
+                            Output = Result<Response<$response>, Status>,
+                        > + Send
+                        + 'async_trait,
+                >,
+            >
+            where
+                'life0: 'async_trait,
+                Self: 'async_trait,
+            {
+                Box::pin(async { Err(Status::unimplemented(stringify!($name))) })
+            }
+        )*
+    };
+}
+
+macro_rules! unimplemented_points {
+    ($($name:ident, $request:ty, $response:ty);* $(;)?) => {
+        $(
+            fn $name<'life0, 'async_trait>(
+                &'life0 self,
+                _: Request<$request>,
+            ) -> std::pin::Pin<
+                Box<
+                    dyn std::future::Future<
+                            Output = Result<Response<$response>, Status>,
+                        > + Send
+                        + 'async_trait,
+                >,
+            >
+            where
+                'life0: 'async_trait,
+                Self: 'async_trait,
+            {
+                Box::pin(async { Err(Status::unimplemented(stringify!($name))) })
+            }
+        )*
+    };
+}
+
+#[tonic::async_trait]
+impl Collections for FakeService {
+    async fn create(
+        &self,
+        request: Request<CreateCollection>,
+    ) -> Result<Response<CollectionOperationResponse>, Status> {
+        let request = request.into_inner();
+        let mut state = self.state.lock().unwrap();
+        state.collections.insert(request.collection_name.clone());
+        state.created_collections.push(request);
+        Ok(Response::new(CollectionOperationResponse {
+            result: true,
+            ..Default::default()
+        }))
+    }
+
+    async fn collection_exists(
+        &self,
+        request: Request<CollectionExistsRequest>,
+    ) -> Result<Response<CollectionExistsResponse>, Status> {
+        let exists = self
+            .state
+            .lock()
+            .unwrap()
+            .collections
+            .contains(&request.into_inner().collection_name);
+        Ok(Response::new(CollectionExistsResponse {
+            result: Some(CollectionExists { exists }),
+            ..Default::default()
+        }))
+    }
+
+    unimplemented_collections!(
+        get, qdrant::GetCollectionInfoRequest, qdrant::GetCollectionInfoResponse;
+        list, qdrant::ListCollectionsRequest, qdrant::ListCollectionsResponse;
+        update, qdrant::UpdateCollection, qdrant::CollectionOperationResponse;
+        delete, qdrant::DeleteCollection, qdrant::CollectionOperationResponse;
+        update_aliases, qdrant::ChangeAliases, qdrant::CollectionOperationResponse;
+        list_collection_aliases, qdrant::ListCollectionAliasesRequest, qdrant::ListAliasesResponse;
+        list_aliases, qdrant::ListAliasesRequest, qdrant::ListAliasesResponse;
+        collection_cluster_info, qdrant::CollectionClusterInfoRequest, qdrant::CollectionClusterInfoResponse;
+        update_collection_cluster_setup, qdrant::UpdateCollectionClusterSetupRequest, qdrant::UpdateCollectionClusterSetupResponse;
+        create_shard_key, qdrant::CreateShardKeyRequest, qdrant::CreateShardKeyResponse;
+        delete_shard_key, qdrant::DeleteShardKeyRequest, qdrant::DeleteShardKeyResponse;
+        list_shard_keys, qdrant::ListShardKeysRequest, qdrant::ListShardKeysResponse;
+    );
+}
+
+#[tonic::async_trait]
+impl Points for FakeService {
+    async fn create_field_index(
+        &self,
+        request: Request<CreateFieldIndexCollection>,
+    ) -> Result<Response<PointsOperationResponse>, Status> {
+        let mut state = self.state.lock().unwrap();
+        state.index_creations += 1;
+        state.field_indexes.push(request.into_inner());
+        if state.fail_field_index {
+            return Err(Status::internal("field index failure"));
+        }
+        Ok(Response::new(PointsOperationResponse::default()))
+    }
+
+    async fn upsert(
+        &self,
+        request: Request<qdrant::UpsertPoints>,
+    ) -> Result<Response<PointsOperationResponse>, Status> {
+        let request = request.into_inner();
+        let mut state = self.state.lock().unwrap();
+        state.upsert_waits.push(request.wait);
+        for point in request.points {
+            let stored = StoredPoint {
+                id: point.id.clone(),
+                vector: dense_vector(point.vectors)?,
+                payload: point.payload,
+            };
+            if let Some(existing) = state
+                .points
+                .iter_mut()
+                .find(|existing| existing.id == stored.id)
+            {
+                *existing = stored;
+            } else {
+                state.points.push(stored);
+            }
+        }
+        Ok(Response::new(PointsOperationResponse::default()))
+    }
+
+    async fn search(
+        &self,
+        request: Request<SearchPoints>,
+    ) -> Result<Response<SearchResponse>, Status> {
+        let request = request.into_inner();
+        let key_filter = keyword_filter(request.filter.as_ref());
+        let state = self.state.lock().unwrap();
+        let mut results = state
+            .points
+            .iter()
+            .filter(|point| {
+                key_filter.as_ref().is_none_or(|(field, expected)| {
+                    point
+                        .payload
+                        .get(field)
+                        .and_then(|value| {
+                            let value: serde_json::Value = value.clone().into();
+                            value
+                                .as_str()
+                                .map(str::to_owned)
+                                .or_else(|| value.as_i64().map(|value| value.to_string()))
+                        })
+                        .is_some_and(|value| value == *expected)
+                })
+            })
+            .map(|point| ScoredPoint {
+                id: point.id.clone(),
+                payload: point.payload.clone(),
+                score: cosine(&request.vector, &point.vector),
+                ..Default::default()
+            })
+            .collect::<Vec<_>>();
+        results.sort_by(|left, right| right.score.total_cmp(&left.score));
+        results.truncate(request.limit as usize);
+        Ok(Response::new(SearchResponse {
+            result: results,
+            ..Default::default()
+        }))
+    }
+
+    unimplemented_points!(
+        delete, qdrant::DeletePoints, qdrant::PointsOperationResponse;
+        get, qdrant::GetPoints, qdrant::GetResponse;
+        update_vectors, qdrant::UpdatePointVectors, qdrant::PointsOperationResponse;
+        delete_vectors, qdrant::DeletePointVectors, qdrant::PointsOperationResponse;
+        set_payload, qdrant::SetPayloadPoints, qdrant::PointsOperationResponse;
+        overwrite_payload, qdrant::SetPayloadPoints, qdrant::PointsOperationResponse;
+        delete_payload, qdrant::DeletePayloadPoints, qdrant::PointsOperationResponse;
+        clear_payload, qdrant::ClearPayloadPoints, qdrant::PointsOperationResponse;
+        delete_field_index, qdrant::DeleteFieldIndexCollection, qdrant::PointsOperationResponse;
+        create_vector_name, qdrant::CreateVectorNameRequest, qdrant::PointsOperationResponse;
+        delete_vector_name, qdrant::DeleteVectorNameRequest, qdrant::PointsOperationResponse;
+        search_batch, qdrant::SearchBatchPoints, qdrant::SearchBatchResponse;
+        search_groups, qdrant::SearchPointGroups, qdrant::SearchGroupsResponse;
+        scroll, qdrant::ScrollPoints, qdrant::ScrollResponse;
+        recommend, qdrant::RecommendPoints, qdrant::RecommendResponse;
+        recommend_batch, qdrant::RecommendBatchPoints, qdrant::RecommendBatchResponse;
+        recommend_groups, qdrant::RecommendPointGroups, qdrant::RecommendGroupsResponse;
+        discover, qdrant::DiscoverPoints, qdrant::DiscoverResponse;
+        discover_batch, qdrant::DiscoverBatchPoints, qdrant::DiscoverBatchResponse;
+        count, qdrant::CountPoints, qdrant::CountResponse;
+        update_batch, qdrant::UpdateBatchPoints, qdrant::UpdateBatchResponse;
+        query, qdrant::QueryPoints, qdrant::QueryResponse;
+        query_batch, qdrant::QueryBatchPoints, qdrant::QueryBatchResponse;
+        query_groups, qdrant::QueryPointGroups, qdrant::QueryGroupsResponse;
+        facet, qdrant::FacetCounts, qdrant::FacetResponse;
+        search_matrix_pairs, qdrant::SearchMatrixPoints, qdrant::SearchMatrixPairsResponse;
+        search_matrix_offsets, qdrant::SearchMatrixPoints, qdrant::SearchMatrixOffsetsResponse;
+    );
+}
+
+fn dense_vector(vectors: Option<Vectors>) -> Result<Vec<f32>, Status> {
+    let Some(Vectors {
+        vectors_options:
+            Some(qdrant::vectors::VectorsOptions::Vector(Vector {
+                vector: Some(qdrant::vector::Vector::Dense(qdrant::DenseVector { data })),
+                ..
+            })),
+    }) = vectors
+    else {
+        return Err(Status::invalid_argument("expected dense vector"));
+    };
+    Ok(data)
+}
+
+fn keyword_filter(filter: Option<&Filter>) -> Option<(String, String)> {
+    filter?
+        .must
+        .iter()
+        .find_map(|condition| match condition.condition_one_of.as_ref()? {
+            qdrant::condition::ConditionOneOf::Field(field) => {
+                let qdrant::r#match::MatchValue::Keyword(value) =
+                    field.r#match.as_ref()?.match_value.as_ref()?
+                else {
+                    return None;
+                };
+                Some((field.key.clone(), value.clone()))
+            }
+            _ => None,
+        })
+}
+
+fn cosine(left: &[f32], right: &[f32]) -> f32 {
+    let dot = left
+        .iter()
+        .zip(right)
+        .map(|(left, right)| left * right)
+        .sum::<f32>();
+    let left_norm = left.iter().map(|value| value * value).sum::<f32>().sqrt();
+    let right_norm = right.iter().map(|value| value * value).sum::<f32>().sqrt();
+    dot / (left_norm * right_norm)
+}

litellm-rust/crates/cache-redis-semantic/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "litellm-cache-redis-semantic"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-cache.workspace = true
+litellm-cache-redis.workspace = true
+litellm-cache-response.workspace = true
+redis = { version = "1.7.0", features = ["tls-rustls"] }
+r2d2 = "0.8.10"
+serde_json.workspace = true
+sha2.workspace = true
+tokio.workspace = true
+
+[dev-dependencies]
+redis-test = "1.0.4"
+serde_json.workspace = true
+tokio.workspace = true

litellm-rust/crates/cache-redis-semantic/src/cache.rs

@@ -0,0 +1,618 @@
+use std::{
+    future::Future,
+    sync::{Arc, OnceLock},
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+
+use litellm_cache::{
+    BaseCache, CacheCodec, CacheConnectionResult, CacheConnectionStatus, Error,
+    SemanticCacheContext,
+};
+use litellm_cache_redis::{
+    RedisTopology,
+    connection::{ConnectionRef, Connections},
+};
+use litellm_cache_response::{CacheEntry, ResponseCacheCodec};
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+
+use crate::prompt::prompt_from_context;
+
+const CACHE_KEY_FIELD: &str = "litellm_cache_key";
+const VECTOR_FIELD: &str = "prompt_vector";
+
+pub trait Embedder: Send + Sync + 'static {
+    fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, Error>;
+
+    fn async_embed(
+        &self,
+        prompt: &str,
+        metadata: Option<&Value>,
+    ) -> impl Future<Output = Result<Vec<f32>, Error>> + Send;
+}
+
+#[derive(Clone, Debug)]
+pub struct RedisSemanticConfig {
+    pub index_name: String,
+    pub similarity_threshold: f32,
+}
+
+struct Inner {
+    index_name: String,
+    distance_threshold: f64,
+    resolved_index: OnceLock<String>,
+    codec: ResponseCacheCodec,
+    clock: fn() -> f64,
+}
+
+impl Inner {
+    fn new(config: RedisSemanticConfig) -> Self {
+        Self {
+            index_name: config.index_name,
+            distance_threshold: 1.0 - f64::from(config.similarity_threshold),
+            resolved_index: OnceLock::new(),
+            codec: ResponseCacheCodec,
+            clock: timestamp,
+        }
+    }
+
+    fn ensure_index(
+        &self,
+        connection: &mut ConnectionRef<'_>,
+        dims: usize,
+    ) -> Result<String, Error> {
+        if let Some(name) = self.resolved_index.get() {
+            return Ok(name.clone());
+        }
+        let name = match index_compatible(connection, &self.index_name, dims)? {
+            Some(true) => self.index_name.clone(),
+            Some(false) => self.isolated_index(connection, dims)?,
+            None => match create_index(connection, &self.index_name, dims) {
+                Ok(()) => self.index_name.clone(),
+                Err(_) => match index_compatible(connection, &self.index_name, dims)? {
+                    Some(true) => self.index_name.clone(),
+                    Some(false) => self.isolated_index(connection, dims)?,
+                    None => return Err(Error::Unavailable),
+                },
+            },
+        };
+        let _ = self.resolved_index.set(name.clone());
+        Ok(name)
+    }
+
+    fn isolated_index(
+        &self,
+        connection: &mut ConnectionRef<'_>,
+        dims: usize,
+    ) -> Result<String, Error> {
+        let name = format!("{}_isolated", self.index_name);
+        match index_compatible(connection, &name, dims)? {
+            Some(true) => Ok(name),
+            Some(false) => {
+                redis::cmd("FT.DROPINDEX")
+                    .arg(&name)
+                    .query::<()>(connection)
+                    .map_err(|_| Error::Unavailable)?;
+                create_index(connection, &name, dims)?;
+                Ok(name)
+            }
+            None => {
+                create_index(connection, &name, dims)?;
+                Ok(name)
+            }
+        }
+    }
+
+    fn store(
+        &self,
+        connection: &mut ConnectionRef<'_>,
+        tag: &str,
+        value: &CacheEntry,
+        prompt: &str,
+        vector: &[f32],
+        ttl: Option<Duration>,
+    ) -> Result<(), Error> {
+        let index = self.ensure_index(connection, vector.len())?;
+        let entry_id = entry_id(prompt, tag);
+        let hash_key = format!("{index}:{entry_id}");
+        let response = self.codec.encode(value)?;
+        redis::cmd("HSET")
+            .arg(&hash_key)
+            .arg("entry_id")
+            .arg(&entry_id)
+            .arg("prompt")
+            .arg(prompt)
+            .arg("response")
+            .arg(response)
+            .arg(VECTOR_FIELD)
+            .arg(vector_buffer(vector))
+            .arg("inserted_at")
+            .arg(format!("{}", (self.clock)()))
+            .arg("updated_at")
+            .arg(format!("{}", (self.clock)()))
+            .arg(CACHE_KEY_FIELD)
+            .arg(tag)
+            .query::<()>(connection)
+            .map_err(|_| Error::Unavailable)?;
+        if let Some(ttl) = ttl {
+            redis::cmd("EXPIRE")
+                .arg(&hash_key)
+                .arg(ttl_seconds(ttl))
+                .query::<()>(connection)
+                .map_err(|_| Error::Unavailable)?;
+        }
+        Ok(())
+    }
+
+    fn lookup(
+        &self,
+        connection: &mut ConnectionRef<'_>,
+        tag: &str,
+        vector: &[f32],
+    ) -> Result<Option<CacheEntry>, Error> {
+        let index = self.ensure_index(connection, vector.len())?;
+        let query = format!(
+            "(@{CACHE_KEY_FIELD}:{{{}}})=>[KNN 1 @{VECTOR_FIELD} $vector AS vector_distance]",
+            escape_tag(tag)
+        );
+        let result = redis::cmd("FT.SEARCH")
+            .arg(&index)
+            .arg(query)
+            .arg("RETURN")
+            .arg(8)
+            .arg("entry_id")
+            .arg("prompt")
+            .arg("response")
+            .arg("inserted_at")
+            .arg("updated_at")
+            .arg("metadata")
+            .arg(CACHE_KEY_FIELD)
+            .arg("vector_distance")
+            .arg("SORTBY")
+            .arg("vector_distance")
+            .arg("ASC")
+            .arg("DIALECT")
+            .arg(2)
+            .arg("LIMIT")
+            .arg(0)
+            .arg(1)
+            .arg("PARAMS")
+            .arg(2)
+            .arg("vector")
+            .arg(vector_buffer(vector))
+            .query::<redis::Value>(connection)
+            .map_err(|_| Error::Unavailable)?;
+        let Some(fields) = first_document(&result) else {
+            return Ok(None);
+        };
+        if string_field(fields, CACHE_KEY_FIELD).as_deref() != Some(tag) {
+            return Ok(None);
+        }
+        if number_field(fields, "vector_distance")
+            .is_none_or(|distance| distance > self.distance_threshold)
+        {
+            return Ok(None);
+        }
+        let Some(response) = bytes_field(fields, "response") else {
+            return Ok(None);
+        };
+        self.codec.decode(&response).map(Some)
+    }
+}
+
+pub struct RedisSemanticCache<E: Embedder, C = redis::Connection> {
+    connections: Arc<Connections<C>>,
+    embedder: E,
+    inner: Arc<Inner>,
+}
+
+impl<E: Embedder> RedisSemanticCache<E> {
+    pub fn new(url: &str, embedder: E, config: RedisSemanticConfig) -> Result<Self, Error> {
+        Ok(Self {
+            connections: Arc::new(Connections::open(url, &RedisTopology::Standalone)?),
+            embedder,
+            inner: Arc::new(Inner::new(config)),
+        })
+    }
+}
+
+impl<E: Embedder, C: redis::ConnectionLike + Send + 'static> RedisSemanticCache<E, C> {
+    pub fn with_connection(connection: C, embedder: E, config: RedisSemanticConfig) -> Self {
+        Self {
+            connections: Arc::new(Connections::fixed(connection)),
+            embedder,
+            inner: Arc::new(Inner::new(config)),
+        }
+    }
+
+    pub fn with_clock(self, clock: fn() -> f64) -> Self {
+        Self {
+            inner: Arc::new(Inner {
+                index_name: self.inner.index_name.clone(),
+                distance_threshold: self.inner.distance_threshold,
+                resolved_index: OnceLock::new(),
+                codec: self.inner.codec,
+                clock,
+            }),
+            ..self
+        }
+    }
+
+    pub fn embedder(&self) -> &E {
+        &self.embedder
+    }
+
+    pub fn index_name(&self) -> &str {
+        &self.inner.index_name
+    }
+
+    pub fn similarity_threshold(&self) -> f32 {
+        (1.0 - self.inner.distance_threshold) as f32
+    }
+
+    fn tag<'a>(key: &'a str, context: &'a SemanticCacheContext) -> &'a str {
+        context.scope.as_deref().unwrap_or(key)
+    }
+}
+
+impl<E: Embedder, C: redis::ConnectionLike + Send + 'static> BaseCache
+    for RedisSemanticCache<E, C>
+{
+    type Value = CacheEntry;
+    type Context = SemanticCacheContext;
+
+    fn get_ttl(&self, context: &Self::Context) -> Option<Duration> {
+        context.ttl
+    }
+
+    fn set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: &Self::Context,
+    ) -> Result<(), Error> {
+        let Some(prompt) = prompt_from_context(context) else {
+            return Ok(());
+        };
+        let vector = self.embedder.embed(&prompt, context.metadata.as_ref())?;
+        let tag = Self::tag(key, context).to_string();
+        self.connections.execute(|connection| {
+            self.inner
+                .store(connection, &tag, &value, &prompt, &vector, context.ttl)
+        })
+    }
+
+    fn get_cache(&self, key: &str, context: &Self::Context) -> Result<Option<Self::Value>, Error> {
+        let Some(prompt) = prompt_from_context(context) else {
+            return Ok(None);
+        };
+        let vector = self.embedder.embed(&prompt, context.metadata.as_ref())?;
+        let tag = Self::tag(key, context).to_string();
+        self.connections
+            .execute(|connection| self.inner.lookup(connection, &tag, &vector))
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: Self::Context,
+    ) -> Result<(), Error> {
+        let Some(prompt) = prompt_from_context(&context) else {
+            return Ok(());
+        };
+        let vector = self
+            .embedder
+            .async_embed(&prompt, context.metadata.as_ref())
+            .await?;
+        let tag = Self::tag(key, &context).to_string();
+        let inner = Arc::clone(&self.inner);
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            inner.store(connection, &tag, &value, &prompt, &vector, context.ttl)
+        })
+        .await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        context: &Self::Context,
+    ) -> Result<Option<Self::Value>, Error> {
+        let Some(prompt) = prompt_from_context(context) else {
+            return Ok(None);
+        };
+        let vector = self
+            .embedder
+            .async_embed(&prompt, context.metadata.as_ref())
+            .await?;
+        let tag = Self::tag(key, context).to_string();
+        let inner = Arc::clone(&self.inner);
+        Connections::run_blocking(Arc::clone(&self.connections), move |connection| {
+            inner.lookup(connection, &tag, &vector)
+        })
+        .await
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        match Connections::run_blocking(Arc::clone(&self.connections), |connection| {
+            Ok(match redis::cmd("PING").query::<String>(connection) {
+                Ok(_) => CacheConnectionResult {
+                    status: CacheConnectionStatus::Success,
+                    message: "Redis cache connection test successful".into(),
+                    error: None,
+                },
+                Err(error) => CacheConnectionResult {
+                    status: CacheConnectionStatus::Failed,
+                    message: format!("Redis connection failed: {error}"),
+                    error: Some(error.to_string()),
+                },
+            })
+        })
+        .await
+        {
+            Ok(result) => Ok(result),
+            Err(error) => Ok(CacheConnectionResult {
+                status: CacheConnectionStatus::Failed,
+                message: format!("Redis connection failed: {error}"),
+                error: Some(error.to_string()),
+            }),
+        }
+    }
+}
+
+fn timestamp() -> f64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|duration| duration.as_secs_f64())
+        .unwrap_or_default()
+}
+
+fn entry_id(prompt: &str, tag: &str) -> String {
+    let mut digest = Sha256::new();
+    digest.update(prompt.as_bytes());
+    digest.update(CACHE_KEY_FIELD.as_bytes());
+    digest.update(tag.as_bytes());
+    format!("{:x}", digest.finalize())
+}
+
+fn vector_buffer(vector: &[f32]) -> Vec<u8> {
+    vector
+        .iter()
+        .flat_map(|component| component.to_le_bytes())
+        .collect()
+}
+
+fn escape_tag(value: &str) -> String {
+    value
+        .chars()
+        .flat_map(|ch| {
+            if matches!(
+                ch,
+                ',' | '.'
+                    | '<'
+                    | '>'
+                    | '{'
+                    | '}'
+                    | '['
+                    | ']'
+                    | '\\'
+                    | '"'
+                    | '\''
+                    | ':'
+                    | ';'
+                    | '!'
+                    | '@'
+                    | '#'
+                    | '$'
+                    | '%'
+                    | '^'
+                    | '&'
+                    | '*'
+                    | '('
+                    | ')'
+                    | '-'
+                    | '+'
+                    | '='
+                    | '~'
+                    | '|'
+                    | '/'
+                    | ' '
+                    | '?'
+            ) {
+                vec!['\\', ch]
+            } else {
+                vec![ch]
+            }
+        })
+        .collect()
+}
+
+fn create_index(connection: &mut ConnectionRef<'_>, name: &str, dims: usize) -> Result<(), Error> {
+    redis::cmd("FT.CREATE")
+        .arg(name)
+        .arg("ON")
+        .arg("HASH")
+        .arg("PREFIX")
+        .arg(1)
+        .arg(name)
+        .arg("SCORE")
+        .arg(1.0)
+        .arg("SCHEMA")
+        .arg("prompt")
+        .arg("TEXT")
+        .arg("WEIGHT")
+        .arg(1)
+        .arg("response")
+        .arg("TEXT")
+        .arg("WEIGHT")
+        .arg(1)
+        .arg("inserted_at")
+        .arg("NUMERIC")
+        .arg("updated_at")
+        .arg("NUMERIC")
+        .arg(VECTOR_FIELD)
+        .arg("VECTOR")
+        .arg("FLAT")
+        .arg(6)
+        .arg("TYPE")
+        .arg("FLOAT32")
+        .arg("DIM")
+        .arg(dims)
+        .arg("DISTANCE_METRIC")
+        .arg("COSINE")
+        .arg(CACHE_KEY_FIELD)
+        .arg("TAG")
+        .arg("SEPARATOR")
+        .arg(",")
+        .query::<()>(connection)
+        .map_err(|_| Error::Unavailable)
+}
+
+fn index_compatible(
+    connection: &mut ConnectionRef<'_>,
+    name: &str,
+    dims: usize,
+) -> Result<Option<bool>, Error> {
+    let info = match redis::cmd("FT.INFO")
+        .arg(name)
+        .query::<redis::Value>(connection)
+    {
+        Ok(info) => info,
+        Err(error) if unknown_index(&error) => return Ok(None),
+        Err(_) => return Err(Error::Unavailable),
+    };
+    Ok(Some(schema_compatible(&info, dims)))
+}
+
+fn unknown_index(error: &redis::RedisError) -> bool {
+    let message = error.to_string().to_lowercase();
+    message.contains("unknown") && message.contains("index")
+}
+
+fn schema_compatible(info: &redis::Value, dims: usize) -> bool {
+    let redis::Value::Array(entries) = info else {
+        return false;
+    };
+    let attributes = entries
+        .as_chunks::<2>()
+        .0
+        .iter()
+        .find(|pair| string_value(&pair[0]).as_deref() == Some("attributes"))
+        .map(|pair| &pair[1]);
+    let Some(redis::Value::Array(attributes)) = attributes else {
+        return false;
+    };
+    let fields = attributes
+        .iter()
+        .map(|attribute| {
+            let redis::Value::Array(attribute) = attribute else {
+                return (None, None, None, None, None);
+            };
+            let mut name = None;
+            let mut field_type = None;
+            let mut dim = None;
+            let mut data_type = None;
+            let mut distance_metric = None;
+            for pair in attribute.as_chunks::<2>().0 {
+                match string_value(&pair[0]).as_deref() {
+                    Some("identifier") => name = string_value(&pair[1]),
+                    Some("type") => field_type = string_value(&pair[1]),
+                    Some("dim") => dim = number_value(&pair[1]),
+                    Some("data_type") => data_type = string_value(&pair[1]),
+                    Some("distance_metric") => distance_metric = string_value(&pair[1]),
+                    _ => {}
+                }
+            }
+            (name, field_type, dim, data_type, distance_metric)
+        })
+        .collect::<Vec<_>>();
+    let has_field = |name: &str, field_type: &str| {
+        fields
+            .iter()
+            .any(|(n, t, ..)| n.as_deref() == Some(name) && t.as_deref() == Some(field_type))
+    };
+    has_field("prompt", "TEXT")
+        && has_field("response", "TEXT")
+        && has_field("inserted_at", "NUMERIC")
+        && has_field("updated_at", "NUMERIC")
+        && has_field(CACHE_KEY_FIELD, "TAG")
+        && fields.iter().any(|(n, t, d, data, metric)| {
+            n.as_deref() == Some(VECTOR_FIELD)
+                && t.as_deref() == Some("VECTOR")
+                && *d == Some(dims as f64)
+                && data
+                    .as_deref()
+                    .is_some_and(|data| data.eq_ignore_ascii_case("float32"))
+                && metric
+                    .as_deref()
+                    .is_some_and(|metric| metric.eq_ignore_ascii_case("cosine"))
+        })
+}
+
+fn string_value(value: &redis::Value) -> Option<String> {
+    match value {
+        redis::Value::BulkString(bytes) => String::from_utf8(bytes.clone()).ok(),
+        redis::Value::SimpleString(text) => Some(text.clone()),
+        redis::Value::VerbatimString { text, .. } => Some(text.clone()),
+        _ => None,
+    }
+}
+
+fn number_value(value: &redis::Value) -> Option<f64> {
+    match value {
+        redis::Value::Int(number) => Some(*number as f64),
+        redis::Value::Double(number) => Some(*number),
+        _ => string_value(value).and_then(|text| text.parse().ok()),
+    }
+}
+
+fn first_document(result: &redis::Value) -> Option<&[redis::Value]> {
+    let redis::Value::Array(items) = result else {
+        return None;
+    };
+    let [count, _document_id, fields, ..] = items.as_slice() else {
+        return None;
+    };
+    if !matches!(count, redis::Value::Int(count) if *count > 0) {
+        return None;
+    }
+    match fields {
+        redis::Value::Array(fields) => Some(fields.as_slice()),
+        _ => None,
+    }
+}
+
+fn field_value<'a>(fields: &'a [redis::Value], name: &str) -> Option<&'a redis::Value> {
+    fields
+        .as_chunks::<2>()
+        .0
+        .iter()
+        .find(|pair| string_value(&pair[0]).as_deref() == Some(name))
+        .map(|pair| &pair[1])
+}
+
+fn string_field(fields: &[redis::Value], name: &str) -> Option<String> {
+    field_value(fields, name).and_then(string_value)
+}
+
+fn number_field(fields: &[redis::Value], name: &str) -> Option<f64> {
+    field_value(fields, name).and_then(number_value)
+}
+
+fn bytes_field(fields: &[redis::Value], name: &str) -> Option<Vec<u8>> {
+    match field_value(fields, name)? {
+        redis::Value::BulkString(bytes) => Some(bytes.clone()),
+        redis::Value::SimpleString(text) => Some(text.clone().into_bytes()),
+        _ => None,
+    }
+}
+
+fn ttl_seconds(ttl: Duration) -> u64 {
+    ttl.as_secs()
+        .saturating_add(u64::from(ttl.subsec_nanos() > 0))
+        .max(1)
+}

litellm-rust/crates/cache-redis-semantic/src/lib.rs

@@ -0,0 +1,5 @@
+mod cache;
+mod prompt;
+
+pub use cache::{Embedder, RedisSemanticCache, RedisSemanticConfig};
+pub use prompt::prompt_from_context;

litellm-rust/crates/cache-redis-semantic/src/prompt.rs

@@ -0,0 +1,97 @@
+use litellm_cache::SemanticCacheContext;
+use serde_json::Value;
+
+pub fn prompt_from_context(context: &SemanticCacheContext) -> Option<String> {
+    if let Some(messages) = context.messages.as_ref().and_then(Value::as_array)
+        && !messages.is_empty()
+    {
+        return Some(messages_text(messages));
+    }
+    let input = context.input.as_ref()?;
+    let mut parts = Vec::new();
+    collect_input_text(input, &mut parts);
+    let prompt = parts.join("\n").trim().to_string();
+    (!prompt.is_empty()).then_some(prompt)
+}
+
+fn messages_text(messages: &[Value]) -> String {
+    let mut text = String::new();
+    for message in messages {
+        let Some(message) = message.as_object() else {
+            continue;
+        };
+        match message.get("content") {
+            Some(Value::String(content)) => text.push_str(content),
+            Some(Value::Array(parts)) => {
+                for part in parts {
+                    if let Some(text_content) = part.get("text").and_then(Value::as_str) {
+                        text.push_str(text_content);
+                    }
+                }
+            }
+            _ => {}
+        }
+        text.push_str(&search_results_text(message.get("search_results")));
+    }
+    text
+}
+
+fn search_results_text(search_results: Option<&Value>) -> String {
+    let Some(Value::Array(results)) = search_results else {
+        return String::new();
+    };
+    let mut text = String::new();
+    for result in results {
+        let Some(result) = result.as_object() else {
+            continue;
+        };
+        for key in ["source", "title"] {
+            if let Some(value) = result.get(key).and_then(Value::as_str) {
+                text.push_str(value);
+            }
+        }
+        if let Some(Value::Array(content)) = result.get("content") {
+            for block in content {
+                if let Some(value) = block.get("text").and_then(Value::as_str) {
+                    text.push_str(value);
+                }
+            }
+        }
+        if let Some(citations) = result.get("citations") {
+            text.push_str(&citations.to_string());
+        }
+    }
+    text
+}
+
+fn collect_input_text(value: &Value, parts: &mut Vec<String>) {
+    match value {
+        Value::String(text) => {
+            let trimmed = text.trim();
+            if !trimmed.is_empty() {
+                parts.push(trimmed.to_string());
+            }
+        }
+        Value::Array(items) => {
+            for item in items {
+                collect_input_text(item, parts);
+            }
+        }
+        Value::Object(map) => {
+            if let Some(content) = map.get("content").filter(|content| !content.is_null()) {
+                collect_input_text(content, parts);
+                return;
+            }
+            for key in ["text", "output", "input_text", "output_text"] {
+                if let Some(Value::String(text)) = map.get(key) {
+                    let trimmed = text.trim();
+                    if !trimmed.is_empty() {
+                        parts.push(trimmed.to_string());
+                        return;
+                    }
+                }
+            }
+        }
+        _ => {}
+    }
+}