Merge remote-tracking branch 'origin/main' into litellm_config_read_source

.circleci/config.yml

@@ -6,6 +6,9 @@ parameters:
   migration_candidate_image:
     type: string
     default: ""
+  migration_baseline_image:
+    type: string
+    default: "ghcr.io/berriai/litellm-database:v1.102.0"
   migration_source_sha:
     type: string
     default: ""
@@ -2946,7 +2949,10 @@ jobs:
     parameters:
       suite:
         type: enum
-        enum: [startup, recovery, legacy]
+        enum: [startup, recovery, legacy, upgrade, shaped]
+      baseline:
+        type: boolean
+        default: false
     machine:
       image: ubuntu-2204:2024.04.1
     resource_class: large
@@ -2954,6 +2960,7 @@ jobs:
     environment:
       LITELLM_MIGRATION_TESTS: "1"
       LITELLM_MIGRATION_TEST_IMAGE: litellm-docker-database:ci
+      LITELLM_MIGRATION_BASELINE_IMAGE: << pipeline.parameters.migration_baseline_image >>
       MIGRATION_TEST_ADMIN_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres
       MIGRATION_TEST_CONTAINER_ADMIN_URL: postgresql://postgres:postgres@host.docker.internal:5432/postgres
       MIGRATION_TEST_OUTPUT: /tmp/migration-results
@@ -2981,6 +2988,16 @@ jobs:
       - wait_for_service:
           url: tcp://localhost:5432
           timeout: "60"
+      - when:
+          condition: << parameters.baseline >>
+          steps:
+            - run:
+                name: Pull the baseline release the upgrade starts from
+                environment:
+                  BASELINE_IMAGE: << pipeline.parameters.migration_baseline_image >>
+                command: |
+                  [[ "$BASELINE_IMAGE" =~ ^ghcr.io/berriai/[a-z0-9._/-]+(@sha256:[0-9a-f]{64}|:v[0-9][0-9a-z.-]*)$ ]] || exit 1
+                  docker pull "$BASELINE_IMAGE"
       - run:
           name: Run migration startup regressions
           environment:
@@ -3033,28 +3050,29 @@ jobs:
       - run:
           name: Run Docker container with bad DATABASE_URL
           command: |
+            set +e
             docker run --name my-app \
               -p 4000:4000 \
               -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
               -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
               -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
               myapp:latest \
-              --port 4000 > docker_output.log 2>&1 || true
+              --port 4000 > docker_output.log 2>&1
+            echo "$?" > docker_exit_code
+            set -e
       - run:
           name: Display Docker logs
           command: cat docker_output.log
       - run:
-          name: Check for expected error
+          name: Proxy must refuse to serve on an unreachable database
           command: |
-            if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \
-               (grep -q "Database setup failed after multiple retries" docker_output.log || \
-                grep -q "ERROR:    Application startup failed. Exiting." docker_output.log); then
-              echo "Expected error found. Test passed."
-            else
-              echo "Expected error not found. Test failed."
-              cat docker_output.log
-              exit 1
-            fi
+            fail() { echo "FAILED: $1"; cat docker_output.log; exit 1; }
+            exit_code="$(cat docker_exit_code)"
+            [ "$exit_code" -ne 0 ] || fail "proxy exited 0 with an unreachable database"
+            grep -q "P1001" docker_output.log || fail "log does not name the unreachable database server"
+            ! grep -q "Application startup complete" docker_output.log || fail "proxy reached serving state"
+            ! docker exec my-app true 2>/dev/null || fail "container is still running"
+            echo "Proxy refused to serve (exit $exit_code) and never reached startup. Test passed."
 
   provider_replay_harness:
     docker:
@@ -3188,6 +3206,16 @@ workflows:
           name: migration-legacy-and-pooling
           suite: legacy
           requires: [build_docker_database_image]
+      - migration_startup_tests:
+          name: migration-upgrade
+          suite: upgrade
+          baseline: true
+          requires: [build_docker_database_image]
+      - migration_startup_tests:
+          name: migration-upgrade-shaped
+          suite: shaped
+          baseline: true
+          requires: [build_docker_database_image]
   migration_startup_scheduled:
     triggers:
       - schedule:

.circleci/scripts/run_integration.sh

@@ -121,6 +121,10 @@ start_proxy() {
       "LITELLM_MODEL_COST_MAP_URL=$INTEGRATION_UPSTREAM_URL/_cost_map"
       "MODEL_COST_MAP_MIN_MODEL_COUNT=1"
       "MODEL_COST_MAP_MAX_SHRINK_RATIO=0"
+      "GEMINI_API_BASE=$INTEGRATION_UPSTREAM_URL"
+      "ANTHROPIC_API_BASE=$INTEGRATION_UPSTREAM_URL"
+      "GEMINI_API_KEY=sk-scripted-provider"
+      "ANTHROPIC_API_KEY=sk-scripted-provider"
     )
   else
     cost_map_env=("LITELLM_LOCAL_MODEL_COST_MAP=True")

.circleci/scripts/run_migration_tests.py

@@ -13,6 +13,8 @@ SUITES: Final = {
     "startup": (("test_startup.py",), 12),
     "recovery": (("test_recovery.py",), 15),
     "legacy": (("test_legacy.py", "test_pooling.py"), 11),
+    "upgrade": (("test_upgrade.py", "test_rolling_upgrade.py"), 5),
+    "shaped": (("test_shaped_database.py",), 1),
 }
 
 
@@ -93,6 +95,7 @@ def main() -> int:
             {
                 **metadata,
                 "suite": suite,
+                "baseline_image": os.environ.get("LITELLM_MIGRATION_BASELINE_IMAGE", ""),
                 "expected_cases": expected,
                 "passed": passed,
                 "pytest_exit_code": result.returncode,

.github/scripts/verify_linux_native_wheel.py

@@ -205,7 +205,7 @@ def main(
     native_module: Final = load_native_module(native_path)
     native_module_loads: Final = native_module is not None
     panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
-    native_size_limit: Final = 25_000_000
+    native_size_limit: Final = 40_000_000
     native_size_within_limit: Final = native_member.file_size <= native_size_limit
     validations: Final = (
         (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
@@ -222,7 +222,7 @@ def main(
         ("Python extension entry point is present", extension_entry_point_present),
         ("Native module loads", native_module_loads),
         ("Production module omits the panic test hook", panic_test_hook_absent),
-        ("Native extension does not exceed 25 MB", native_size_within_limit),
+        (f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit),
         ("Wheel contents are valid", not unexpected_members),
     )
 
@@ -267,7 +267,8 @@ def main(
             ),
             (
                 not native_size_within_limit,
-                f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB",
+                f"native extension exceeds {native_size_limit / 1_000_000:.0f} MB: "
+                f"{native_member.file_size / 1_000_000:.2f} MB",
             ),
             (bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"),
         )

.github/workflows/test-linting.yml

@@ -130,6 +130,10 @@ jobs:
           echo "File content around line 43:"
           head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
 
+      - name: Check MCP operation boundary
+        if: steps.changes.outputs.decision != 'skip'
+        run: uv run --no-sync python scripts/check_mcp_operation_boundary.py
+
       - name: Run Ruff linting
         if: steps.changes.outputs.decision != 'skip'
         run: |

.github/workflows/test-rust.yml

@@ -130,7 +130,7 @@ jobs:
       - name: Test secret manager feature combinations
         run: |
           cargo test -p litellm-auth-gcp --locked --no-default-features
-          for features in '' aws google aws,google; do
+          for features in '' aws google azure cyberark aws,google aws,azure google,azure aws,google,azure aws,google,cyberark aws,google,azure,cyberark; do
             cargo test -p litellm-secrets --locked --no-default-features --features "$features"
           done
 

Makefile

@@ -164,6 +164,7 @@ lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
 
 # Linting targets
 lint-ruff: $(LINT_DEP_INSTALL)
+	$(UV_RUN) python scripts/check_mcp_operation_boundary.py
 	cd litellm && $(UV_RUN) ruff check . && cd ..
 	$(UV_RUN) ruff check --config ruff-tests.toml tests
 

README.md

@@ -307,6 +307,7 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse
 | [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ |  |  | ✅ |  |  |  |  |
 | [DeepInfra (`deepinfra`)](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |
 | [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ |  |  |  |
 | [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ |  | ✅ |  |  |  |  |  |

docker/README.md

@@ -2,6 +2,17 @@
 
 This guide provides instructions for building and running the LiteLLM application using Docker and Docker Compose.
 
+> **Just want to run LiteLLM?** This guide builds from source. To run the published
+> image instead, use `docker-compose.quickstart.yml` in this directory — the
+> two-service stack (gateway + Postgres) that the
+> [Docker quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start) documents:
+>
+> ```bash
+> curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml
+> printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env
+> docker compose -f docker-compose.quickstart.yml up -d
+> ```
+
 ## Prerequisites
 
 - Docker

docker/docker-compose.quickstart.yml

@@ -0,0 +1,41 @@
+# LiteLLM quickstart stack: the gateway plus a Postgres database that stores
+# models, virtual keys, and spend logs. Used by
+# https://docs.litellm.ai/docs/proxy/docker_quick_start
+#
+#   curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml
+#   printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env
+#   docker compose -f docker-compose.quickstart.yml up -d
+#
+# Compose reads .env from this directory. Keep it: regenerating LITELLM_SALT_KEY
+# makes credentials already stored in the database unreadable. For anything
+# beyond local evaluation, pin the image to a specific release tag.
+services:
+  litellm:
+    image: docker.litellm.ai/berriai/litellm:main-stable
+    ports:
+      - "4000:4000"
+    environment:
+      LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?set it in .env - see the header of this file}
+      LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?set it in .env - see the header of this file}
+      DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
+      STORE_MODEL_IN_DB: "True"
+    depends_on:
+      db:
+        condition: service_healthy
+
+  db:
+    image: postgres:16
+    environment:
+      POSTGRES_USER: litellm
+      POSTGRES_PASSWORD: litellm
+      POSTGRES_DB: litellm
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U litellm"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+
+volumes:
+  postgres_data:

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -2,10 +2,11 @@
 Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if the cost has been tracked.
 """
 
+from collections.abc import Sequence
 from dataclasses import replace as dataclasses_replace
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, List, Literal, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tuple, cast
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -18,8 +19,8 @@ if TYPE_CHECKING:
     from prisma import models as prisma_models
 
     from litellm.integrations.prometheus import PrometheusLogger
-    from litellm.proxy._types import LiteLLM_ManagedObjectTable
     from litellm.proxy.utils import PrismaClient, ProxyLogging
+    from litellm.repositories.prisma_protocols import TableActions
     from litellm.router import Router
     from litellm.types.router import Deployment
     from litellm.types.utils import LiteLLMBatch
@@ -41,6 +42,42 @@ TERMINAL_MANAGED_OBJECT_STATUSES: Final[Tuple[str, ...]] = (
 )
 
 
+class _ManagedObjectRow(Protocol):
+    @property
+    def id(self) -> str: ...
+
+    @property
+    def unified_object_id(self) -> str: ...
+
+    @property
+    def created_by(self) -> str | None: ...
+
+    @property
+    def file_object(self) -> object: ...
+
+
+def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]":
+    table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable
+    return table
+
+
+def _user_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_UserTable]":
+    table: Final[TableActions[prisma_models.LiteLLM_UserTable]] = prisma_client.db.litellm_usertable
+    return table
+
+
+def _token_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_VerificationToken]":
+    table: Final[TableActions[prisma_models.LiteLLM_VerificationToken]] = (
+        prisma_client.db.litellm_verificationtoken
+    )
+    return table
+
+
+def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_TeamTable]":
+    table: Final[TableActions[prisma_models.LiteLLM_TeamTable]] = prisma_client.db.litellm_teamtable
+    return table
+
+
 class CheckBatchCost:
     def __init__(
         self,
@@ -73,7 +110,7 @@ class CheckBatchCost:
         inline for a batch the first poll cycle then accounts again.
         """
         try:
-            await self.prisma_client.db.litellm_managedobjecttable.find_first(
+            await _managed_object_table(self.prisma_client).find_first(
                 where={"file_purpose": "batch", "batch_processed": False}
             )
         except Exception as probe_err:
@@ -97,10 +134,8 @@ class CheckBatchCost:
         if not user_id:
             return {}
         try:
-            user_row: prisma_models.LiteLLM_UserTable | None = (
-                await self.prisma_client.db.litellm_usertable.find_unique(
-                    where={"user_id": user_id}
-                )
+            user_row: prisma_models.LiteLLM_UserTable | None = await _user_table(self.prisma_client).find_unique(
+                where={"user_id": user_id}
             )
             if user_row is None:
                 return {}
@@ -117,11 +152,9 @@ class CheckBatchCost:
         if not api_key:
             return None
         try:
-            key_row: prisma_models.LiteLLM_VerificationToken | None = (
-                await self.prisma_client.db.litellm_verificationtoken.find_unique(
-                    where={"token": api_key}
-                )
-            )
+            key_row: prisma_models.LiteLLM_VerificationToken | None = await _token_table(
+                self.prisma_client
+            ).find_unique(where={"token": api_key})
             return getattr(key_row, "key_alias", None) if key_row is not None else None
         except Exception as e:
             verbose_proxy_logger.error(f"CheckBatchCost: could not look up key alias for batch {batch_id}: {e}")
@@ -132,17 +165,15 @@ class CheckBatchCost:
         if not team_id:
             return None
         try:
-            team_row: prisma_models.LiteLLM_TeamTable | None = (
-                await self.prisma_client.db.litellm_teamtable.find_unique(
-                    where={"team_id": team_id}
-                )
+            team_row: prisma_models.LiteLLM_TeamTable | None = await _team_table(self.prisma_client).find_unique(
+                where={"team_id": team_id}
             )
             return getattr(team_row, "team_alias", None) if team_row is not None else None
         except Exception as e:
             verbose_proxy_logger.error(f"CheckBatchCost: could not look up team alias for team {team_id}: {e}")
             return None
 
-    async def _get_org_id(self, job: "LiteLLM_ManagedObjectTable", batch_id: str) -> str | None:
+    async def _get_org_id(self, job: "_ManagedObjectRow", batch_id: str) -> str | None:
         org_id = getattr(job, "org_id", None)
         if org_id:
             return org_id
@@ -150,11 +181,9 @@ class CheckBatchCost:
         team_id = getattr(job, "team_id", None)
         if api_key:
             try:
-                key_row: prisma_models.LiteLLM_VerificationToken | None = (
-                    await self.prisma_client.db.litellm_verificationtoken.find_unique(
-                        where={"token": api_key}
-                    )
-                )
+                key_row: prisma_models.LiteLLM_VerificationToken | None = await _token_table(
+                    self.prisma_client
+                ).find_unique(where={"token": api_key})
                 key_org_id = getattr(key_row, "organization_id", None) if key_row is not None else None
                 if key_org_id:
                     return key_org_id
@@ -166,10 +195,8 @@ class CheckBatchCost:
         if not team_id:
             return None
         try:
-            team_row: prisma_models.LiteLLM_TeamTable | None = (
-                await self.prisma_client.db.litellm_teamtable.find_unique(
-                    where={"team_id": team_id}
-                )
+            team_row: prisma_models.LiteLLM_TeamTable | None = await _team_table(self.prisma_client).find_unique(
+                where={"team_id": team_id}
             )
             return getattr(team_row, "organization_id", None) if team_row is not None else None
         except Exception as e:
@@ -177,7 +204,7 @@ class CheckBatchCost:
             return None
 
     async def _build_creator_attribution_metadata(
-        self, job: "LiteLLM_ManagedObjectTable", batch_id: str
+        self, job: "_ManagedObjectRow", batch_id: str
     ) -> dict[str, object]:
         """
         Rebuild the spend-tracking metadata for the key, team, and tags that created the
@@ -225,7 +252,7 @@ class CheckBatchCost:
         should not be polled.
         """
         cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
-        result: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
+        result: Final = await _managed_object_table(self.prisma_client).update_many(
             where={
                 "file_purpose": "batch",
                 "status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)},
@@ -244,7 +271,7 @@ class CheckBatchCost:
 
         # A row already in a terminal status is never rewritten by the sweep above, so
         # without this it keeps a poll-page slot forever and starves newer batches.
-        retired: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
+        retired: Final = await _managed_object_table(self.prisma_client).update_many(
             where={
                 "file_purpose": "batch",
                 "batch_processed": False,
@@ -259,9 +286,9 @@ class CheckBatchCost:
                 f"{MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days that were never costed"
             )
 
-    async def _fallback_find_jobs(self) -> list:
+    async def _fallback_find_jobs(self) -> "Sequence[_ManagedObjectRow]":
         """Query batch jobs without the batch_processed filter (for older schemas)."""
-        return await self.prisma_client.db.litellm_managedobjecttable.find_many(
+        return await _managed_object_table(self.prisma_client).find_many(
             where={
                 "file_purpose": "batch",
                 "status": {
@@ -279,7 +306,7 @@ class CheckBatchCost:
             order={"created_at": "asc"},
         )
 
-    async def _retire_job(self, job: "LiteLLM_ManagedObjectTable", reason: str) -> None:
+    async def _retire_job(self, job: "_ManagedObjectRow", reason: str) -> None:
         """
         Take a row that can never be costed out of the poll page. Leaving it selectable
         would burn one of the MAX_OBJECTS_PER_POLL_CYCLE slots on every future cycle, and
@@ -292,7 +319,7 @@ class CheckBatchCost:
             else {"status": "stale_expired"}
         )
         try:
-            await self.prisma_client.db.litellm_managedobjecttable.update(
+            await _managed_object_table(self.prisma_client).update(
                 where={"id": job.id},
                 data=data,
             )
@@ -306,7 +333,7 @@ class CheckBatchCost:
             "so it will no longer be polled"
         )
 
-    async def _claim_job_for_costing(self, job: "LiteLLM_ManagedObjectTable") -> bool:
+    async def _claim_job_for_costing(self, job: "_ManagedObjectRow") -> bool:
         """
         Atomically flip batch_processed from false to true, returning whether this pod won
         the row. Every pod and uvicorn worker schedules its own poller against the shared
@@ -321,7 +348,7 @@ class CheckBatchCost:
         if not self._has_batch_processed_column:
             return True
         try:
-            claimed: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
+            claimed: Final = await _managed_object_table(self.prisma_client).update_many(
                 where={"id": job.id, "batch_processed": False},
                 data={"batch_processed": True},
             )
@@ -332,7 +359,7 @@ class CheckBatchCost:
             return False
         return claimed > 0
 
-    async def _release_job_claim(self, job: "LiteLLM_ManagedObjectTable") -> None:
+    async def _release_job_claim(self, job: "_ManagedObjectRow") -> None:
         """Give a claimed row back once billing it failed, so a later poll cycle retries it.
 
         Safe to match on batch_processed=True: while this poller is active the retrieve
@@ -342,7 +369,7 @@ class CheckBatchCost:
         if not self._has_batch_processed_column:
             return
         try:
-            await self.prisma_client.db.litellm_managedobjecttable.update_many(
+            await _managed_object_table(self.prisma_client).update_many(
                 where={"id": job.id, "batch_processed": True},
                 data={"batch_processed": False},
             )
@@ -353,7 +380,7 @@ class CheckBatchCost:
             )
 
     @staticmethod
-    def _has_unified_id_without_model(job: "LiteLLM_ManagedObjectTable") -> bool:
+    def _has_unified_id_without_model(job: "_ManagedObjectRow") -> bool:
         """A unified id that decodes but carries no model_id can never be routed."""
         from litellm.proxy.openai_files_endpoints.common_utils import (
             convert_b64_uid_to_unified_uid,
@@ -402,7 +429,7 @@ class CheckBatchCost:
         return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
 
     async def _finalize_unbilled_terminal_job(
-        self, job: "prisma_models.LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
+        self, job: "_ManagedObjectRow", response: "LiteLLMBatch"
     ) -> None:
         """Persist a terminal batch that has nothing billable, converting any raw
         provider file ids to managed ids, and take it out of the poll page."""
@@ -426,7 +453,7 @@ class CheckBatchCost:
                 "file_object": response.model_dump_json(),
                 **({"batch_processed": True} if self._has_batch_processed_column else {}),
             }
-            await self.prisma_client.db.litellm_managedobjecttable.update(
+            await _managed_object_table(self.prisma_client).update(
                 where={"id": job.id},
                 data=update_data,
             )
@@ -447,7 +474,7 @@ class CheckBatchCost:
 
     def _resolve_job_routing(
         self,
-        job: "LiteLLM_ManagedObjectTable",
+        job: "_ManagedObjectRow",
         prom_logger: Optional["PrometheusLogger"],
     ) -> Optional[Tuple[str, str]]:
         """
@@ -524,7 +551,7 @@ class CheckBatchCost:
 
     def _resolve_unmanaged_provider_routing(
         self,
-        job: "LiteLLM_ManagedObjectTable",
+        job: "_ManagedObjectRow",
         prom_logger: Optional["PrometheusLogger"],
         llm_provider: str,
         bare_model_name: str,
@@ -620,7 +647,7 @@ class CheckBatchCost:
     @classmethod
     def _get_managed_file_model_name(
         cls,
-        job: "LiteLLM_ManagedObjectTable",
+        job: "_ManagedObjectRow",
         deployment_info: "Deployment",
     ) -> Optional[str]:
         """
@@ -640,7 +667,7 @@ class CheckBatchCost:
         )
 
     @staticmethod
-    def _get_input_file_id(job: "LiteLLM_ManagedObjectTable") -> Optional[str]:
+    def _get_input_file_id(job: "_ManagedObjectRow") -> Optional[str]:
         import json
 
         from litellm.types.utils import LiteLLMBatch
@@ -660,7 +687,7 @@ class CheckBatchCost:
 
     async def _track_completed_batch_cost(
         self,
-        job: "LiteLLM_ManagedObjectTable",
+        job: "_ManagedObjectRow",
         response: "LiteLLMBatch",
         model_id: str,
         batch_id: str,
@@ -936,7 +963,7 @@ class CheckBatchCost:
                 # endpoint may transition a batch to "complete" before
                 # CheckBatchCost runs.  The batch_processed=False filter
                 # already prevents reprocessing finished batches.
-                jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many(
+                jobs = await _managed_object_table(self.prisma_client).find_many(
                     where={
                         "file_purpose": "batch",
                         "batch_processed": False,
@@ -1038,7 +1065,7 @@ class CheckBatchCost:
                     }
                     if self._has_batch_processed_column:
                         update_data["batch_processed"] = True
-                    await self.prisma_client.db.litellm_managedobjecttable.update(
+                    await _managed_object_table(self.prisma_client).update(
                         where={"id": job.id},
                         data=update_data,
                     )

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -6,7 +6,7 @@ same route are non-inference and free.
 """
 
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Dict, Optional, cast
+from typing import TYPE_CHECKING, Dict, Final, Optional, Protocol, cast
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -22,11 +22,31 @@ from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
 
 if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient, ProxyLogging
+    from litellm.repositories.prisma_protocols import TableActions
     from litellm.router import Router
 
 TERMINAL_RESPONSE_STATUSES = frozenset({"completed", "failed", "cancelled", "incomplete"})
 
 
+class _ManagedObjectRow(Protocol):
+    @property
+    def id(self) -> str: ...
+
+    @property
+    def unified_object_id(self) -> str: ...
+
+    @property
+    def created_by(self) -> str | None: ...
+
+    @property
+    def file_object(self) -> object: ...
+
+
+def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]":
+    table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable
+    return table
+
+
 class CheckResponsesCost:
     def __init__(
         self,
@@ -128,7 +148,7 @@ class CheckResponsesCost:
                 f"CheckResponsesCost: stale cleanup failed (poll will continue): {cleanup_err}"
             )
 
-        jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many(
+        jobs = await _managed_object_table(self.prisma_client).find_many(
             where={
                 "status": {"in": ["queued", "in_progress"]},
                 "file_purpose": "response",
@@ -138,7 +158,7 @@ class CheckResponsesCost:
         )
         
         verbose_proxy_logger.debug(f"Found {len(jobs)} response jobs to check")
-        completed_jobs = []
+        completed_jobs: Final[list[_ManagedObjectRow]] = []
 
         for job in jobs:
             unified_object_id = job.unified_object_id
@@ -189,7 +209,7 @@ class CheckResponsesCost:
 
         # Mark completed jobs in the database
         if len(completed_jobs) > 0:
-            await self.prisma_client.db.litellm_managedobjecttable.update_many(
+            await _managed_object_table(self.prisma_client).update_many(
                 where={"id": {"in": [job.id for job in completed_jobs]}},
                 data={"status": "completed"},
             )

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -481,10 +481,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         """
         if self.prisma_client is None:
             return
-        managed_object = (
-            await self.prisma_client.db.litellm_managedobjecttable.find_first(
-                where={"OR": [{"unified_object_id": object_id}, {"model_object_id": object_id}]}
-            )
+        managed_object = await _managed_object_table(self.prisma_client).find_first(
+            where={"OR": [{"unified_object_id": object_id}, {"model_object_id": object_id}]}
         )
         if managed_object is None:
             return
@@ -509,10 +507,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         """
         if self.prisma_client is None:
             return
-        managed_file = (
-            await self.prisma_client.db.litellm_managedfiletable.find_first(
-                where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]}
-            )
+        managed_file = await _managed_file_table(self.prisma_client).find_first(
+            where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]}
         )
         if managed_file is None:
             return
@@ -535,8 +531,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         provider_file_ids = tuple(
             file_id
             for file_id in (
-                getattr(response, "output_file_id", None),
-                getattr(response, "error_file_id", None),
+                response.output_file_id,
+                response.error_file_id,
             )
             if file_id and not _is_base64_encoded_unified_file_id(file_id)
         )
@@ -544,10 +540,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             return
         if self.prisma_client is None:
             return
-        batch_row = (
-            await self.prisma_client.db.litellm_managedobjecttable.find_first(
-                where={"unified_object_id": response.id}
-            )
+        batch_row = await _managed_object_table(self.prisma_client).find_first(
+            where={"unified_object_id": response.id}
         )
         if batch_row is None or (
             batch_row.created_by is None and batch_row.team_id is None

litellm-proxy-extras/litellm_proxy_extras/migrations/20260921000000_add_password_reset_columns/migration.sql

@@ -0,0 +1,3 @@
+ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "password_reset_required" BOOLEAN;
+
+ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "last_breach_check_at" TIMESTAMP(3);

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -246,6 +246,8 @@ model LiteLLM_UserTable {
     organization_id String?
     object_permission_id String?
     password  String?
+    password_reset_required Boolean?
+    last_breach_check_at DateTime?
     teams    String[] @default([])
     user_role  String?
 		max_budget Float?

litellm-rust/Cargo.lock

@@ -115,6 +115,28 @@ dependencies = [
  "pin-project-lite",
 ]
 
+[[package]]
+name = "async-stream"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476"
+dependencies = [
+ "async-stream-impl",
+ "futures-core",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "async-stream-impl"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
 [[package]]
 name = "async-trait"
 version = "0.1.91"
@@ -599,6 +621,37 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "azure_storage_blob"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17b10207ecf7d666df6940b50051f433b3cd5d2b9b1dd190613208d7a84e7eed"
+dependencies = [
+ "async-stream",
+ "async-trait",
+ "azure_core",
+ "azure_storage_common",
+ "bytes",
+ "futures",
+ "percent-encoding",
+ "pin-project",
+ "serde",
+ "serde_json",
+ "time",
+ "tokio",
+]
+
+[[package]]
+name = "azure_storage_common"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0af2e6aeb8d76b17fc998f453c320913f73787b944e3cc29509d19411fa0321d"
+dependencies = [
+ "azure_core",
+ "serde",
+ "time",
+]
+
 [[package]]
 name = "base64"
 version = "0.13.1"
@@ -927,6 +980,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crc16"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff"
+
 [[package]]
 name = "crc32fast"
 version = "1.5.1"
@@ -1318,6 +1377,18 @@ dependencies = [
  "pin-project-lite",
 ]
 
+[[package]]
+name = "fallible-iterator"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
+
+[[package]]
+name = "fallible-streaming-iterator"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
+
 [[package]]
 name = "fancy-regex"
 version = "0.17.0"
@@ -1369,6 +1440,12 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foldhash"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.2"
@@ -1833,11 +1910,32 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
+[[package]]
+name = "hashbrown"
+version = "0.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
+dependencies = [
+ "foldhash",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "foldhash",
+]
+
+[[package]]
+name = "hashlink"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb"
+dependencies = [
+ "hashbrown 0.17.1",
+]
 
 [[package]]
 name = "heck"
@@ -2218,6 +2316,12 @@ version = "2.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
 
+[[package]]
+name = "iter-read"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071ed4cc1afd86650602c7b11aa2e1ce30762a1c27193201cb5cee9c6ebb1294"
+
 [[package]]
 name = "itertools"
 version = "0.13.0"
@@ -2376,6 +2480,17 @@ version = "0.2.186"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
 
+[[package]]
+name = "libsqlite3-sys"
+version = "0.38.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8"
+dependencies = [
+ "cc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.12.1"
@@ -2464,6 +2579,38 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "litellm-cache-azure-blob"
+version = "0.1.0"
+dependencies = [
+ "async-trait",
+ "azure_core",
+ "azure_storage_blob",
+ "futures-util",
+ "litellm-auth-azure",
+ "litellm-auth-types",
+ "litellm-cache",
+ "litellm-cache-response",
+ "serde_json",
+ "tokio",
+ "url",
+]
+
+[[package]]
+name = "litellm-cache-disk"
+version = "0.1.0"
+dependencies = [
+ "litellm-cache",
+ "py_literal",
+ "rand 0.8.7",
+ "rstest",
+ "rusqlite",
+ "serde-pickle",
+ "serde_json",
+ "tempfile",
+ "tokio",
+]
+
 [[package]]
 name = "litellm-cache-memory"
 version = "0.1.0"
@@ -2666,6 +2813,8 @@ dependencies = [
  "litellm-auth",
  "litellm-auth-gcp",
  "litellm-cache",
+ "litellm-cache-azure-blob",
+ "litellm-cache-disk",
  "litellm-cache-memory",
  "litellm-cache-redis",
  "litellm-cache-response",
@@ -2682,6 +2831,7 @@ dependencies = [
  "rstest",
  "serde",
  "serde_json",
+ "serde_with",
  "tokio",
  "tokio-tungstenite",
 ]
@@ -2697,6 +2847,8 @@ dependencies = [
  "jsonwebtoken",
  "litellm-core-utils",
  "litellm-secrets-aws",
+ "litellm-secrets-azure",
+ "litellm-secrets-cyberark",
  "litellm-secrets-google",
  "litellm-secrets-types",
  "moka",
@@ -2731,6 +2883,46 @@ dependencies = [
  "wiremock",
 ]
 
+[[package]]
+name = "litellm-secrets-azure"
+version = "0.1.0"
+dependencies = [
+ "litellm-auth-azure",
+ "litellm-auth-types",
+ "litellm-core-utils",
+ "litellm-secrets-types",
+ "percent-encoding",
+ "reqwest 0.12.28",
+ "rstest",
+ "serde",
+ "serde_json",
+ "sha2 0.10.9",
+ "thiserror 2.0.19",
+ "tokio",
+ "veil",
+ "wiremock",
+]
+
+[[package]]
+name = "litellm-secrets-cyberark"
+version = "0.1.0"
+dependencies = [
+ "base64 0.22.1",
+ "litellm-core-utils",
+ "litellm-secrets-types",
+ "moka",
+ "percent-encoding",
+ "reqwest 0.12.28",
+ "rstest",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.19",
+ "tokio",
+ "tracing",
+ "veil",
+ "wiremock",
+]
+
 [[package]]
 name = "litellm-secrets-google"
 version = "0.1.0"
@@ -3487,6 +3679,16 @@ version = "1.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
+[[package]]
+name = "quick-xml"
+version = "0.41.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
 [[package]]
 name = "quinn"
 version = "0.11.11"
@@ -3709,9 +3911,11 @@ checksum = "2acbc41a996f7652b2ddd9dfd98cc4ff602cfd742ae35382f07f608405ab50ed"
 dependencies = [
  "arcstr",
  "combine",
+ "crc16",
  "itoa",
  "num-bigint 0.5.1",
  "percent-encoding",
+ "rand 0.10.2",
  "rustls 0.23.42",
  "rustls-native-certs",
  "ryu",
@@ -3901,6 +4105,16 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rsqlite-vfs"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c"
+dependencies = [
+ "hashbrown 0.16.1",
+ "thiserror 2.0.19",
+]
+
 [[package]]
 name = "rstest"
 version = "0.26.1"
@@ -3941,6 +4155,21 @@ dependencies = [
  "syn 2.0.119",
 ]
 
+[[package]]
+name = "rusqlite"
+version = "0.40.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3"
+dependencies = [
+ "bitflags 2.13.1",
+ "fallible-iterator",
+ "fallible-streaming-iterator",
+ "hashlink",
+ "libsqlite3-sys",
+ "smallvec",
+ "sqlite-wasm-rs",
+]
+
 [[package]]
 name = "rustc-hash"
 version = "2.1.3"
@@ -4198,6 +4427,19 @@ dependencies = [
  "serde_derive",
 ]
 
+[[package]]
+name = "serde-pickle"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b641fdc8bcf2781ee78b30c599700d64ad4f412976143e4c5d0b9df906bb4843"
+dependencies = [
+ "byteorder",
+ "iter-read",
+ "num-bigint 0.4.8",
+ "num-traits",
+ "serde",
+]
+
 [[package]]
 name = "serde_core"
 version = "1.0.229"
@@ -4425,6 +4667,18 @@ dependencies = [
  "unicode-segmentation",
 ]
 
+[[package]]
+name = "sqlite-wasm-rs"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75"
+dependencies = [
+ "cc",
+ "js-sys",
+ "rsqlite-vfs",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "sse-stream"
 version = "0.2.6"
@@ -5030,6 +5284,7 @@ dependencies = [
  "base64 0.22.1",
  "bytes",
  "futures",
+ "quick-xml",
  "serde",
  "serde_json",
  "url",
@@ -5170,6 +5425,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
 
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
 [[package]]
 name = "veil"
 version = "0.3.0"

litellm-rust/Cargo.toml

@@ -22,13 +22,17 @@ litellm-secrets = { path = "crates/secrets" }
 litellm-secrets-types = { path = "crates/secrets-types" }
 litellm-secrets-aws = { path = "crates/secrets-aws" }
 litellm-secrets-google = { path = "crates/secrets-google" }
+litellm-secrets-azure = { path = "crates/secrets-azure" }
+litellm-secrets-cyberark = { path = "crates/secrets-cyberark" }
 litellm-http = { path = "crates/http" }
 litellm-llms = { path = "crates/llms" }
 litellm-types = { path = "crates/types" }
 litellm-core-utils = { path = "crates/core-utils" }
 litellm-cache = { path = "crates/cache" }
+litellm-cache-azure-blob = { path = "crates/cache-azure-blob" }
 litellm-cache-memory = { path = "crates/cache-memory" }
 litellm-cache-redis = { path = "crates/cache-redis" }
+litellm-cache-disk = { path = "crates/cache-disk" }
 litellm-cache-response = { path = "crates/cache-response" }
 litellm-token-counter = { path = "crates/token-counter" }
 litellm-token-counter-fast = { path = "crates/token-counter-fast" }

litellm-rust/crates/auth-azure/src/lib.rs

@@ -4,4 +4,4 @@ mod resolve;
 mod types;
 
 pub use resolve::AzureAuthService;
-pub use types::AzureAuthInputs;
+pub use types::{AzureAuthInputs, ConfigValue};

litellm-rust/crates/auth-azure/src/types.rs

@@ -51,6 +51,21 @@ pub struct AzureAuthInputs {
 }
 
 impl AzureAuthInputs {
+    pub fn default_credential_for_scope(scope: &str) -> Self {
+        Self {
+            azure_scope: ConfigValue::Value(Sourced::new(
+                scope.to_string(),
+                InputSource::Deployment,
+            )),
+            azure_credential: ConfigValue::Value(Sourced::new(
+                "DefaultAzureCredential".to_string(),
+                InputSource::Deployment,
+            )),
+            enable_azure_ad_token_refresh: Sourced::new(true, InputSource::Deployment),
+            ..Self::default()
+        }
+    }
+
     pub fn or_configured_token_refresh(self, enabled: bool) -> Self {
         if *self.enable_azure_ad_token_refresh.value() || !enabled {
             return self;

litellm-rust/crates/cache-azure-blob/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "litellm-cache-azure-blob"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-auth-azure.workspace = true
+litellm-auth-types.workspace = true
+litellm-cache.workspace = true
+
+async-trait = "0.1"
+azure_core = "1.1.0"
+azure_storage_blob = "1.1.0"
+futures-util.workspace = true
+tokio.workspace = true
+url.workspace = true
+
+[dev-dependencies]
+litellm-cache-response.workspace = true
+serde_json.workspace = true

litellm-rust/crates/cache-azure-blob/src/cache.rs

@@ -0,0 +1,254 @@
+use std::{sync::Arc, time::Duration};
+
+use azure_core::{
+    credentials::TokenCredential,
+    error::ErrorKind,
+    http::{ClientOptions, RequestContent},
+};
+use azure_storage_blob::{
+    BlobContainerClient, BlobContainerClientOptions,
+    models::{BlobClientUploadOptions, StorageErrorCode},
+};
+use futures_util::{TryStreamExt, future::try_join_all};
+use litellm_cache::{
+    BaseCache, BatchCache, CacheCodec, CacheConnectionResult, CacheConnectionStatus, Error,
+    ExactCacheContext, FlushCache,
+};
+use tokio::runtime::Handle;
+use url::Url;
+
+use crate::credential::AzureBlobCredential;
+
+pub struct AzureBlobCache<C> {
+    container: BlobContainerClient,
+    codec: C,
+    runtime: Handle,
+    account_url: String,
+    container_name: String,
+}
+
+impl<C: CacheCodec> AzureBlobCache<C> {
+    pub async fn connect(
+        account_url: &str,
+        container: &str,
+        codec: C,
+        runtime: Handle,
+    ) -> Result<Self, Error> {
+        Self::connect_with_options(
+            account_url,
+            container,
+            Some(Arc::new(AzureBlobCredential::default())),
+            ClientOptions::default(),
+            codec,
+            runtime,
+        )
+        .await
+    }
+
+    pub async fn connect_with_options(
+        account_url: &str,
+        container: &str,
+        credential: Option<Arc<dyn TokenCredential>>,
+        client_options: ClientOptions,
+        codec: C,
+        runtime: Handle,
+    ) -> Result<Self, Error> {
+        let parsed = Url::parse(account_url).map_err(|_| Error::Unavailable)?;
+        let account_url = parsed.as_str().trim_end_matches('/').to_string();
+        let container_url = {
+            let mut url = parsed;
+            url.path_segments_mut()
+                .map_err(|()| Error::Unavailable)?
+                .pop_if_empty()
+                .push(container);
+            url
+        };
+        let client = BlobContainerClient::new(
+            container_url,
+            credential,
+            Some(BlobContainerClientOptions {
+                client_options,
+                ..BlobContainerClientOptions::default()
+            }),
+        )
+        .map_err(|_| Error::Unavailable)?;
+        let cache = Self {
+            container: client,
+            codec,
+            runtime,
+            account_url,
+            container_name: container.to_string(),
+        };
+        cache.create_container().await?;
+        Ok(cache)
+    }
+
+    pub fn account_url(&self) -> &str {
+        &self.account_url
+    }
+
+    pub fn container_name(&self) -> &str {
+        &self.container_name
+    }
+
+    async fn create_container(&self) -> Result<(), Error> {
+        match self.container.create(None).await {
+            Ok(_) => Ok(()),
+            Err(error) if is_storage_error(&error, StorageErrorCode::ContainerAlreadyExists) => {
+                Ok(())
+            }
+            Err(_) => Err(Error::Unavailable),
+        }
+    }
+
+    async fn upload(&self, key: &str, value: &C::Value, overwrite: bool) -> Result<(), Error> {
+        let payload = self.codec.encode(value)?;
+        let options = (!overwrite).then(|| BlobClientUploadOptions::default().if_not_exists());
+        match self
+            .container
+            .blob_client(key)
+            .upload(RequestContent::from(payload), options)
+            .await
+        {
+            Ok(_) => Ok(()),
+            Err(error) if !overwrite && is_already_present(&error) => Ok(()),
+            Err(_) => Err(Error::Unavailable),
+        }
+    }
+
+    async fn download(&self, key: &str) -> Result<Option<C::Value>, Error> {
+        let response = match self.container.blob_client(key).download(None).await {
+            Ok(response) => response,
+            Err(error) if is_storage_error(&error, StorageErrorCode::BlobNotFound) => {
+                return Ok(None);
+            }
+            Err(_) => return Err(Error::Unavailable),
+        };
+        let bytes = response
+            .body
+            .collect()
+            .await
+            .map_err(|_| Error::Unavailable)?;
+        self.codec.decode(&bytes).map(Some)
+    }
+
+    async fn delete_all_blobs(&self) -> Result<(), Error> {
+        let mut pages = self
+            .container
+            .list_blobs(None)
+            .map_err(|_| Error::Unavailable)?
+            .into_pages();
+        while let Some(page) = pages.try_next().await.map_err(|_| Error::Unavailable)? {
+            let page = page.into_model().map_err(|_| Error::Unavailable)?;
+            for name in page.blob_items.into_iter().filter_map(|item| item.name) {
+                self.container
+                    .blob_client(&name)
+                    .delete(None)
+                    .await
+                    .map_err(|_| Error::Unavailable)?;
+            }
+        }
+        Ok(())
+    }
+
+    fn block_on<T>(&self, future: impl Future<Output = T>) -> T {
+        self.runtime.block_on(future)
+    }
+}
+
+fn is_already_present(error: &azure_core::Error) -> bool {
+    is_storage_error(error, StorageErrorCode::BlobAlreadyExists)
+        || is_storage_error(error, StorageErrorCode::ConditionNotMet)
+}
+
+fn is_storage_error(error: &azure_core::Error, code: StorageErrorCode) -> bool {
+    matches!(
+        error.kind(),
+        ErrorKind::HttpResponse {
+            error_code: Some(error_code),
+            ..
+        } if error_code == code.as_ref()
+    )
+}
+
+impl<C: CacheCodec> BaseCache for AzureBlobCache<C> {
+    type Value = C::Value;
+    type Context = ExactCacheContext;
+
+    fn get_ttl(&self, _: &ExactCacheContext) -> Option<Duration> {
+        None
+    }
+
+    fn set_cache(&self, key: &str, value: C::Value, _: &ExactCacheContext) -> Result<(), Error> {
+        self.block_on(self.upload(key, &value, false))
+    }
+
+    fn get_cache(&self, key: &str, _: &ExactCacheContext) -> Result<Option<C::Value>, Error> {
+        self.block_on(self.download(key))
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: C::Value,
+        _: ExactCacheContext,
+    ) -> Result<(), Error> {
+        self.upload(key, &value, true).await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        _: &ExactCacheContext,
+    ) -> Result<Option<C::Value>, Error> {
+        self.download(key).await
+    }
+
+    async fn async_set_cache_pipeline(
+        &self,
+        entries: Vec<(String, C::Value)>,
+        _: ExactCacheContext,
+    ) -> Result<(), Error> {
+        try_join_all(
+            entries
+                .iter()
+                .map(|(key, value)| self.upload(key, value, true)),
+        )
+        .await
+        .map(drop)
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        Ok(match self.container.get_properties(None).await {
+            Ok(_) => CacheConnectionResult {
+                status: CacheConnectionStatus::Success,
+                message: "Azure Blob cache connection test successful".into(),
+                error: None,
+            },
+            Err(error) => CacheConnectionResult {
+                status: CacheConnectionStatus::Failed,
+                message: format!("Azure Blob connection failed: {error}"),
+                error: Some(error.to_string()),
+            },
+        })
+    }
+}
+
+impl<C: CacheCodec> BatchCache for AzureBlobCache<C> {}
+
+impl<C: CacheCodec> FlushCache for AzureBlobCache<C> {
+    fn flush_cache(&self) -> Result<(), Error> {
+        self.block_on(self.delete_all_blobs())
+    }
+
+    async fn async_flush_cache(&self) -> Result<(), Error> {
+        self.delete_all_blobs().await
+    }
+}
+
+#[cfg(test)]
+mod tests;

litellm-rust/crates/cache-azure-blob/src/cache/tests.rs

@@ -0,0 +1,746 @@
+use std::{
+    collections::BTreeMap,
+    sync::{Arc, Mutex},
+    time::Duration,
+};
+
+use azure_core::http::{
+    AsyncRawResponse, Body, ClientOptions, HttpClient, Method, Request, StatusCode, Transport,
+    headers::{HeaderName, Headers},
+};
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheConnectionStatus, Error, ExactCacheContext, FlushCache,
+};
+use litellm_cache_response::{
+    CacheEntry, CacheKeyField, CacheKeyInput, ResponseCache, ResponseCacheCodec,
+    ResponseCacheRequest, cache_key,
+};
+use serde_json::json;
+use tokio::runtime::Runtime;
+
+use super::AzureBlobCache;
+
+const ACCOUNT_URL: &str = "https://example.blob.core.windows.net";
+const CONTAINER: &str = "litellm-cache";
+const IF_NONE_MATCH: HeaderName = HeaderName::from_static("if-none-match");
+const ERROR_CODE: HeaderName = HeaderName::from_static("x-ms-error-code");
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct RecordedRequest {
+    method: Method,
+    path: String,
+    query: String,
+    if_none_match: Option<String>,
+}
+
+#[derive(Default)]
+struct FakeState {
+    container_exists: bool,
+    blobs: BTreeMap<String, Vec<u8>>,
+    requests: Vec<RecordedRequest>,
+    failing: bool,
+    precondition_conflicts: bool,
+}
+
+#[derive(Clone, Default)]
+struct FakeBlobService {
+    state: Arc<Mutex<FakeState>>,
+}
+
+impl std::fmt::Debug for FakeBlobService {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("FakeBlobService")
+    }
+}
+
+impl FakeBlobService {
+    fn with_existing_container() -> Self {
+        let service = Self::default();
+        service.state.lock().unwrap().container_exists = true;
+        service
+    }
+
+    fn blob(&self, name: &str) -> Option<Vec<u8>> {
+        self.state.lock().unwrap().blobs.get(name).cloned()
+    }
+
+    fn blob_names(&self) -> Vec<String> {
+        self.state.lock().unwrap().blobs.keys().cloned().collect()
+    }
+
+    fn seed_blob(&self, name: &str, bytes: &[u8]) {
+        self.state
+            .lock()
+            .unwrap()
+            .blobs
+            .insert(name.to_string(), bytes.to_vec());
+    }
+
+    fn set_failing(&self, failing: bool) {
+        self.state.lock().unwrap().failing = failing;
+    }
+
+    fn set_precondition_conflicts(&self, enabled: bool) {
+        self.state.lock().unwrap().precondition_conflicts = enabled;
+    }
+
+    fn requests(&self) -> Vec<RecordedRequest> {
+        self.state.lock().unwrap().requests.clone()
+    }
+
+    fn container_exists(&self) -> bool {
+        self.state.lock().unwrap().container_exists
+    }
+
+    fn respond(status: StatusCode, error_code: Option<&str>, body: Vec<u8>) -> AsyncRawResponse {
+        let mut headers = Headers::new();
+        if let Some(code) = error_code {
+            headers.insert(ERROR_CODE, code.to_string());
+        }
+        AsyncRawResponse::from_bytes(status, headers, body)
+    }
+
+    fn list_body(state: &FakeState) -> Vec<u8> {
+        let mut xml = String::from(
+            r#"<?xml version="1.0" encoding="utf-8"?><EnumerationResults ServiceEndpoint="https://example.blob.core.windows.net/" ContainerName="litellm-cache"><Blobs>"#,
+        );
+        for name in state.blobs.keys() {
+            xml.push_str(&format!(
+                "<Blob><Name>{name}</Name><Properties><BlobType>BlockBlob</BlobType></Properties></Blob>"
+            ));
+        }
+        xml.push_str("</Blobs><NextMarker /></EnumerationResults>");
+        xml.into_bytes()
+    }
+}
+
+#[async_trait::async_trait]
+impl HttpClient for FakeBlobService {
+    async fn execute_request(&self, request: &Request) -> azure_core::Result<AsyncRawResponse> {
+        let mut state = self.state.lock().unwrap();
+        let path = request.url().path().to_string();
+        let query = request.url().query().unwrap_or_default().to_string();
+        let if_none_match = request
+            .headers()
+            .get_optional_str(&IF_NONE_MATCH)
+            .map(str::to_owned);
+        state.requests.push(RecordedRequest {
+            method: request.method(),
+            path: path.clone(),
+            query: query.clone(),
+            if_none_match: if_none_match.clone(),
+        });
+        if state.failing {
+            return Ok(Self::respond(
+                StatusCode::Forbidden,
+                Some("AuthorizationFailure"),
+                Vec::new(),
+            ));
+        }
+        let container_path = format!("/{CONTAINER}");
+        let blob_name = path
+            .strip_prefix(&format!("{container_path}/"))
+            .map(str::to_owned);
+        let is_container = path == container_path && query.contains("restype=container");
+        let response = match (request.method(), is_container, blob_name) {
+            (Method::Put, true, None) if state.container_exists => Self::respond(
+                StatusCode::Conflict,
+                Some("ContainerAlreadyExists"),
+                Vec::new(),
+            ),
+            (Method::Put, true, None) => {
+                state.container_exists = true;
+                Self::respond(StatusCode::Created, None, Vec::new())
+            }
+            (Method::Get, true, None) if query.contains("comp=list") => {
+                Self::respond(StatusCode::Ok, None, Self::list_body(&state))
+            }
+            (Method::Get, true, None) if state.container_exists => {
+                Self::respond(StatusCode::Ok, None, Vec::new())
+            }
+            (Method::Get, true, None) => {
+                Self::respond(StatusCode::NotFound, Some("ContainerNotFound"), Vec::new())
+            }
+            (Method::Put, false, Some(name)) => {
+                if if_none_match.as_deref() == Some("*") && state.blobs.contains_key(&name) {
+                    if state.precondition_conflicts {
+                        Self::respond(
+                            StatusCode::PreconditionFailed,
+                            Some("ConditionNotMet"),
+                            Vec::new(),
+                        )
+                    } else {
+                        Self::respond(StatusCode::Conflict, Some("BlobAlreadyExists"), Vec::new())
+                    }
+                } else {
+                    let bytes = match request.body() {
+                        Body::Bytes(bytes) => bytes.to_vec(),
+                        Body::SeekableStream(_) => panic!("unexpected streaming upload"),
+                    };
+                    state.blobs.insert(name, bytes);
+                    Self::respond(StatusCode::Created, None, Vec::new())
+                }
+            }
+            (Method::Get, false, Some(name)) => match state.blobs.get(&name) {
+                Some(bytes) => Self::respond(StatusCode::Ok, None, bytes.clone()),
+                None => Self::respond(StatusCode::NotFound, Some("BlobNotFound"), Vec::new()),
+            },
+            (Method::Delete, false, Some(name)) => match state.blobs.remove(&name) {
+                Some(_) => Self::respond(StatusCode::Accepted, None, Vec::new()),
+                None => Self::respond(StatusCode::NotFound, Some("BlobNotFound"), Vec::new()),
+            },
+            (method, _, _) => panic!("unexpected request {method:?} {path}?{query}"),
+        };
+        Ok(response)
+    }
+}
+
+struct Fixture {
+    runtime: Runtime,
+    service: FakeBlobService,
+    cache: Arc<AzureBlobCache<ResponseCacheCodec>>,
+}
+
+impl Fixture {
+    fn new(service: FakeBlobService) -> Self {
+        let runtime = Runtime::new().unwrap();
+        let cache = runtime
+            .block_on(Self::connect(&service, runtime.handle().clone()))
+            .unwrap();
+        Self {
+            runtime,
+            service,
+            cache: Arc::new(cache),
+        }
+    }
+
+    async fn connect(
+        service: &FakeBlobService,
+        handle: tokio::runtime::Handle,
+    ) -> Result<AzureBlobCache<ResponseCacheCodec>, Error> {
+        AzureBlobCache::connect_with_options(
+            ACCOUNT_URL,
+            CONTAINER,
+            None,
+            ClientOptions {
+                transport: Some(Transport::new(Arc::new(service.clone()))),
+                ..ClientOptions::default()
+            },
+            ResponseCacheCodec,
+            handle,
+        )
+        .await
+    }
+
+    fn response_cache(&self) -> ResponseCache<AzureBlobCache<ResponseCacheCodec>> {
+        ResponseCache::new(self.cache.clone())
+    }
+
+    fn stored_json(&self, key: &str) -> serde_json::Value {
+        serde_json::from_slice(&self.service.blob(key).expect("blob should exist")).unwrap()
+    }
+}
+
+fn request(model: &str) -> ResponseCacheRequest {
+    ResponseCacheRequest::new(CacheKeyInput {
+        fields: vec![CacheKeyField {
+            name: "model".into(),
+            value: Some(model.into()),
+            api_parameter: true,
+            internal_parameter: false,
+        }],
+        preset: None,
+        namespace: None,
+        include_provider_parameters: false,
+    })
+}
+
+fn now() -> Duration {
+    Duration::from_secs(1_700_000_000)
+}
+
+fn entry(value: serde_json::Value) -> CacheEntry {
+    CacheEntry {
+        timestamp: Some(1_700_000_000.5),
+        response: value,
+    }
+}
+
+fn no_ttl() -> ExactCacheContext {
+    ExactCacheContext::default()
+}
+
+fn with_ttl(seconds: u64) -> ExactCacheContext {
+    ExactCacheContext {
+        ttl: Some(Duration::from_secs(seconds)),
+    }
+}
+
+#[test]
+fn connect_creates_the_container_once() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    assert!(fixture.service.container_exists());
+    assert_eq!(
+        fixture.service.requests(),
+        vec![RecordedRequest {
+            method: Method::Put,
+            path: format!("/{CONTAINER}"),
+            query: "restype=container".into(),
+            if_none_match: None,
+        }]
+    );
+    assert_eq!(fixture.cache.account_url(), ACCOUNT_URL);
+    assert_eq!(fixture.cache.container_name(), CONTAINER);
+}
+
+#[test]
+fn connect_accepts_an_existing_container() {
+    let fixture = Fixture::new(FakeBlobService::with_existing_container());
+    assert!(fixture.service.container_exists());
+    assert_eq!(fixture.service.requests().len(), 1);
+}
+
+#[test]
+fn connect_accepts_account_urls_with_trailing_slash() {
+    let runtime = Runtime::new().unwrap();
+    let service = FakeBlobService::default();
+    let cache = runtime
+        .block_on(AzureBlobCache::connect_with_options(
+            "https://example.blob.core.windows.net/",
+            CONTAINER,
+            None,
+            ClientOptions {
+                transport: Some(Transport::new(Arc::new(service.clone()))),
+                ..ClientOptions::default()
+            },
+            ResponseCacheCodec,
+            runtime.handle().clone(),
+        ))
+        .unwrap();
+    assert_eq!(service.requests()[0].path, format!("/{CONTAINER}"));
+    assert_eq!(cache.account_url(), "https://example.blob.core.windows.net");
+}
+
+#[test]
+fn connect_keeps_account_url_query_parameters_on_the_container_path() {
+    let runtime = Runtime::new().unwrap();
+    let service = FakeBlobService::default();
+    runtime
+        .block_on(AzureBlobCache::connect_with_options(
+            "https://example.blob.core.windows.net/?sv=2024-01-01&sig=abc",
+            CONTAINER,
+            None,
+            ClientOptions {
+                transport: Some(Transport::new(Arc::new(service.clone()))),
+                ..ClientOptions::default()
+            },
+            ResponseCacheCodec,
+            runtime.handle().clone(),
+        ))
+        .unwrap();
+    let create = &service.requests()[0];
+    assert_eq!(create.path, format!("/{CONTAINER}"));
+    assert!(create.query.contains("sig=abc"));
+}
+
+#[test]
+fn connect_surfaces_service_failures() {
+    let runtime = Runtime::new().unwrap();
+    let service = FakeBlobService::default();
+    service.set_failing(true);
+    let result = runtime.block_on(Fixture::connect(&service, runtime.handle().clone()));
+    assert!(matches!(result, Err(Error::Unavailable)));
+}
+
+#[test]
+fn sync_set_and_get_round_trip_python_json_shape() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    let value = entry(json!({"choices": [{"message": {"content": "héllo 🌍"}}]}));
+    fixture
+        .cache
+        .set_cache("key-1", value.clone(), &no_ttl())
+        .unwrap();
+
+    assert_eq!(
+        fixture.stored_json("key-1"),
+        json!({
+            "timestamp": 1_700_000_000.5,
+            "response": {"choices": [{"message": {"content": "héllo 🌍"}}]}
+        })
+    );
+    assert_eq!(
+        fixture.cache.get_cache("key-1", &no_ttl()).unwrap(),
+        Some(value)
+    );
+}
+
+#[test]
+fn sync_set_does_not_overwrite_an_existing_blob() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture
+        .cache
+        .set_cache("key", entry(json!({"v": "first"})), &no_ttl())
+        .unwrap();
+    fixture
+        .cache
+        .set_cache("key", entry(json!({"v": "second"})), &no_ttl())
+        .unwrap();
+
+    assert_eq!(
+        fixture.stored_json("key")["response"],
+        json!({"v": "first"})
+    );
+    let uploads: Vec<_> = fixture
+        .service
+        .requests()
+        .into_iter()
+        .filter(|request| request.method == Method::Put && request.path.ends_with("/key"))
+        .collect();
+    assert_eq!(uploads.len(), 2);
+    assert!(
+        uploads
+            .iter()
+            .all(|request| request.if_none_match.as_deref() == Some("*"))
+    );
+}
+
+#[test]
+fn sync_set_treats_a_precondition_conflict_as_an_existing_blob() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture.service.set_precondition_conflicts(true);
+    fixture
+        .cache
+        .set_cache("key", entry(json!({"v": "first"})), &no_ttl())
+        .unwrap();
+    fixture
+        .cache
+        .set_cache("key", entry(json!({"v": "second"})), &no_ttl())
+        .unwrap();
+
+    assert_eq!(
+        fixture.stored_json("key")["response"],
+        json!({"v": "first"})
+    );
+}
+
+#[test]
+fn async_set_overwrites_an_existing_blob() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture.runtime.block_on(async {
+        fixture
+            .cache
+            .async_set_cache("key", entry(json!({"v": "first"})), no_ttl())
+            .await
+            .unwrap();
+        fixture
+            .cache
+            .async_set_cache("key", entry(json!({"v": "second"})), no_ttl())
+            .await
+            .unwrap();
+        assert_eq!(
+            fixture
+                .cache
+                .async_get_cache("key", &no_ttl())
+                .await
+                .unwrap(),
+            Some(entry(json!({"v": "second"})))
+        );
+    });
+    assert_eq!(
+        fixture.stored_json("key")["response"],
+        json!({"v": "second"})
+    );
+    assert!(
+        fixture
+            .service
+            .requests()
+            .iter()
+            .filter(|request| request.method == Method::Put && request.path.ends_with("/key"))
+            .all(|request| request.if_none_match.is_none())
+    );
+}
+
+#[test]
+fn missing_blobs_are_misses() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    assert_eq!(fixture.cache.get_cache("absent", &no_ttl()).unwrap(), None);
+    assert_eq!(
+        fixture
+            .runtime
+            .block_on(fixture.cache.async_get_cache("absent", &no_ttl()))
+            .unwrap(),
+        None
+    );
+}
+
+#[test]
+fn ttl_is_ignored_and_entries_never_expire() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    assert_eq!(fixture.cache.get_ttl(&with_ttl(1)), None);
+    assert_eq!(fixture.cache.get_ttl(&no_ttl()), None);
+
+    fixture
+        .cache
+        .set_cache("key", entry(json!("value")), &with_ttl(1))
+        .unwrap();
+    std::thread::sleep(Duration::from_millis(1100));
+    assert_eq!(
+        fixture.cache.get_cache("key", &with_ttl(1)).unwrap(),
+        Some(entry(json!("value")))
+    );
+    assert!(
+        fixture
+            .service
+            .requests()
+            .iter()
+            .all(|request| !request.query.contains("expiry"))
+    );
+}
+
+#[test]
+fn malformed_blobs_are_invalid_entries_and_response_cache_misses() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture.service.seed_blob("broken-json", b"{not json");
+    fixture
+        .service
+        .seed_blob("broken-utf8", &[0xff, 0xfe, 0x22]);
+    fixture
+        .service
+        .seed_blob("wrong-shape", br#"{"timestamp": "yesterday"}"#);
+
+    for key in ["broken-json", "broken-utf8", "wrong-shape"] {
+        assert!(matches!(
+            fixture.cache.get_cache(key, &no_ttl()),
+            Err(Error::InvalidEntry)
+        ));
+    }
+
+    let response_cache = fixture.response_cache();
+    let broken = request("broken");
+    fixture
+        .service
+        .seed_blob(&cache_key(&broken.key), b"{not json");
+    assert_eq!(response_cache.lookup(&broken, now()).unwrap(), None);
+    assert_eq!(
+        fixture
+            .runtime
+            .block_on(response_cache.async_lookup(&broken, now()))
+            .unwrap(),
+        None
+    );
+}
+
+#[test]
+fn batch_get_preserves_order_and_marks_misses_and_invalid_entries() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture
+        .cache
+        .set_cache("a", entry(json!("A")), &no_ttl())
+        .unwrap();
+    fixture
+        .cache
+        .set_cache("c", entry(json!("C")), &no_ttl())
+        .unwrap();
+    fixture.service.seed_blob("bad", b"nope");
+    let keys = ["c", "missing", "a", "bad"].map(String::from);
+
+    let sync = fixture.cache.batch_get_cache(&keys, &no_ttl()).unwrap();
+    assert_eq!(
+        sync,
+        vec![
+            BatchEntry::Hit(entry(json!("C"))),
+            BatchEntry::Miss,
+            BatchEntry::Hit(entry(json!("A"))),
+            BatchEntry::Invalid,
+        ]
+    );
+
+    let asynchronous = fixture
+        .runtime
+        .block_on(fixture.cache.async_batch_get_cache(keys.to_vec(), no_ttl()))
+        .unwrap();
+    assert_eq!(asynchronous, sync);
+
+    let response_cache = fixture.response_cache();
+    let requests = [request("hit"), request("missing"), request("bad")];
+    response_cache
+        .store(&requests[0], json!("HIT"), now())
+        .unwrap();
+    fixture
+        .service
+        .seed_blob(&cache_key(&requests[2].key), b"nope");
+    let hits = response_cache.lookup_batch(&requests, now()).unwrap();
+    assert_eq!(hits.values, vec![Some(json!("HIT")), None, None]);
+    assert_eq!(hits.missing_indices, vec![1, 2]);
+    let async_hits = fixture
+        .runtime
+        .block_on(response_cache.async_lookup_batch(&requests, now()))
+        .unwrap();
+    assert_eq!(async_hits.values, hits.values);
+}
+
+#[test]
+fn async_pipeline_writes_every_entry_with_overwrite() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture.service.seed_blob("k2", b"stale");
+    fixture
+        .runtime
+        .block_on(fixture.cache.async_set_cache_pipeline(
+            vec![
+                ("k1".into(), entry(json!({"n": 1}))),
+                ("k2".into(), entry(json!({"n": 2}))),
+                ("k3".into(), entry(json!({"n": 3}))),
+            ],
+            with_ttl(30),
+        ))
+        .unwrap();
+    assert_eq!(fixture.service.blob_names(), ["k1", "k2", "k3"]);
+    assert_eq!(fixture.stored_json("k2")["response"], json!({"n": 2}));
+}
+
+#[test]
+fn flush_deletes_every_blob_in_the_container() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    for key in ["x", "y", "z"] {
+        fixture
+            .cache
+            .set_cache(key, entry(json!(key)), &no_ttl())
+            .unwrap();
+    }
+    fixture.cache.flush_cache().unwrap();
+    assert!(fixture.service.blob_names().is_empty());
+    assert!(fixture.service.container_exists());
+
+    fixture
+        .cache
+        .set_cache("again", entry(json!(1)), &no_ttl())
+        .unwrap();
+    fixture
+        .runtime
+        .block_on(fixture.cache.async_flush_cache())
+        .unwrap();
+    assert!(fixture.service.blob_names().is_empty());
+}
+
+#[test]
+fn service_failures_map_to_unavailable() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture.service.set_failing(true);
+    assert!(matches!(
+        fixture.cache.get_cache("key", &no_ttl()),
+        Err(Error::Unavailable)
+    ));
+    assert!(matches!(
+        fixture.cache.set_cache("key", entry(json!(1)), &no_ttl()),
+        Err(Error::Unavailable)
+    ));
+    assert!(matches!(
+        fixture.cache.flush_cache(),
+        Err(Error::Unavailable)
+    ));
+    assert!(matches!(
+        fixture.runtime.block_on(
+            fixture
+                .cache
+                .async_set_cache_pipeline(vec![("k".into(), entry(json!(1)))], no_ttl())
+        ),
+        Err(Error::Unavailable)
+    ));
+}
+
+#[test]
+fn test_connection_reports_container_reachability() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    let ok = fixture
+        .runtime
+        .block_on(fixture.cache.test_connection())
+        .unwrap();
+    assert_eq!(ok.status, CacheConnectionStatus::Success);
+    assert!(ok.error.is_none());
+
+    fixture.service.set_failing(true);
+    let failed = fixture
+        .runtime
+        .block_on(fixture.cache.test_connection())
+        .unwrap();
+    assert_eq!(failed.status, CacheConnectionStatus::Failed);
+    assert!(failed.error.is_some());
+}
+
+#[test]
+fn disconnect_is_idempotent_and_keeps_data() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture
+        .cache
+        .set_cache("key", entry(json!(1)), &no_ttl())
+        .unwrap();
+    fixture.runtime.block_on(async {
+        fixture.cache.disconnect().await.unwrap();
+        fixture.cache.disconnect().await.unwrap();
+    });
+    assert_eq!(
+        fixture.cache.get_cache("key", &no_ttl()).unwrap(),
+        Some(entry(json!(1)))
+    );
+}
+
+#[test]
+fn response_cache_stores_and_reads_through_the_backend() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    let response_cache = fixture.response_cache();
+    let mut request = request("gpt");
+    request.context = with_ttl(60);
+    let response = json!({"id": "chatcmpl-1"});
+    response_cache
+        .store(&request, response.clone(), now())
+        .unwrap();
+    assert_eq!(
+        fixture.stored_json(&cache_key(&request.key)),
+        json!({"timestamp": 1_700_000_000.0, "response": {"id": "chatcmpl-1"}})
+    );
+    assert_eq!(
+        response_cache
+            .lookup(&request, now() + Duration::from_secs(3600))
+            .unwrap(),
+        Some(response.clone())
+    );
+    assert_eq!(
+        fixture
+            .runtime
+            .block_on(response_cache.async_lookup(&request, now() + Duration::from_secs(3600)))
+            .unwrap(),
+        Some(response.clone())
+    );
+    fixture.runtime.block_on(async {
+        response_cache
+            .async_store(&request, json!("replaced"), now())
+            .await
+            .unwrap();
+        assert_eq!(
+            response_cache.async_lookup(&request, now()).await.unwrap(),
+            Some(json!("replaced"))
+        );
+        response_cache.async_flush().await.unwrap();
+        assert_eq!(
+            response_cache.async_lookup(&request, now()).await.unwrap(),
+            None
+        );
+    });
+}
+
+#[test]
+fn non_object_responses_are_written_serialized_like_python() {
+    let fixture = Fixture::new(FakeBlobService::default());
+    fixture
+        .cache
+        .set_cache("s", entry(json!("plain")), &no_ttl())
+        .unwrap();
+    assert_eq!(
+        fixture.stored_json("s"),
+        json!({"timestamp": 1_700_000_000.5, "response": "\"plain\""})
+    );
+    assert_eq!(
+        fixture.cache.get_cache("s", &no_ttl()).unwrap(),
+        Some(entry(json!("plain")))
+    );
+}

litellm-rust/crates/cache-azure-blob/src/credential.rs

@@ -0,0 +1,84 @@
+use std::{
+    fmt,
+    sync::Arc,
+    time::{Duration, SystemTime},
+};
+
+use azure_core::{
+    credentials::{AccessToken, TokenCredential, TokenRequestOptions},
+    error::ErrorKind,
+    time::OffsetDateTime,
+};
+use litellm_auth_azure::{AzureAuthInputs, AzureAuthService};
+use litellm_auth_types::ResolvedCredential;
+
+const STATIC_TOKEN_LIFETIME: Duration = Duration::from_secs(300);
+const LLM_TOKEN_ENV: &str = "AZURE_AD_TOKEN";
+
+type EnvLookup = Arc<dyn Fn(&str) -> Option<String> + Send + Sync>;
+
+pub struct AzureBlobCredential {
+    service: AzureAuthService,
+    env_lookup: EnvLookup,
+}
+
+impl fmt::Debug for AzureBlobCredential {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str("AzureBlobCredential")
+    }
+}
+
+impl Default for AzureBlobCredential {
+    fn default() -> Self {
+        Self::new(
+            AzureAuthService::default(),
+            Arc::new(|name| std::env::var(name).ok()),
+        )
+    }
+}
+
+impl AzureBlobCredential {
+    pub fn new(service: AzureAuthService, env_lookup: EnvLookup) -> Self {
+        Self {
+            service,
+            env_lookup,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl TokenCredential for AzureBlobCredential {
+    async fn get_token(
+        &self,
+        scopes: &[&str],
+        _options: Option<TokenRequestOptions<'_>>,
+    ) -> azure_core::Result<AccessToken> {
+        let env_lookup = &self.env_lookup;
+        let lookup = move |name: &str| (name != LLM_TOKEN_ENV).then(|| env_lookup(name)).flatten();
+        let credential = self
+            .service
+            .get_azure_ad_token(
+                &AzureAuthInputs::default_credential_for_scope(&scopes.join(" ")),
+                &lookup,
+            )
+            .await
+            .map_err(|error| {
+                azure_core::Error::with_message(ErrorKind::Credential, error.to_string())
+            })?
+            .ok_or_else(|| {
+                azure_core::Error::with_message(
+                    ErrorKind::Credential,
+                    "no Azure credential is available for blob storage",
+                )
+            })?;
+        let (token, expires_on) = match credential.into_value() {
+            ResolvedCredential::AccessToken { token, expires_on } => (token, expires_on),
+            ResolvedCredential::Static(token) => (token, None),
+        };
+        let expires_on = expires_on.unwrap_or_else(|| SystemTime::now() + STATIC_TOKEN_LIFETIME);
+        Ok(AccessToken::new(
+            token.expose().to_string(),
+            OffsetDateTime::from(expires_on),
+        ))
+    }
+}

litellm-rust/crates/cache-azure-blob/src/lib.rs

@@ -0,0 +1,5 @@
+mod cache;
+mod credential;
+
+pub use cache::AzureBlobCache;
+pub use credential::AzureBlobCredential;

litellm-rust/crates/cache-disk/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "litellm-cache-disk"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-cache.workspace = true
+py_literal = "0.4.0"
+rand.workspace = true
+rusqlite = { version = "0.40", features = ["bundled"] }
+serde-pickle = "1.2"
+serde_json.workspace = true
+tokio.workspace = true
+
+[dev-dependencies]
+rstest.workspace = true
+tempfile = "3.27.0"

litellm-rust/crates/cache-disk/src/adapter.rs

@@ -0,0 +1,10 @@
+use litellm_cache::Error;
+
+use crate::StoredValue;
+
+pub trait ValueAdapter: Send + Sync + 'static {
+    fn read(&self, value: StoredValue) -> Result<Option<Vec<u8>>, Error>;
+    fn write(&self, payload: Vec<u8>) -> StoredValue;
+    fn counter_seed(&self, value: Option<StoredValue>) -> Result<f64, Error>;
+    fn counter_value(&self, value: f64) -> StoredValue;
+}

litellm-rust/crates/cache-disk/src/cache.rs

@@ -0,0 +1,301 @@
+use std::{
+    path::Path,
+    sync::Arc,
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, CacheConnectionStatus,
+    CounterCache, DeleteCache, Error, ExactCacheContext, FlushCache,
+};
+
+use crate::{DiskStore, DiskcacheSqliteStore, PythonDiskCacheAdapter, StoredValue, ValueAdapter};
+
+pub struct DiskCache<S, D = DiskcacheSqliteStore, A = PythonDiskCacheAdapter> {
+    store: Arc<D>,
+    adapter: Arc<A>,
+    codec: S,
+}
+
+impl<S: CacheCodec> DiskCache<S> {
+    pub fn open(directory: impl AsRef<Path>, codec: S) -> Result<Self, Error> {
+        Ok(Self {
+            store: Arc::new(DiskcacheSqliteStore::open(directory)?),
+            adapter: Arc::new(PythonDiskCacheAdapter),
+            codec,
+        })
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore> DiskCache<S, D, PythonDiskCacheAdapter> {
+    pub fn with_store(store: D, codec: S) -> Self {
+        Self {
+            store: Arc::new(store),
+            adapter: Arc::new(PythonDiskCacheAdapter),
+            codec,
+        }
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> DiskCache<S, D, A> {
+    pub fn with_adapter(store: D, adapter: A, codec: S) -> Self {
+        Self {
+            store: Arc::new(store),
+            adapter: Arc::new(adapter),
+            codec,
+        }
+    }
+
+    pub fn directory(&self) -> &Path {
+        self.store.directory()
+    }
+
+    fn decode_stored(&self, value: StoredValue) -> Result<Option<S::Value>, Error> {
+        let Some(bytes) = self.adapter.read(value)? else {
+            return Ok(None);
+        };
+        self.codec.decode(&bytes).map(Some)
+    }
+
+    async fn run_blocking<T, F>(store: Arc<D>, operation: F) -> Result<T, Error>
+    where
+        T: Send + 'static,
+        F: FnOnce(&D) -> Result<T, Error> + Send + 'static,
+    {
+        tokio::task::spawn_blocking(move || operation(&store))
+            .await
+            .map_err(|_| Error::Unavailable)?
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> BaseCache for DiskCache<S, D, A> {
+    type Value = S::Value;
+    type Context = ExactCacheContext;
+
+    fn get_ttl(&self, context: &Self::Context) -> Option<Duration> {
+        context.ttl
+    }
+
+    fn set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: &Self::Context,
+    ) -> Result<(), Error> {
+        let value = self.adapter.write(self.codec.encode(&value)?);
+        let expire_time = context.ttl.map(|ttl| unix_now() + ttl.as_secs_f64());
+        self.store.set(key, value, expire_time, unix_now())
+    }
+
+    fn get_cache(&self, key: &str, _: &Self::Context) -> Result<Option<Self::Value>, Error> {
+        self.store
+            .get(key, unix_now())?
+            .map(|value| self.decode_stored(value))
+            .transpose()
+            .map(|value| value.flatten())
+    }
+
+    async fn async_set_cache(
+        &self,
+        key: &str,
+        value: Self::Value,
+        context: ExactCacheContext,
+    ) -> Result<(), Error> {
+        let value = self.adapter.write(self.codec.encode(&value)?);
+        let ttl = context.ttl;
+        let key = key.to_string();
+        Self::run_blocking(Arc::clone(&self.store), move |store| {
+            let expire_time = ttl.map(|ttl| unix_now() + ttl.as_secs_f64());
+            store.set(&key, value, expire_time, unix_now())
+        })
+        .await
+    }
+
+    async fn async_get_cache(
+        &self,
+        key: &str,
+        _: &ExactCacheContext,
+    ) -> Result<Option<Self::Value>, Error> {
+        let key = key.to_string();
+        let value = Self::run_blocking(Arc::clone(&self.store), move |store| {
+            store.get(&key, unix_now())
+        })
+        .await?;
+        value
+            .map(|value| self.decode_stored(value))
+            .transpose()
+            .map(|value| value.flatten())
+    }
+
+    async fn async_set_cache_pipeline(
+        &self,
+        entries: Vec<(String, Self::Value)>,
+        context: ExactCacheContext,
+    ) -> Result<(), Error> {
+        let entries = entries
+            .into_iter()
+            .map(|(key, value)| {
+                self.codec
+                    .encode(&value)
+                    .map(|value| (key, self.adapter.write(value)))
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+        let expire_after = context.ttl;
+        Self::run_blocking(Arc::clone(&self.store), move |store| {
+            for (key, value) in entries {
+                let expire_time = expire_after.map(|ttl| unix_now() + ttl.as_secs_f64());
+                store.set(&key, value, expire_time, unix_now())?;
+            }
+            Ok(())
+        })
+        .await
+    }
+
+    async fn disconnect(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
+        let result = Self::run_blocking(Arc::clone(&self.store), |store| {
+            store.probe().map(|_| CacheConnectionResult {
+                status: CacheConnectionStatus::Success,
+                message: "Disk cache connection test successful".into(),
+                error: None,
+            })
+        })
+        .await;
+        Ok(match result {
+            Ok(result) => result,
+            Err(error) => CacheConnectionResult {
+                status: CacheConnectionStatus::Failed,
+                message: format!("Disk cache connection failed: {error}"),
+                error: Some(error.to_string()),
+            },
+        })
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> BatchCache for DiskCache<S, D, A> {
+    fn batch_get_cache(
+        &self,
+        keys: &[String],
+        context: &ExactCacheContext,
+    ) -> Result<Vec<BatchEntry<Self::Value>>, Error> {
+        keys.iter()
+            .map(|key| match self.get_cache(key, context) {
+                Ok(Some(value)) => Ok(BatchEntry::Hit(value)),
+                Ok(None) => Ok(BatchEntry::Miss),
+                Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid),
+                Err(error) => Err(error),
+            })
+            .collect()
+    }
+
+    async fn async_batch_get_cache(
+        &self,
+        keys: Vec<String>,
+        _: ExactCacheContext,
+    ) -> Result<Vec<BatchEntry<Self::Value>>, Error> {
+        let values = Self::run_blocking(Arc::clone(&self.store), move |store| {
+            keys.into_iter()
+                .map(|key| store.get(&key, unix_now()).map(|value| (key, value)))
+                .collect::<Result<Vec<_>, _>>()
+        })
+        .await?;
+        values
+            .into_iter()
+            .map(|(_, value)| match value {
+                None => Ok(BatchEntry::Miss),
+                Some(value) => match self.decode_stored(value) {
+                    Ok(Some(value)) => Ok(BatchEntry::Hit(value)),
+                    Ok(None) => Ok(BatchEntry::Miss),
+                    Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid),
+                    Err(error) => Err(error),
+                },
+            })
+            .collect()
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> DeleteCache for DiskCache<S, D, A> {
+    fn delete_cache(&self, key: &str) -> Result<(), Error> {
+        self.store.pop(key, unix_now()).map(|_| ())
+    }
+
+    async fn async_delete_cache(&self, key: &str) -> Result<(), Error> {
+        let key = key.to_string();
+        Self::run_blocking(Arc::clone(&self.store), move |store| {
+            store.pop(&key, unix_now()).map(|_| ())
+        })
+        .await
+    }
+}
+
+impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> FlushCache for DiskCache<S, D, A> {
+    fn flush_cache(&self) -> Result<(), Error> {
+        self.store.clear()
+    }
+
+    async fn async_flush_cache(&self) -> Result<(), Error> {
+        Self::run_blocking(Arc::clone(&self.store), |store| store.clear()).await
+    }
+}
+
+impl<S: CacheCodec<Value = f64>, D: DiskStore, A: ValueAdapter> CounterCache
+    for DiskCache<S, D, A>
+{
+    fn increment_cache(
+        &self,
+        key: &str,
+        amount: f64,
+        context: ExactCacheContext,
+    ) -> Result<f64, Error> {
+        increment(
+            self.adapter.as_ref(),
+            self.store.as_ref(),
+            key,
+            amount,
+            context.ttl,
+        )
+    }
+
+    async fn async_increment(
+        &self,
+        key: &str,
+        amount: f64,
+        context: ExactCacheContext,
+    ) -> Result<f64, Error> {
+        let key = key.to_string();
+        let adapter = Arc::clone(&self.adapter);
+        Self::run_blocking(Arc::clone(&self.store), move |store| {
+            increment(adapter.as_ref(), store, &key, amount, context.ttl)
+        })
+        .await
+    }
+}
+
+fn increment<A: ValueAdapter, D: DiskStore>(
+    adapter: &A,
+    store: &D,
+    key: &str,
+    amount: f64,
+    ttl: Option<Duration>,
+) -> Result<f64, Error> {
+    let mut result = None;
+    let mut apply = |current: Option<StoredValue>| {
+        let initial = adapter.counter_seed(current)?;
+        let value = initial + amount;
+        let stored = adapter.counter_value(value);
+        result = Some(value);
+        Ok((stored, ttl.map(|ttl| unix_now() + ttl.as_secs_f64())))
+    };
+    store.update(key, unix_now(), &mut apply)?;
+    result.ok_or(Error::InvalidEntry)
+}
+
+fn unix_now() -> f64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs_f64()
+}

litellm-rust/crates/cache-disk/src/lib.rs

@@ -0,0 +1,11 @@
+mod adapter;
+mod cache;
+mod python;
+mod sqlite;
+mod store;
+
+pub use adapter::ValueAdapter;
+pub use cache::DiskCache;
+pub use python::PythonDiskCacheAdapter;
+pub use sqlite::DiskcacheSqliteStore;
+pub use store::{DiskStore, StoredValue};

litellm-rust/crates/cache-disk/src/python/mod.rs

@@ -0,0 +1,77 @@
+mod value;
+
+use litellm_cache::Error;
+use py_literal::Value;
+
+use crate::{StoredValue, ValueAdapter};
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct PythonDiskCacheAdapter;
+
+impl PythonDiskCacheAdapter {
+    fn python_get_cache(value: StoredValue) -> Result<Option<Value>, Error> {
+        let value = match value {
+            StoredValue::Bytes(value) => Value::Bytes(value),
+            StoredValue::Text(value) => Value::String(value),
+            StoredValue::Integer(value) => Value::Integer(value.into()),
+            StoredValue::Float(value) => Value::Float(value),
+            StoredValue::Pickle(value) => value::from_pickle(&value)?,
+        };
+        if !value::is_truthy(&value) {
+            return Ok(None);
+        }
+        match value {
+            Value::String(text) => Ok(Some(
+                value::from_json_text(&text).unwrap_or(Value::String(text)),
+            )),
+            Value::Bytes(bytes) => match std::str::from_utf8(&bytes) {
+                Ok(text) => Ok(Some(
+                    value::from_json_text(text).unwrap_or(Value::Bytes(bytes)),
+                )),
+                Err(_) => Ok(Some(Value::Bytes(bytes))),
+            },
+            value => Ok(Some(value)),
+        }
+    }
+}
+
+impl ValueAdapter for PythonDiskCacheAdapter {
+    fn read(&self, value: StoredValue) -> Result<Option<Vec<u8>>, Error> {
+        match value {
+            StoredValue::Text(value) => Ok((!value.is_empty()).then(|| value.into_bytes())),
+            StoredValue::Bytes(value) => Ok((!value.is_empty()).then_some(value)),
+            value => {
+                let Some(value) = Self::python_get_cache(value)? else {
+                    return Ok(None);
+                };
+                value::to_json(&value).map(Some)
+            }
+        }
+    }
+
+    fn write(&self, payload: Vec<u8>) -> StoredValue {
+        StoredValue::Bytes(payload)
+    }
+
+    fn counter_seed(&self, value: Option<StoredValue>) -> Result<f64, Error> {
+        let Some(value) = value else {
+            return Ok(0.0);
+        };
+        let Some(value) = Self::python_get_cache(value)? else {
+            return Ok(0.0);
+        };
+        Ok(if value::is_int(&value) {
+            value::to_f64(&value).unwrap_or(0.0)
+        } else {
+            0.0
+        })
+    }
+
+    fn counter_value(&self, value: f64) -> StoredValue {
+        if value.fract() == 0.0 && value >= i64::MIN as f64 && value <= i64::MAX as f64 {
+            StoredValue::Integer(value as i64)
+        } else {
+            StoredValue::Float(value)
+        }
+    }
+}

litellm-rust/crates/cache-disk/src/python/value.rs

@@ -0,0 +1,173 @@
+use litellm_cache::Error;
+use py_literal::Value;
+use serde_json::{Map, Number};
+
+pub(crate) fn from_pickle(bytes: &[u8]) -> Result<Value, Error> {
+    let value = serde_pickle::value_from_slice(bytes, Default::default())
+        .map_err(|_| Error::InvalidEntry)?;
+    from_pickle_value(value)
+}
+
+fn from_pickle_value(value: serde_pickle::Value) -> Result<Value, Error> {
+    match value {
+        serde_pickle::Value::None => Ok(Value::None),
+        serde_pickle::Value::Bool(value) => Ok(Value::Boolean(value)),
+        serde_pickle::Value::I64(value) => integer(value.to_string()),
+        serde_pickle::Value::Int(value) => integer(value.to_string()),
+        serde_pickle::Value::F64(value) => Ok(Value::Float(value)),
+        serde_pickle::Value::String(value) => Ok(Value::String(value)),
+        serde_pickle::Value::Bytes(value) => Ok(Value::Bytes(value)),
+        serde_pickle::Value::List(values) => values
+            .into_iter()
+            .map(from_pickle_value)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::List),
+        serde_pickle::Value::Tuple(values) => values
+            .into_iter()
+            .map(from_pickle_value)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::Tuple),
+        serde_pickle::Value::Set(values) => values
+            .into_iter()
+            .map(from_pickle_hashable)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::Set),
+        serde_pickle::Value::FrozenSet(values) => values
+            .into_iter()
+            .map(from_pickle_hashable)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::Set),
+        serde_pickle::Value::Dict(values) => values
+            .into_iter()
+            .map(|(key, value)| Ok((from_pickle_hashable(key)?, from_pickle_value(value)?)))
+            .collect::<Result<Vec<_>, Error>>()
+            .map(Value::Dict),
+    }
+}
+
+fn from_pickle_hashable(value: serde_pickle::HashableValue) -> Result<Value, Error> {
+    Ok(match value {
+        serde_pickle::HashableValue::None => Value::None,
+        serde_pickle::HashableValue::Bool(value) => Value::Boolean(value),
+        serde_pickle::HashableValue::I64(value) => integer(value.to_string())?,
+        serde_pickle::HashableValue::Int(value) => integer(value.to_string())?,
+        serde_pickle::HashableValue::F64(value) => Value::Float(value),
+        serde_pickle::HashableValue::Bytes(value) => Value::Bytes(value),
+        serde_pickle::HashableValue::String(value) => Value::String(value),
+        serde_pickle::HashableValue::Tuple(values) => Value::Tuple(
+            values
+                .into_iter()
+                .map(from_pickle_hashable)
+                .collect::<Result<Vec<_>, _>>()?,
+        ),
+        serde_pickle::HashableValue::FrozenSet(values) => Value::Set(
+            values
+                .into_iter()
+                .map(from_pickle_hashable)
+                .collect::<Result<Vec<_>, _>>()?,
+        ),
+    })
+}
+
+fn integer(value: String) -> Result<Value, Error> {
+    value.parse().map_err(|_| Error::InvalidEntry)
+}
+
+pub(crate) fn from_json(value: serde_json::Value) -> Value {
+    match value {
+        serde_json::Value::Null => Value::None,
+        serde_json::Value::Bool(value) => Value::Boolean(value),
+        serde_json::Value::Number(value) => {
+            if value.is_i64() || value.is_u64() {
+                integer(value.to_string())
+                    .unwrap_or(Value::Float(value.as_f64().unwrap_or(f64::NAN)))
+            } else {
+                Value::Float(value.as_f64().unwrap_or(f64::NAN))
+            }
+        }
+        serde_json::Value::String(value) => Value::String(value),
+        serde_json::Value::Array(values) => {
+            Value::List(values.into_iter().map(from_json).collect())
+        }
+        serde_json::Value::Object(values) => Value::Dict(
+            values
+                .into_iter()
+                .map(|(key, value)| (Value::String(key), from_json(value)))
+                .collect(),
+        ),
+    }
+}
+
+pub(crate) fn from_json_text(value: &str) -> Result<Value, Error> {
+    serde_json::from_str(value)
+        .map(from_json)
+        .map_err(|_| Error::InvalidEntry)
+}
+
+pub(crate) fn is_truthy(value: &Value) -> bool {
+    match value {
+        Value::None => false,
+        Value::Boolean(value) => *value,
+        Value::Integer(value) => value.to_string() != "0",
+        Value::Float(value) => *value != 0.0,
+        Value::Complex(value) => value.re != 0.0 || value.im != 0.0,
+        Value::String(value) => !value.is_empty(),
+        Value::Bytes(value) => !value.is_empty(),
+        Value::Tuple(value) | Value::List(value) | Value::Set(value) => !value.is_empty(),
+        Value::Dict(value) => !value.is_empty(),
+    }
+}
+
+pub(crate) fn is_int(value: &Value) -> bool {
+    matches!(value, Value::Integer(_) | Value::Boolean(_))
+}
+
+pub(crate) fn to_f64(value: &Value) -> Option<f64> {
+    match value {
+        Value::Integer(value) => value.to_string().parse().ok(),
+        Value::Boolean(value) => Some(if *value { 1.0 } else { 0.0 }),
+        _ => None,
+    }
+}
+
+pub(crate) fn to_json(value: &Value) -> Result<Vec<u8>, Error> {
+    serde_json::to_vec(&to_json_value(value)?).map_err(|_| Error::InvalidEntry)
+}
+
+fn to_json_value(value: &Value) -> Result<serde_json::Value, Error> {
+    Ok(match value {
+        Value::None => serde_json::Value::Null,
+        Value::Boolean(value) => serde_json::Value::Bool(*value),
+        Value::Integer(value) => serde_json::Value::Number(
+            value
+                .to_string()
+                .parse::<Number>()
+                .map_err(|_| Error::InvalidEntry)?,
+        ),
+        Value::Float(value) => {
+            serde_json::Value::Number(Number::from_f64(*value).ok_or(Error::InvalidEntry)?)
+        }
+        Value::Complex(_) | Value::Bytes(_) => return Err(Error::InvalidEntry),
+        Value::String(value) => serde_json::Value::String(value.clone()),
+        Value::Tuple(values) | Value::List(values) | Value::Set(values) => {
+            serde_json::Value::Array(
+                values
+                    .iter()
+                    .map(to_json_value)
+                    .collect::<Result<Vec<_>, _>>()?,
+            )
+        }
+        Value::Dict(values) => {
+            let values = values
+                .iter()
+                .map(|(key, value)| {
+                    let Value::String(key) = key else {
+                        return Err(Error::InvalidEntry);
+                    };
+                    Ok((key.clone(), to_json_value(value)?))
+                })
+                .collect::<Result<Map<String, serde_json::Value>, _>>()?;
+            serde_json::Value::Object(values)
+        }
+    })
+}

litellm-rust/crates/cache-disk/src/sqlite.rs

@@ -0,0 +1,817 @@
+use std::{
+    collections::HashMap,
+    fs::{self, OpenOptions},
+    io::Write,
+    path::{Path, PathBuf},
+    sync::Mutex,
+};
+
+use litellm_cache::Error;
+use rand::RngCore;
+use rusqlite::{Connection, OptionalExtension, params, types::Value};
+
+use crate::{DiskStore, StoredValue};
+
+const MODE_RAW: i64 = 1;
+const MODE_BINARY: i64 = 2;
+const MODE_TEXT: i64 = 3;
+const MODE_PICKLE: i64 = 4;
+
+const DEFAULT_DISK_MIN_FILE_SIZE: i64 = 2_i64.pow(15);
+const DEFAULT_SIZE_LIMIT: i64 = 2_i64.pow(30);
+const DEFAULT_CULL_LIMIT: i64 = 10;
+
+pub struct DiskcacheSqliteStore {
+    directory: PathBuf,
+    connection: Mutex<Connection>,
+    min_file_size: usize,
+    eviction_policy: String,
+    size_limit: i64,
+    cull_limit: i64,
+    statistics: bool,
+}
+
+struct StoredColumns {
+    size: i64,
+    mode: i64,
+    filename: Option<String>,
+    value: Option<Value>,
+}
+
+struct Row {
+    rowid: i64,
+    mode: i64,
+    filename: Option<String>,
+    value: Value,
+}
+
+impl DiskcacheSqliteStore {
+    pub fn open(directory: impl AsRef<Path>) -> Result<Self, Error> {
+        let directory = directory.as_ref().to_path_buf();
+        fs::create_dir_all(&directory).map_err(|_| Error::Unavailable)?;
+        let directory = std::path::absolute(&directory).map_err(|_| Error::Unavailable)?;
+        let database = directory.join("cache.db");
+        let connection = Connection::open(database).map_err(|_| Error::Unavailable)?;
+        connection
+            .busy_timeout(std::time::Duration::from_secs(60))
+            .map_err(|_| Error::Unavailable)?;
+
+        let mut settings = read_settings(&connection)?;
+        for (key, value) in default_settings() {
+            settings.entry(key).or_insert(value);
+        }
+        for (key, value) in settings
+            .iter()
+            .filter(|(key, _)| key.starts_with("sqlite_"))
+        {
+            apply_pragma(&connection, key, value)?;
+        }
+
+        connection
+            .execute_batch(
+                "CREATE TABLE IF NOT EXISTS Settings (
+                    key TEXT NOT NULL UNIQUE,
+                    value
+                )",
+            )
+            .map_err(|_| Error::Unavailable)?;
+        for (key, value) in &settings {
+            if !matches!(key.as_str(), "count" | "size" | "hits" | "misses") {
+                connection
+                    .execute(
+                        "INSERT OR REPLACE INTO Settings VALUES (?, ?)",
+                        params![key, value],
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+        }
+        for (key, value) in [
+            ("count", Value::Integer(0)),
+            ("size", Value::Integer(0)),
+            ("hits", Value::Integer(0)),
+            ("misses", Value::Integer(0)),
+        ] {
+            connection
+                .execute(
+                    "INSERT OR IGNORE INTO Settings VALUES (?, ?)",
+                    params![key, value],
+                )
+                .map_err(|_| Error::Unavailable)?;
+        }
+        connection
+            .execute_batch(
+                "CREATE TABLE IF NOT EXISTS Cache (
+                    rowid INTEGER PRIMARY KEY,
+                    key BLOB,
+                    raw INTEGER,
+                    store_time REAL,
+                    expire_time REAL,
+                    access_time REAL,
+                    access_count INTEGER DEFAULT 0,
+                    tag BLOB,
+                    size INTEGER DEFAULT 0,
+                    mode INTEGER DEFAULT 0,
+                    filename TEXT,
+                    value BLOB
+                );
+                CREATE UNIQUE INDEX IF NOT EXISTS Cache_key_raw ON Cache(key, raw);
+                CREATE INDEX IF NOT EXISTS Cache_expire_time ON Cache(expire_time);",
+            )
+            .map_err(|_| Error::Unavailable)?;
+
+        let eviction_policy = setting_string(&settings, "eviction_policy")
+            .unwrap_or_else(|| "least-recently-stored".to_string());
+        match eviction_policy.as_str() {
+            "none" => {}
+            "least-recently-stored" => {
+                connection
+                    .execute_batch(
+                        "CREATE INDEX IF NOT EXISTS Cache_store_time ON Cache(store_time)",
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            "least-recently-used" => {
+                connection
+                    .execute_batch(
+                        "CREATE INDEX IF NOT EXISTS Cache_access_time ON Cache(access_time)",
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            "least-frequently-used" => {
+                connection
+                    .execute_batch(
+                        "CREATE INDEX IF NOT EXISTS Cache_access_count ON Cache(access_count)",
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            _ => return Err(Error::Unavailable),
+        }
+        connection
+            .execute_batch(
+                "CREATE TRIGGER IF NOT EXISTS Settings_count_insert
+                    AFTER INSERT ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings SET value = value + 1
+                    WHERE key = \"count\"; END;
+                 CREATE TRIGGER IF NOT EXISTS Settings_count_delete
+                    AFTER DELETE ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings SET value = value - 1
+                    WHERE key = \"count\"; END;
+                 CREATE TRIGGER IF NOT EXISTS Settings_size_insert
+                    AFTER INSERT ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings SET value = value + NEW.size
+                    WHERE key = \"size\"; END;
+                 CREATE TRIGGER IF NOT EXISTS Settings_size_update
+                    AFTER UPDATE ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings
+                    SET value = value + NEW.size - OLD.size
+                    WHERE key = \"size\"; END;
+                 CREATE TRIGGER IF NOT EXISTS Settings_size_delete
+                    AFTER DELETE ON Cache FOR EACH ROW BEGIN
+                    UPDATE Settings SET value = value - OLD.size
+                    WHERE key = \"size\"; END;",
+            )
+            .map_err(|_| Error::Unavailable)?;
+
+        let min_file_size = setting_i64(&settings, "disk_min_file_size")
+            .unwrap_or(DEFAULT_DISK_MIN_FILE_SIZE)
+            .try_into()
+            .map_err(|_| Error::Unavailable)?;
+        let size_limit = setting_i64(&settings, "size_limit").unwrap_or(DEFAULT_SIZE_LIMIT);
+        let cull_limit = setting_i64(&settings, "cull_limit").unwrap_or(DEFAULT_CULL_LIMIT);
+        let statistics = setting_i64(&settings, "statistics").unwrap_or_default() != 0;
+
+        Ok(Self {
+            directory,
+            connection: Mutex::new(connection),
+            min_file_size,
+            eviction_policy,
+            size_limit,
+            cull_limit,
+            statistics,
+        })
+    }
+
+    fn set_locked(
+        &self,
+        connection: &Connection,
+        key: &str,
+        columns: StoredColumns,
+        expire_time: Option<f64>,
+        now: f64,
+    ) -> Result<Vec<String>, Error> {
+        let mut cleanup = Vec::new();
+        if let Some(old_filename) = connection
+            .query_row(
+                "SELECT filename FROM Cache WHERE key = ? AND raw = 1",
+                params![key],
+                |row| row.get::<_, Option<String>>(0),
+            )
+            .optional()
+            .map_err(|_| Error::Unavailable)?
+            .flatten()
+        {
+            cleanup.push(old_filename);
+        }
+        let (size, mode, filename, value) =
+            (columns.size, columns.mode, columns.filename, columns.value);
+        let rowid = connection
+            .query_row(
+                "SELECT rowid FROM Cache WHERE key = ? AND raw = 1",
+                params![key],
+                |row| row.get::<_, i64>(0),
+            )
+            .optional()
+            .map_err(|_| Error::Unavailable)?;
+        if let Some(rowid) = rowid {
+            connection
+                .execute(
+                    "UPDATE Cache SET store_time = ?, expire_time = ?, access_time = ?,
+                        access_count = 0, tag = NULL, size = ?, mode = ?, filename = ?, value = ?
+                        WHERE rowid = ?",
+                    params![now, expire_time, now, size, mode, filename, value, rowid],
+                )
+                .map_err(|_| Error::Unavailable)?;
+        } else {
+            connection
+                .execute(
+                    "INSERT INTO Cache(
+                        key, raw, store_time, expire_time, access_time, access_count,
+                        tag, size, mode, filename, value
+                    ) VALUES (?, 1, ?, ?, ?, 0, NULL, ?, ?, ?, ?)",
+                    params![key, now, expire_time, now, size, mode, filename, value],
+                )
+                .map_err(|_| Error::Unavailable)?;
+        }
+        cleanup.extend(self.cull(connection, now)?);
+        Ok(cleanup)
+    }
+
+    fn cull(&self, connection: &Connection, now: f64) -> Result<Vec<String>, Error> {
+        if self.cull_limit <= 0 {
+            return Ok(Vec::new());
+        }
+        let mut cleanup = Vec::new();
+        let expired = connection
+            .prepare(
+                "SELECT rowid, filename FROM Cache
+                 WHERE expire_time IS NOT NULL AND expire_time < ?
+                 ORDER BY expire_time LIMIT ?",
+            )
+            .map_err(|_| Error::Unavailable)?
+            .query_map(params![now, self.cull_limit], |row| {
+                Ok((row.get::<_, i64>(0)?, row.get::<_, Option<String>>(1)?))
+            })
+            .map_err(|_| Error::Unavailable)?
+            .collect::<Result<Vec<_>, _>>()
+            .map_err(|_| Error::Unavailable)?;
+        for (_, filename) in &expired {
+            if let Some(filename) = filename {
+                cleanup.push(filename.clone());
+            }
+        }
+        for (rowid, _) in &expired {
+            connection
+                .execute("DELETE FROM Cache WHERE rowid = ?", params![rowid])
+                .map_err(|_| Error::Unavailable)?;
+        }
+        let remaining = self.cull_limit - i64::try_from(expired.len()).unwrap_or(self.cull_limit);
+        if remaining <= 0 || self.volume(connection)? < self.size_limit {
+            return Ok(cleanup);
+        }
+        let order = match self.eviction_policy.as_str() {
+            "none" => return Ok(cleanup),
+            "least-recently-stored" => "store_time",
+            "least-recently-used" => "access_time",
+            "least-frequently-used" => "access_count",
+            _ => return Err(Error::Unavailable),
+        };
+        let rows = connection
+            .prepare(&format!(
+                "SELECT rowid, filename FROM Cache ORDER BY {order} LIMIT ?"
+            ))
+            .map_err(|_| Error::Unavailable)?
+            .query_map(params![remaining], |row| {
+                Ok((row.get::<_, i64>(0)?, row.get::<_, Option<String>>(1)?))
+            })
+            .map_err(|_| Error::Unavailable)?
+            .collect::<Result<Vec<_>, _>>()
+            .map_err(|_| Error::Unavailable)?;
+        for (_, filename) in &rows {
+            if let Some(filename) = filename {
+                cleanup.push(filename.clone());
+            }
+        }
+        for (rowid, _) in rows {
+            connection
+                .execute("DELETE FROM Cache WHERE rowid = ?", params![rowid])
+                .map_err(|_| Error::Unavailable)?;
+        }
+        Ok(cleanup)
+    }
+
+    fn volume(&self, connection: &Connection) -> Result<i64, Error> {
+        let page_count: i64 = connection
+            .query_row("PRAGMA page_count", [], |row| row.get(0))
+            .map_err(|_| Error::Unavailable)?;
+        let page_size: i64 = connection
+            .query_row("PRAGMA page_size", [], |row| row.get(0))
+            .map_err(|_| Error::Unavailable)?;
+        let size: i64 = connection
+            .query_row("SELECT value FROM Settings WHERE key = 'size'", [], |row| {
+                row.get(0)
+            })
+            .map_err(|_| Error::Unavailable)?;
+        Ok(page_count.saturating_mul(page_size).saturating_add(size))
+    }
+}
+
+impl DiskStore for DiskcacheSqliteStore {
+    fn directory(&self) -> &Path {
+        &self.directory
+    }
+
+    fn get(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let select = "SELECT rowid, expire_time, mode, filename, value FROM Cache
+            WHERE key = ? AND raw = 1 AND (expire_time IS NULL OR expire_time > ?)";
+        let row = connection
+            .query_row(select, params![key, now], row_from_query)
+            .optional()
+            .map_err(|_| Error::Unavailable)?;
+        if !self.statistics && !has_get_update(&self.eviction_policy) {
+            return row
+                .map(|row| fetch_row(&self.directory, row))
+                .transpose()
+                .map(|value| value.flatten());
+        }
+        transactional(&connection, |connection| {
+            let row = connection
+                .query_row(select, params![key, now], row_from_query)
+                .optional()
+                .map_err(|_| Error::Unavailable)?;
+            let Some(row) = row else {
+                if self.statistics {
+                    connection
+                        .execute(
+                            "UPDATE Settings SET value = value + 1 WHERE key = 'misses'",
+                            [],
+                        )
+                        .map_err(|_| Error::Unavailable)?;
+                }
+                return Ok(None);
+            };
+            let rowid = row.rowid;
+            let value = fetch_row(&self.directory, row);
+            let hit = value.as_ref().is_ok_and(Option::is_some);
+            if hit && self.statistics {
+                connection
+                    .execute(
+                        "UPDATE Settings SET value = value + 1 WHERE key = 'hits'",
+                        [],
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            } else if !hit && self.statistics {
+                connection
+                    .execute(
+                        "UPDATE Settings SET value = value + 1 WHERE key = 'misses'",
+                        [],
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+            }
+            if has_get_update(&self.eviction_policy) && hit {
+                let update = match self.eviction_policy.as_str() {
+                    "least-recently-used" => "UPDATE Cache SET access_time = ? WHERE rowid = ?",
+                    "least-frequently-used" => {
+                        "UPDATE Cache SET access_count = access_count + 1 WHERE rowid = ?"
+                    }
+                    _ => return Err(Error::Unavailable),
+                };
+                if self.eviction_policy == "least-recently-used" {
+                    connection
+                        .execute(update, params![now, rowid])
+                        .map_err(|_| Error::Unavailable)?;
+                } else {
+                    connection
+                        .execute(update, params![rowid])
+                        .map_err(|_| Error::Unavailable)?;
+                }
+            }
+            value
+        })
+    }
+
+    fn set(
+        &self,
+        key: &str,
+        value: StoredValue,
+        expire_time: Option<f64>,
+        now: f64,
+    ) -> Result<(), Error> {
+        let columns = store_value(&self.directory, self.min_file_size, value)?;
+        let new_filename = columns.filename.clone();
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let result = transactional(&connection, |connection| {
+            self.set_locked(connection, key, columns, expire_time, now)
+        });
+        match result {
+            Ok(cleanup) => {
+                cleanup_files(&self.directory, cleanup);
+                Ok(())
+            }
+            Err(error) => {
+                if let Some(filename) = new_filename {
+                    remove_file(&self.directory, &filename);
+                }
+                Err(error)
+            }
+        }
+    }
+
+    fn pop(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let selected = transactional(&connection, |connection| {
+            let row = connection
+                .query_row(
+                    "SELECT rowid, expire_time, mode, filename, value FROM Cache
+                     WHERE key = ? AND raw = 1
+                     AND (expire_time IS NULL OR expire_time > ?)",
+                    params![key, now],
+                    row_from_query,
+                )
+                .optional()
+                .map_err(|_| Error::Unavailable)?;
+            let Some(row) = row else {
+                return Ok(None);
+            };
+            connection
+                .execute("DELETE FROM Cache WHERE rowid = ?", params![row.rowid])
+                .map_err(|_| Error::Unavailable)?;
+            Ok(Some(row))
+        })?;
+        let Some(row) = selected else {
+            return Ok(None);
+        };
+        let filename = row.filename.clone();
+        let result = fetch_row(&self.directory, row)?;
+        if let Some(filename) = filename {
+            remove_file(&self.directory, &filename);
+        }
+        Ok(result)
+    }
+
+    fn clear(&self) -> Result<(), Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let mut last_rowid = 0_i64;
+        loop {
+            let batch = transactional(&connection, |connection| {
+                let rows = connection
+                    .prepare(
+                        "SELECT rowid, filename FROM Cache
+                         WHERE rowid > ? ORDER BY rowid LIMIT 100",
+                    )
+                    .map_err(|_| Error::Unavailable)?
+                    .query_map(params![last_rowid], |row| {
+                        Ok((row.get::<_, i64>(0)?, row.get::<_, Option<String>>(1)?))
+                    })
+                    .map_err(|_| Error::Unavailable)?
+                    .collect::<Result<Vec<_>, _>>()
+                    .map_err(|_| Error::Unavailable)?;
+                if rows.is_empty() {
+                    return Ok(rows);
+                }
+                let ids = rows
+                    .iter()
+                    .map(|(rowid, _)| rowid.to_string())
+                    .collect::<Vec<_>>()
+                    .join(",");
+                connection
+                    .execute(&format!("DELETE FROM Cache WHERE rowid IN ({ids})"), [])
+                    .map_err(|_| Error::Unavailable)?;
+                Ok(rows)
+            })?;
+            if batch.is_empty() {
+                return Ok(());
+            }
+            last_rowid = batch.last().map(|(rowid, _)| *rowid).unwrap_or(last_rowid);
+            cleanup_files(
+                &self.directory,
+                batch
+                    .into_iter()
+                    .filter_map(|(_, filename)| filename)
+                    .collect(),
+            );
+        }
+    }
+
+    fn update(
+        &self,
+        key: &str,
+        now: f64,
+        apply: &mut dyn FnMut(Option<StoredValue>) -> Result<(StoredValue, Option<f64>), Error>,
+    ) -> Result<(), Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        let mut created_filename = None;
+        let result = transactional(&connection, |connection| {
+            let current = connection
+                .query_row(
+                    "SELECT rowid, expire_time, mode, filename, value FROM Cache
+                     WHERE key = ? AND raw = 1
+                     AND (expire_time IS NULL OR expire_time > ?)",
+                    params![key, now],
+                    row_from_query,
+                )
+                .optional()
+                .map_err(|_| Error::Unavailable)?
+                .map(|row| fetch_row(&self.directory, row))
+                .transpose()?
+                .flatten();
+            let (value, expire_time) = apply(current)?;
+            let columns = store_value(&self.directory, self.min_file_size, value)?;
+            created_filename = columns.filename.clone();
+            let cleanup = self.set_locked(connection, key, columns, expire_time, now)?;
+            Ok(cleanup)
+        });
+        match result {
+            Ok(cleanup) => {
+                cleanup_files(&self.directory, cleanup);
+                Ok(())
+            }
+            Err(error) => {
+                if let Some(filename) = created_filename {
+                    remove_file(&self.directory, &filename);
+                }
+                Err(error)
+            }
+        }
+    }
+
+    fn probe(&self) -> Result<(), Error> {
+        let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
+        connection
+            .query_row(
+                "SELECT value FROM Settings WHERE key = 'count'",
+                [],
+                |row| row.get::<_, i64>(0),
+            )
+            .map(|_| ())
+            .map_err(|_| Error::Unavailable)
+    }
+}
+
+fn default_settings() -> HashMap<String, Value> {
+    HashMap::from([
+        ("statistics".to_string(), Value::Integer(0)),
+        ("tag_index".to_string(), Value::Integer(0)),
+        (
+            "eviction_policy".to_string(),
+            Value::Text("least-recently-stored".to_string()),
+        ),
+        ("size_limit".to_string(), Value::Integer(DEFAULT_SIZE_LIMIT)),
+        ("cull_limit".to_string(), Value::Integer(DEFAULT_CULL_LIMIT)),
+        ("sqlite_auto_vacuum".to_string(), Value::Integer(1)),
+        ("sqlite_cache_size".to_string(), Value::Integer(8192)),
+        (
+            "sqlite_journal_mode".to_string(),
+            Value::Text("wal".to_string()),
+        ),
+        (
+            "sqlite_mmap_size".to_string(),
+            Value::Integer(2_i64.pow(26)),
+        ),
+        ("sqlite_synchronous".to_string(), Value::Integer(1)),
+        (
+            "disk_min_file_size".to_string(),
+            Value::Integer(DEFAULT_DISK_MIN_FILE_SIZE),
+        ),
+        ("disk_pickle_protocol".to_string(), Value::Integer(5)),
+    ])
+}
+
+fn read_settings(connection: &Connection) -> Result<HashMap<String, Value>, Error> {
+    let mut statement = match connection.prepare("SELECT key, value FROM Settings") {
+        Ok(statement) => statement,
+        Err(_) => return Ok(HashMap::new()),
+    };
+    statement
+        .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
+        .map_err(|_| Error::Unavailable)?
+        .collect::<Result<HashMap<_, _>, _>>()
+        .map_err(|_| Error::Unavailable)
+}
+
+fn apply_pragma(connection: &Connection, key: &str, value: &Value) -> Result<(), Error> {
+    let pragma = key.strip_prefix("sqlite_").ok_or(Error::Unavailable)?;
+    match value {
+        Value::Integer(value) => connection
+            .pragma_update(None, pragma, value)
+            .map_err(|_| Error::Unavailable),
+        Value::Text(value) => connection
+            .pragma_update(None, pragma, value)
+            .map_err(|_| Error::Unavailable),
+        _ => Err(Error::Unavailable),
+    }
+}
+
+fn setting_i64(settings: &HashMap<String, Value>, key: &str) -> Option<i64> {
+    match settings.get(key) {
+        Some(Value::Integer(value)) => Some(*value),
+        _ => None,
+    }
+}
+
+fn setting_string(settings: &HashMap<String, Value>, key: &str) -> Option<String> {
+    match settings.get(key) {
+        Some(Value::Text(value)) => Some(value.clone()),
+        _ => None,
+    }
+}
+
+fn has_get_update(policy: &str) -> bool {
+    matches!(policy, "least-recently-used" | "least-frequently-used")
+}
+
+fn row_from_query(row: &rusqlite::Row<'_>) -> rusqlite::Result<Row> {
+    Ok(Row {
+        rowid: row.get(0)?,
+        mode: row.get(2)?,
+        filename: row.get(3)?,
+        value: row.get(4)?,
+    })
+}
+
+fn fetch_row(directory: &Path, row: Row) -> Result<Option<StoredValue>, Error> {
+    match row.mode {
+        MODE_RAW => match row.value {
+            Value::Blob(value) => Ok(Some(StoredValue::Bytes(value))),
+            Value::Text(value) => Ok(Some(StoredValue::Text(value))),
+            Value::Integer(value) => Ok(Some(StoredValue::Integer(value))),
+            Value::Real(value) => Ok(Some(StoredValue::Float(value))),
+            Value::Null => Err(Error::InvalidEntry),
+        },
+        MODE_BINARY | MODE_PICKLE => {
+            let bytes = match row.value {
+                Value::Blob(value) => value,
+                Value::Null => {
+                    let Some(value) = read_file(directory, row.filename.as_deref())? else {
+                        return Ok(None);
+                    };
+                    value
+                }
+                _ => return Err(Error::InvalidEntry),
+            };
+            Ok(Some(if row.mode == MODE_BINARY {
+                StoredValue::Bytes(bytes)
+            } else {
+                StoredValue::Pickle(bytes)
+            }))
+        }
+        MODE_TEXT => {
+            let bytes = match row.value {
+                Value::Null => {
+                    let Some(value) = read_file(directory, row.filename.as_deref())? else {
+                        return Ok(None);
+                    };
+                    value
+                }
+                Value::Blob(value) => value,
+                Value::Text(value) => value.into_bytes(),
+                _ => return Err(Error::InvalidEntry),
+            };
+            Ok(Some(StoredValue::Text(
+                String::from_utf8(bytes).map_err(|_| Error::InvalidEntry)?,
+            )))
+        }
+        _ => Err(Error::InvalidEntry),
+    }
+}
+
+fn read_file(directory: &Path, filename: Option<&str>) -> Result<Option<Vec<u8>>, Error> {
+    let Some(filename) = filename else {
+        return Err(Error::InvalidEntry);
+    };
+    match fs::read(directory.join(filename)) {
+        Ok(value) => Ok(Some(value)),
+        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
+        Err(_) => Err(Error::Unavailable),
+    }
+}
+
+fn store_value(
+    directory: &Path,
+    min_file_size: usize,
+    value: StoredValue,
+) -> Result<StoredColumns, Error> {
+    match value {
+        StoredValue::Integer(value) => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_RAW,
+            filename: None,
+            value: Some(Value::Integer(value)),
+        }),
+        StoredValue::Float(value) => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_RAW,
+            filename: None,
+            value: Some(Value::Real(value)),
+        }),
+        StoredValue::Text(value) if value.chars().count() < min_file_size => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_RAW,
+            filename: None,
+            value: Some(Value::Text(value)),
+        }),
+        StoredValue::Text(value) => {
+            let bytes = value.into_bytes();
+            let filename = write_file(directory, &bytes)?;
+            Ok(StoredColumns {
+                size: i64::try_from(bytes.len()).map_err(|_| Error::Unavailable)?,
+                mode: MODE_TEXT,
+                filename: Some(filename),
+                value: None,
+            })
+        }
+        StoredValue::Bytes(value) if value.len() < min_file_size => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_RAW,
+            filename: None,
+            value: Some(Value::Blob(value)),
+        }),
+        StoredValue::Bytes(value) => {
+            let filename = write_file(directory, &value)?;
+            Ok(StoredColumns {
+                size: i64::try_from(value.len()).map_err(|_| Error::Unavailable)?,
+                mode: MODE_BINARY,
+                filename: Some(filename),
+                value: None,
+            })
+        }
+        StoredValue::Pickle(value) if value.len() < min_file_size => Ok(StoredColumns {
+            size: 0,
+            mode: MODE_PICKLE,
+            filename: None,
+            value: Some(Value::Blob(value)),
+        }),
+        StoredValue::Pickle(value) => {
+            let filename = write_file(directory, &value)?;
+            Ok(StoredColumns {
+                size: i64::try_from(value.len()).map_err(|_| Error::Unavailable)?,
+                mode: MODE_PICKLE,
+                filename: Some(filename),
+                value: None,
+            })
+        }
+    }
+}
+
+fn write_file(directory: &Path, bytes: &[u8]) -> Result<String, Error> {
+    let mut random = [0_u8; 16];
+    rand::rngs::OsRng.fill_bytes(&mut random);
+    let hex = random
+        .iter()
+        .map(|byte| format!("{byte:02x}"))
+        .collect::<String>();
+    let filename = format!("{}/{}/{}.val", &hex[..2], &hex[2..4], &hex[4..]);
+    let path = directory.join(&filename);
+    if let Some(parent) = path.parent() {
+        fs::create_dir_all(parent).map_err(|_| Error::Unavailable)?;
+    }
+    let mut file = OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(path)
+        .map_err(|_| Error::Unavailable)?;
+    file.write_all(bytes).map_err(|_| Error::Unavailable)?;
+    Ok(filename)
+}
+
+fn cleanup_files(directory: &Path, filenames: Vec<String>) {
+    for filename in filenames {
+        remove_file(directory, &filename);
+    }
+}
+
+fn remove_file(directory: &Path, filename: &str) {
+    let path = directory.join(filename);
+    let _ = fs::remove_file(&path);
+}
+
+fn transactional<T>(
+    connection: &Connection,
+    operation: impl FnOnce(&Connection) -> Result<T, Error>,
+) -> Result<T, Error> {
+    connection
+        .execute_batch("BEGIN IMMEDIATE")
+        .map_err(|_| Error::Unavailable)?;
+    match operation(connection) {
+        Ok(value) => {
+            connection
+                .execute_batch("COMMIT")
+                .map_err(|_| Error::Unavailable)?;
+            Ok(value)
+        }
+        Err(error) => {
+            let _ = connection.execute_batch("ROLLBACK");
+            Err(error)
+        }
+    }
+}

litellm-rust/crates/cache-disk/src/store.rs

@@ -0,0 +1,33 @@
+use std::path::Path;
+
+use litellm_cache::Error;
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum StoredValue {
+    Bytes(Vec<u8>),
+    Text(String),
+    Integer(i64),
+    Float(f64),
+    Pickle(Vec<u8>),
+}
+
+pub trait DiskStore: Send + Sync + 'static {
+    fn directory(&self) -> &Path;
+    fn get(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error>;
+    fn set(
+        &self,
+        key: &str,
+        value: StoredValue,
+        expire_time: Option<f64>,
+        now: f64,
+    ) -> Result<(), Error>;
+    fn pop(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error>;
+    fn clear(&self) -> Result<(), Error>;
+    fn update(
+        &self,
+        key: &str,
+        now: f64,
+        apply: &mut dyn FnMut(Option<StoredValue>) -> Result<(StoredValue, Option<f64>), Error>,
+    ) -> Result<(), Error>;
+    fn probe(&self) -> Result<(), Error>;
+}

litellm-rust/crates/cache-disk/tests/cache.rs

@@ -0,0 +1,431 @@
+use std::{
+    fs,
+    path::{Path, PathBuf},
+    sync::Arc,
+    thread,
+    time::Duration,
+};
+
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheCodec, CounterCache, DeleteCache, ExactCacheContext,
+    FlushCache, JsonCodec,
+};
+use litellm_cache_disk::{DiskCache, DiskStore, DiskcacheSqliteStore, StoredValue, ValueAdapter};
+use rstest::{fixture, rstest};
+use rusqlite::Connection;
+use serde_json::{Value, json};
+use tempfile::TempDir;
+
+struct Sandbox {
+    directory: TempDir,
+}
+
+#[fixture]
+fn sandbox() -> Sandbox {
+    Sandbox {
+        directory: tempfile::tempdir().unwrap(),
+    }
+}
+
+impl Sandbox {
+    fn store(&self) -> DiskcacheSqliteStore {
+        DiskcacheSqliteStore::open(self.directory.path()).unwrap()
+    }
+
+    fn cache<V>(&self) -> DiskCache<JsonCodec<V>>
+    where
+        JsonCodec<V>: CacheCodec,
+    {
+        DiskCache::open(self.directory.path(), JsonCodec::new()).unwrap()
+    }
+
+    fn db(&self) -> Connection {
+        Connection::open(self.directory.path().join("cache.db")).unwrap()
+    }
+
+    fn value_files(&self) -> Vec<PathBuf> {
+        fn visit(directory: &Path, files: &mut Vec<PathBuf>) {
+            for entry in fs::read_dir(directory).unwrap() {
+                let path = entry.unwrap().path();
+                if path.is_dir() {
+                    visit(&path, files);
+                } else if path.extension().is_some_and(|extension| extension == "val") {
+                    files.push(path);
+                }
+            }
+        }
+
+        let mut files = Vec::new();
+        visit(self.directory.path(), &mut files);
+        files
+    }
+}
+
+#[rstest]
+fn relative_store_directory_is_absolutized(sandbox: Sandbox) {
+    let relative = PathBuf::from(format!(
+        ".litellm-cache-disk-{}",
+        sandbox
+            .directory
+            .path()
+            .file_name()
+            .unwrap()
+            .to_string_lossy()
+    ));
+    let store = DiskcacheSqliteStore::open(&relative).unwrap();
+    assert!(store.directory().is_absolute());
+    assert!(store.directory().ends_with(&relative));
+    let directory = store.directory().to_path_buf();
+    drop(store);
+    fs::remove_dir_all(directory).unwrap();
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+struct TextAdapter;
+
+impl ValueAdapter for TextAdapter {
+    fn read(&self, value: StoredValue) -> Result<Option<Vec<u8>>, litellm_cache::Error> {
+        match value {
+            StoredValue::Text(value) => Ok(Some(value.into_bytes())),
+            _ => Ok(None),
+        }
+    }
+
+    fn write(&self, payload: Vec<u8>) -> StoredValue {
+        StoredValue::Text(String::from_utf8(payload).unwrap())
+    }
+
+    fn counter_seed(&self, _: Option<StoredValue>) -> Result<f64, litellm_cache::Error> {
+        Ok(0.0)
+    }
+
+    fn counter_value(&self, value: f64) -> StoredValue {
+        if value.fract() == 0.0 {
+            StoredValue::Integer(value as i64)
+        } else {
+            StoredValue::Float(value)
+        }
+    }
+}
+
+#[rstest]
+fn roundtrip_persists_and_reopens(sandbox: Sandbox) {
+    let context = ExactCacheContext::default();
+    let opened = sandbox.cache::<Value>();
+    opened
+        .set_cache("key", json!({"answer": 42}), &context)
+        .unwrap();
+    assert_eq!(
+        opened.get_cache("key", &context).unwrap(),
+        Some(json!({"answer": 42}))
+    );
+    drop(opened);
+    let reopened = sandbox.cache::<Value>();
+    assert_eq!(
+        reopened.get_cache("key", &context).unwrap(),
+        Some(json!({"answer": 42}))
+    );
+}
+
+#[rstest]
+fn ttl_and_expired_culling_match_cache_contract(sandbox: Sandbox) {
+    let store = sandbox.store();
+    store
+        .set(
+            "expired",
+            StoredValue::Bytes(b"old".to_vec()),
+            Some(10.0),
+            0.0,
+        )
+        .unwrap();
+    assert_eq!(store.get("expired", 10.0).unwrap(), None);
+    store
+        .set("new", StoredValue::Bytes(b"new".to_vec()), None, 11.0)
+        .unwrap();
+    assert_eq!(
+        sandbox
+            .db()
+            .query_row("SELECT COUNT(*) FROM Cache", [], |row| row.get::<_, i64>(0))
+            .unwrap(),
+        1
+    );
+    assert_eq!(
+        sandbox
+            .db()
+            .query_row(
+                "SELECT value FROM Settings WHERE key = 'count'",
+                [],
+                |row| row.get::<_, i64>(0)
+            )
+            .unwrap(),
+        1
+    );
+}
+
+#[rstest]
+fn batch_preserves_order_and_classifies_misses_and_invalid_values(sandbox: Sandbox) {
+    let store = sandbox.store();
+    store
+        .set(
+            "hit",
+            StoredValue::Bytes(br#"{"ok":true}"#.to_vec()),
+            None,
+            0.0,
+        )
+        .unwrap();
+    store
+        .set(
+            "invalid",
+            StoredValue::Pickle(vec![0x80, 0x05, 0x2e]),
+            None,
+            0.0,
+        )
+        .unwrap();
+    let entries = sandbox
+        .cache::<Value>()
+        .batch_get_cache(
+            &["hit".into(), "missing".into(), "invalid".into()],
+            &ExactCacheContext::default(),
+        )
+        .unwrap();
+    assert_eq!(
+        entries,
+        vec![
+            BatchEntry::Hit(json!({"ok": true})),
+            BatchEntry::Miss,
+            BatchEntry::Invalid
+        ]
+    );
+}
+
+#[rstest]
+#[case(StoredValue::Bytes(Vec::new()))]
+#[case(StoredValue::Text(String::new()))]
+#[case(StoredValue::Integer(0))]
+#[case(StoredValue::Float(0.0))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x89, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x00, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x94, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x5d, 0x94, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x7d, 0x94, 0x2e]))]
+#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x29, 0x2e]))]
+fn falsy_values_are_misses(sandbox: Sandbox, #[case] value: StoredValue) {
+    sandbox.store().set("key", value, None, 0.0).unwrap();
+    assert_eq!(
+        sandbox
+            .cache::<Value>()
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap(),
+        None
+    );
+}
+
+#[rstest]
+#[case(Some(StoredValue::Integer(2)), 1.5, 3.5, "real")]
+#[case(Some(StoredValue::Integer(2)), 1.0, 3.0, "integer")]
+#[case(Some(StoredValue::Float(3.5)), 1.0, 1.0, "integer")]
+#[case(Some(StoredValue::Text("not a number".into())), 2.0, 2.0, "integer")]
+#[case(Some(StoredValue::Text("5".into())), 2.0, 7.0, "integer")]
+#[case(Some(StoredValue::Text("3.5".into())), 2.0, 2.0, "integer")]
+#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e])), 1.0, 2.0, "integer")]
+#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x02, 0x2e])), 1.0, 3.0, "integer")]
+#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e])), 1.0, 1.0, "integer")]
+#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e])), 4.0, 4.0, "integer")]
+fn counters_follow_python_initialization(
+    sandbox: Sandbox,
+    #[case] initial: Option<StoredValue>,
+    #[case] amount: f64,
+    #[case] expected: f64,
+    #[case] sqlite_type: &str,
+) {
+    if let Some(initial) = initial {
+        sandbox.store().set("counter", initial, None, 0.0).unwrap();
+    }
+    let cache = sandbox.cache::<f64>();
+    assert_eq!(
+        cache
+            .increment_cache("counter", amount, ExactCacheContext::default())
+            .unwrap(),
+        expected
+    );
+    assert_eq!(
+        sandbox
+            .db()
+            .query_row(
+                "SELECT typeof(value) FROM Cache WHERE key = 'counter'",
+                [],
+                |row| row.get::<_, String>(0)
+            )
+            .unwrap(),
+        sqlite_type
+    );
+}
+
+#[rstest]
+fn counters_are_atomic_across_concurrent_callers(sandbox: Sandbox) {
+    let cache = Arc::new(sandbox.cache::<f64>());
+    let workers = (0..8)
+        .map(|_| {
+            let cache = Arc::clone(&cache);
+            thread::spawn(move || {
+                for _ in 0..25 {
+                    cache
+                        .increment_cache("counter", 1.0, ExactCacheContext::default())
+                        .unwrap();
+                }
+            })
+        })
+        .collect::<Vec<_>>();
+    for worker in workers {
+        worker.join().unwrap();
+    }
+    assert_eq!(
+        cache
+            .increment_cache("counter", 0.0, ExactCacheContext::default())
+            .unwrap(),
+        200.0
+    );
+}
+
+#[rstest]
+fn fractional_then_integer_increment_follows_python_behavior(sandbox: Sandbox) {
+    let cache = sandbox.cache::<f64>();
+    assert_eq!(
+        cache
+            .increment_cache("counter", 3.5, ExactCacheContext::default())
+            .unwrap(),
+        3.5
+    );
+    assert_eq!(
+        cache
+            .increment_cache("counter", 1.0, ExactCacheContext::default())
+            .unwrap(),
+        1.0
+    );
+}
+
+#[rstest]
+fn increment_ttl_replacement_clears_expiry_without_ttl(sandbox: Sandbox) {
+    let cache = sandbox.cache::<f64>();
+    cache
+        .increment_cache(
+            "counter",
+            1.0,
+            ExactCacheContext {
+                ttl: Some(Duration::from_secs(60)),
+            },
+        )
+        .unwrap();
+    assert!(
+        sandbox
+            .db()
+            .query_row(
+                "SELECT expire_time IS NOT NULL FROM Cache WHERE key = 'counter'",
+                [],
+                |row| row.get::<_, bool>(0)
+            )
+            .unwrap()
+    );
+    cache
+        .increment_cache("counter", 1.0, ExactCacheContext::default())
+        .unwrap();
+    assert!(
+        !sandbox
+            .db()
+            .query_row(
+                "SELECT expire_time IS NOT NULL FROM Cache WHERE key = 'counter'",
+                [],
+                |row| row.get::<_, bool>(0)
+            )
+            .unwrap()
+    );
+}
+
+#[rstest]
+fn custom_adapter_controls_storage_and_reads(sandbox: Sandbox) {
+    let cache = DiskCache::with_adapter(sandbox.store(), TextAdapter, JsonCodec::<Value>::new());
+    cache
+        .set_cache("key", json!({"answer": 42}), &ExactCacheContext::default())
+        .unwrap();
+    assert!(matches!(
+        sandbox.store().get("key", 0.0).unwrap(),
+        Some(StoredValue::Text(_))
+    ));
+    assert_eq!(
+        cache
+            .get_cache("key", &ExactCacheContext::default())
+            .unwrap(),
+        Some(json!({"answer": 42}))
+    );
+}
+
+#[rstest]
+fn delete_flush_and_spilled_file_replacement_clean_up_storage(sandbox: Sandbox) {
+    let large = vec![b'x'; 32 * 1024];
+    sandbox
+        .store()
+        .set("large", StoredValue::Bytes(large.clone()), None, 0.0)
+        .unwrap();
+    assert_eq!(sandbox.value_files().len(), 1);
+    sandbox
+        .store()
+        .set(
+            "large",
+            StoredValue::Bytes(vec![b'y'; 32 * 1024]),
+            None,
+            0.0,
+        )
+        .unwrap();
+    assert_eq!(sandbox.value_files().len(), 1);
+    sandbox.store().pop("large", 0.0).unwrap();
+    assert!(sandbox.value_files().is_empty());
+    sandbox
+        .store()
+        .set("a", StoredValue::Bytes(large.clone()), None, 0.0)
+        .unwrap();
+    sandbox
+        .store()
+        .set("b", StoredValue::Bytes(large), None, 0.0)
+        .unwrap();
+    sandbox.store().clear().unwrap();
+    assert!(sandbox.value_files().is_empty());
+}
+
+#[rstest]
+#[tokio::test]
+async fn async_operations_connection_and_delete_match_sync_operations(sandbox: Sandbox) {
+    let cache = sandbox.cache::<Value>();
+    let context = ExactCacheContext {
+        ttl: Some(Duration::from_secs(60)),
+    };
+    cache
+        .async_set_cache("a", json!(1), context.clone())
+        .await
+        .unwrap();
+    cache
+        .async_set_cache_pipeline(
+            vec![("b".into(), json!(2)), ("c".into(), json!(3))],
+            context.clone(),
+        )
+        .await
+        .unwrap();
+    assert_eq!(
+        cache.async_get_cache("a", &context).await.unwrap(),
+        Some(json!(1))
+    );
+    assert_eq!(
+        cache
+            .async_batch_get_cache(vec!["c".into(), "missing".into()], context.clone())
+            .await
+            .unwrap(),
+        vec![BatchEntry::Hit(json!(3)), BatchEntry::Miss]
+    );
+    cache.async_delete_cache("a").await.unwrap();
+    cache.async_flush_cache().await.unwrap();
+    assert_eq!(
+        cache.test_connection().await.unwrap().status,
+        litellm_cache::CacheConnectionStatus::Success
+    );
+}

litellm-rust/crates/cache-disk/tests/python_compat.rs

@@ -0,0 +1,113 @@
+use litellm_cache::Error;
+use litellm_cache_disk::{PythonDiskCacheAdapter, StoredValue, ValueAdapter};
+use rstest::rstest;
+
+enum ReadExpectation {
+    Bytes(&'static [u8]),
+    Miss,
+    Invalid,
+}
+
+#[rstest]
+#[case::pickled_dictionary_with_string_keys(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e]),
+    ReadExpectation::Bytes(br#"{"a":1}"#)
+)]
+#[case::pickled_list_of_integers(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x94, 0x28, 0x4b, 0x01, 0x4b, 0x02, 0x65, 0x2e]),
+    ReadExpectation::Bytes(br#"[1,2]"#)
+)]
+#[case::pickled_tuple_of_integers(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x01, 0x4b, 0x02, 0x86, 0x94, 0x2e]),
+    ReadExpectation::Bytes(br#"[1,2]"#)
+)]
+#[case::pickled_set_of_integers(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x94, 0x28, 0x4b, 0x01, 0x4b, 0x02, 0x90, 0x2e]),
+    ReadExpectation::Bytes(br#"[1,2]"#)
+)]
+#[case::pickled_response_envelope(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x28, 0x8c, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x94, 0x47, 0x3f, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x94, 0x8c, 0x08, 0x7b, 0x22, 0x61, 0x22, 0x3a, 0x20, 0x31, 0x7d, 0x94, 0x75, 0x2e]),
+    ReadExpectation::Bytes(br#"{"response":"{\"a\": 1}","timestamp":1.5}"#)
+)]
+#[case::non_json_text(
+    StoredValue::Text("not json".into()),
+    ReadExpectation::Bytes(b"not json")
+)]
+#[case::json_text(
+    StoredValue::Text("{\"a\": 1}".into()),
+    ReadExpectation::Bytes(br#"{"a": 1}"#)
+)]
+#[case::non_utf8_bytes(
+    StoredValue::Bytes(vec![0xff, 0xfe]),
+    ReadExpectation::Bytes(&[0xff, 0xfe])
+)]
+#[case::integer_seven(StoredValue::Integer(7), ReadExpectation::Bytes(b"7"))]
+#[case::float_one_point_five(StoredValue::Float(1.5), ReadExpectation::Bytes(b"1.5"))]
+#[case::pickled_true(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e]),
+    ReadExpectation::Bytes(b"true")
+)]
+#[case::pickled_negative_integer(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0xfd, 0xff, 0xff, 0xff, 0x2e]),
+    ReadExpectation::Bytes(b"-3")
+)]
+#[case::pickled_bytes(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x05, 0x62, 0x79, 0x74, 0x65, 0x73, 0x94, 0x2e]),
+    ReadExpectation::Invalid
+)]
+#[case::pickled_dictionary_with_integer_key(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x4b, 0x01, 0x8c, 0x01, 0x61, 0x94, 0x73, 0x2e]),
+    ReadExpectation::Invalid
+)]
+#[case::pickled_complex(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x2e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x08, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x73, 0x94, 0x8c, 0x07, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x94, 0x93, 0x94, 0x47, 0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0x94, 0x52, 0x94, 0x2e]),
+    ReadExpectation::Invalid
+)]
+#[case::truncated_pickle(
+    StoredValue::Pickle(vec![0x80, 0x05, 0x2e]),
+    ReadExpectation::Invalid
+)]
+#[case::empty_bytes(StoredValue::Bytes(Vec::new()), ReadExpectation::Miss)]
+#[case::empty_text(StoredValue::Text(String::new()), ReadExpectation::Miss)]
+#[case::zero_integer(StoredValue::Integer(0), ReadExpectation::Miss)]
+#[case::zero_float(StoredValue::Float(0.0), ReadExpectation::Miss)]
+#[case::pickled_none(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_false(StoredValue::Pickle(vec![0x80, 0x05, 0x89, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_zero(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x00, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_zero_float(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_empty_string(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x94, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_empty_list(StoredValue::Pickle(vec![0x80, 0x05, 0x5d, 0x94, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_empty_dictionary(StoredValue::Pickle(vec![0x80, 0x05, 0x7d, 0x94, 0x2e]), ReadExpectation::Miss)]
+#[case::pickled_empty_tuple(StoredValue::Pickle(vec![0x80, 0x05, 0x29, 0x2e]), ReadExpectation::Miss)]
+fn python_read_cases(#[case] row: StoredValue, #[case] expected: ReadExpectation) {
+    let result = PythonDiskCacheAdapter.read(row);
+    match expected {
+        ReadExpectation::Bytes(expected) => assert_eq!(result.unwrap().unwrap(), expected),
+        ReadExpectation::Miss => assert_eq!(result.unwrap(), None),
+        ReadExpectation::Invalid => assert!(matches!(result, Err(Error::InvalidEntry))),
+    }
+}
+
+#[rstest]
+#[case::integer_two(Some(StoredValue::Integer(2)), 2.0)]
+#[case::float_three_point_five(Some(StoredValue::Float(3.5)), 0.0)]
+#[case::text_not_a_number(Some(StoredValue::Text("not a number".into())), 0.0)]
+#[case::text_five(Some(StoredValue::Text("5".into())), 5.0)]
+#[case::text_three_point_five(Some(StoredValue::Text("3.5".into())), 0.0)]
+#[case::pickled_true(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e])), 1.0)]
+#[case::pickled_two(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x02, 0x2e])), 2.0)]
+#[case::pickled_dictionary(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e])), 0.0)]
+#[case::missing(None, 0.0)]
+#[case::pickled_none(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e])), 0.0)]
+fn python_counter_seed_cases(#[case] row: Option<StoredValue>, #[case] expected: f64) {
+    assert_eq!(PythonDiskCacheAdapter.counter_seed(row).unwrap(), expected);
+}
+
+#[rstest]
+#[case::integer_three(3.0, StoredValue::Integer(3))]
+#[case::fractional_three_point_five(3.5, StoredValue::Float(3.5))]
+#[case::negative_zero(-0.0, StoredValue::Integer(0))]
+#[case::large_float(1e300, StoredValue::Float(1e300))]
+fn python_counter_value_cases(#[case] value: f64, #[case] expected: StoredValue) {
+    assert_eq!(PythonDiskCacheAdapter.counter_value(value), expected);
+}

litellm-rust/crates/cache-redis/Cargo.toml

@@ -7,7 +7,7 @@ repository.workspace = true
 
 [dependencies]
 litellm-cache.workspace = true
-redis = { version = "1.7.0", features = ["tls-rustls"] }
+redis = { version = "1.7.0", features = ["cluster", "tls-rustls"] }
 r2d2 = "0.8.10"
 tokio.workspace = true
 

litellm-rust/crates/cache-redis/src/cache.rs

@@ -9,8 +9,14 @@ use litellm_cache::{
 };
 use redis::Commands;
 
+use crate::topology::RedisTopology;
+
+mod connection;
 mod operations;
 
+pub(crate) use connection::ConnectionRef;
+use connection::{ClusterConnectionManager, ConnectionManager};
+
 pub use operations::{
     RedisArg, RedisLpopOperation, RedisLpopResult, RedisRpushOperation, RedisScript,
 };
@@ -19,40 +25,6 @@ const DEFAULT_TTL: Duration = Duration::from_secs(600);
 const REDIS_TIMEOUT: Duration = Duration::from_secs(5);
 const REDIS_POOL_SIZE: u32 = 16;
 
-struct PooledConnection {
-    connection: redis::Connection,
-    failed: bool,
-}
-
-/// Pools connections without a checkout PING, which would double every operation's round trips.
-/// A timed-out command leaves its reply on the socket while redis still reports the connection
-/// open, so any connection whose operation failed is discarded instead of being reused.
-struct ConnectionManager(redis::Client);
-
-impl r2d2::ManageConnection for ConnectionManager {
-    type Connection = PooledConnection;
-    type Error = redis::RedisError;
-
-    fn connect(&self) -> Result<PooledConnection, redis::RedisError> {
-        let connection = self.0.get_connection()?;
-        connection.set_read_timeout(Some(REDIS_TIMEOUT))?;
-        connection.set_write_timeout(Some(REDIS_TIMEOUT))?;
-        Ok(PooledConnection {
-            connection,
-            failed: false,
-        })
-    }
-
-    fn is_valid(&self, connection: &mut PooledConnection) -> Result<(), redis::RedisError> {
-        redis::cmd("PING").query::<String>(&mut connection.connection)?;
-        Ok(())
-    }
-
-    fn has_broken(&self, connection: &mut PooledConnection) -> bool {
-        connection.failed || !redis::ConnectionLike::is_open(&connection.connection)
-    }
-}
-
 const INCREMENT_SCRIPT: &str = concat!(
     "local value = redis.call('INCRBYFLOAT', KEYS[1], ARGV[1]); ",
     "if redis.call('TTL', KEYS[1]) == -1 then ",
@@ -70,42 +42,10 @@ const CLAIM_ATTEMPTS: usize = 8;
 
 enum Connections<C> {
     Pool(r2d2::Pool<ConnectionManager>),
+    Cluster(r2d2::Pool<ClusterConnectionManager>),
     Fixed(Mutex<C>),
 }
 
-struct ConnectionRef<'a>(&'a mut dyn redis::ConnectionLike);
-
-impl redis::ConnectionLike for ConnectionRef<'_> {
-    fn req_packed_command(&mut self, cmd: &[u8]) -> redis::RedisResult<redis::Value> {
-        self.0.req_packed_command(cmd)
-    }
-
-    fn req_packed_commands(
-        &mut self,
-        cmd: &[u8],
-        offset: usize,
-        count: usize,
-    ) -> redis::RedisResult<Vec<redis::Value>> {
-        self.0.req_packed_commands(cmd, offset, count)
-    }
-
-    fn get_db(&self) -> i64 {
-        self.0.get_db()
-    }
-
-    fn supports_pipelining(&self) -> bool {
-        self.0.supports_pipelining()
-    }
-
-    fn check_connection(&mut self) -> bool {
-        self.0.check_connection()
-    }
-
-    fn is_open(&self) -> bool {
-        self.0.is_open()
-    }
-}
-
 impl<C> Connections<C>
 where
     C: redis::ConnectionLike + Send + 'static,
@@ -117,13 +57,19 @@ where
         match self {
             Self::Pool(pool) => {
                 let mut pooled = pool.get().map_err(|_| Error::Unavailable)?;
-                let result = operation(&mut ConnectionRef(&mut pooled.connection));
+                let result = operation(&mut ConnectionRef::Node(&mut pooled.connection));
+                pooled.failed = matches!(result, Err(Error::Unavailable));
+                result
+            }
+            Self::Cluster(pool) => {
+                let mut pooled = pool.get().map_err(|_| Error::Unavailable)?;
+                let result = operation(&mut ConnectionRef::Cluster(&mut pooled.connection));
                 pooled.failed = matches!(result, Err(Error::Unavailable));
                 result
             }
             Self::Fixed(connection) => {
                 let mut connection = connection.lock().map_err(|_| Error::Unavailable)?;
-                operation(&mut ConnectionRef(&mut *connection))
+                operation(&mut ConnectionRef::Node(&mut *connection))
             }
         }
     }
@@ -134,27 +80,46 @@ pub struct RedisCache<S, C = redis::Connection> {
     default_ttl: Duration,
     codec: S,
     namespace: Option<String>,
+    topology: RedisTopology,
 }
 
 impl<S: CacheCodec> RedisCache<S> {
     pub fn new(url: &str, default_ttl: Option<Duration>, codec: S) -> Result<Self, Error> {
-        let client = redis::Client::open(url).map_err(|_| Error::Unavailable)?;
-        let pool = r2d2::Pool::builder()
-            .max_size(REDIS_POOL_SIZE)
-            .min_idle(Some(0))
-            .connection_timeout(REDIS_TIMEOUT)
-            .test_on_check_out(false)
-            .build(ConnectionManager(client))
-            .map_err(|_| Error::Unavailable)?;
+        Self::connect(url, &RedisTopology::Standalone, default_ttl, codec)
+    }
+
+    pub fn connect(
+        url: &str,
+        topology: &RedisTopology,
+        default_ttl: Option<Duration>,
+        codec: S,
+    ) -> Result<Self, Error> {
+        let connections = match topology {
+            RedisTopology::Standalone => Connections::Pool(pool(ConnectionManager::open(url)?)?),
+            RedisTopology::Cluster { startup_nodes } => {
+                Connections::Cluster(pool(ClusterConnectionManager::open(url, startup_nodes)?)?)
+            }
+        };
         Ok(Self {
-            connections: Arc::new(Connections::Pool(pool)),
+            connections: Arc::new(connections),
             default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
             codec,
             namespace: None,
+            topology: topology.clone(),
         })
     }
 }
 
+fn pool<M: r2d2::ManageConnection>(manager: M) -> Result<r2d2::Pool<M>, Error> {
+    r2d2::Pool::builder()
+        .max_size(REDIS_POOL_SIZE)
+        .min_idle(Some(0))
+        .connection_timeout(REDIS_TIMEOUT)
+        .test_on_check_out(false)
+        .build(manager)
+        .map_err(|_| Error::Unavailable)
+}
+
 impl<S, C> RedisCache<S, C>
 where
     S: CacheCodec,
@@ -166,6 +131,7 @@ where
             default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
             codec,
             namespace: None,
+            topology: RedisTopology::Standalone,
         }
     }
 
@@ -180,6 +146,10 @@ where
         self.namespace.as_deref()
     }
 
+    pub fn topology(&self) -> &RedisTopology {
+        &self.topology
+    }
+
     fn namespaced_key(&self, key: &str) -> String {
         namespaced_key(self.namespace.as_deref(), key)
     }
@@ -200,26 +170,14 @@ where
     }
 
     fn flush_matching(connection: &mut ConnectionRef<'_>, pattern: &str) -> Result<(), Error> {
-        let mut cursor = 0u64;
-        loop {
-            let (next_cursor, keys): (u64, Vec<String>) = redis::cmd("SCAN")
-                .cursor_arg(cursor)
-                .arg("MATCH")
-                .arg(pattern)
-                .arg("COUNT")
-                .arg(1000)
-                .query(connection)
-                .map_err(|_| Error::Unavailable)?;
+        connection.scan(pattern, 1000, |connection, keys| {
             if !keys.is_empty() {
                 connection
                     .del::<_, usize>(keys)
                     .map_err(|_| Error::Unavailable)?;
             }
-            if next_cursor == 0 {
-                return Ok(());
-            }
-            cursor = next_cursor;
-        }
+            Ok(true)
+        })
     }
 
     fn decode_response(&self, value: redis::Value) -> Result<Option<S::Value>, Error> {
@@ -350,19 +308,19 @@ where
             })
             .collect::<Result<Vec<_>, _>>()?;
         let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl));
+        if entries.is_empty() {
+            return Ok(());
+        }
         Self::run_blocking(Arc::clone(&self.connections), move |connection| {
-            let mut pipeline = redis::pipe();
-            for (key, payload) in entries {
-                pipeline
-                    .cmd("SETEX")
-                    .arg(key)
-                    .arg(ttl)
-                    .arg(payload)
-                    .ignore();
-            }
-            pipeline
-                .query::<()>(connection)
-                .map_err(|_| Error::Unavailable)
+            let commands = entries
+                .into_iter()
+                .map(|(key, payload)| {
+                    let mut command = redis::cmd("SETEX");
+                    command.arg(key).arg(ttl).arg(payload);
+                    command
+                })
+                .collect();
+            connection.pipeline(commands).map(drop)
         })
         .await
     }
@@ -373,7 +331,7 @@ where
 
     async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
         match Self::run_blocking(Arc::clone(&self.connections), |connection| {
-            Ok(match redis::cmd("PING").query::<String>(connection) {
+            Ok(match connection.ping() {
                 Ok(_) => CacheConnectionResult {
                     status: CacheConnectionStatus::Success,
                     message: "Redis cache connection test successful".into(),

litellm-rust/crates/cache-redis/src/cache/connection.rs

@@ -0,0 +1,392 @@
+use std::collections::HashMap;
+
+use litellm_cache::Error;
+use redis::{
+    ConnectionAddr, ConnectionInfo, ConnectionLike, IntoConnectionInfo,
+    cluster::{ClusterClient, ClusterClientBuilder, ClusterConnection, NodeAddress},
+    cluster_routing::{
+        MultipleNodeRoutingInfo, ResponsePolicy, RoutingInfo, SingleNodeRoutingInfo, Slot,
+    },
+};
+
+use super::REDIS_TIMEOUT;
+use crate::topology::RedisNode;
+
+pub(super) struct PooledConnection<C> {
+    pub(super) connection: C,
+    pub(super) failed: bool,
+}
+
+/// Pools connections without a checkout PING, which would double every operation's round trips.
+/// A timed-out command leaves its reply on the socket while redis still reports the connection
+/// open, so any connection whose operation failed is discarded instead of being reused.
+pub(super) struct ConnectionManager(redis::Client);
+
+impl ConnectionManager {
+    pub(super) fn open(url: &str) -> Result<Self, Error> {
+        redis::Client::open(url)
+            .map(Self)
+            .map_err(|_| Error::Unavailable)
+    }
+}
+
+impl r2d2::ManageConnection for ConnectionManager {
+    type Connection = PooledConnection<redis::Connection>;
+    type Error = redis::RedisError;
+
+    fn connect(&self) -> Result<Self::Connection, redis::RedisError> {
+        let connection = self.0.get_connection()?;
+        connection.set_read_timeout(Some(REDIS_TIMEOUT))?;
+        connection.set_write_timeout(Some(REDIS_TIMEOUT))?;
+        Ok(PooledConnection {
+            connection,
+            failed: false,
+        })
+    }
+
+    fn is_valid(&self, connection: &mut Self::Connection) -> Result<(), redis::RedisError> {
+        redis::cmd("PING").query::<String>(&mut connection.connection)?;
+        Ok(())
+    }
+
+    fn has_broken(&self, connection: &mut Self::Connection) -> bool {
+        connection.failed || !redis::ConnectionLike::is_open(&connection.connection)
+    }
+}
+
+pub(super) struct ClusterConnectionManager(ClusterClient);
+
+impl ClusterConnectionManager {
+    pub(super) fn open(url: &str, startup_nodes: &[RedisNode]) -> Result<Self, Error> {
+        if startup_nodes.is_empty() {
+            return Err(Error::Unavailable);
+        }
+        let info = url.into_connection_info().map_err(|_| Error::Unavailable)?;
+        let nodes = startup_nodes
+            .iter()
+            .map(|node| node_info(&info, node))
+            .collect::<Result<Vec<_>, _>>()?;
+        ClusterClientBuilder::new(nodes)
+            .connection_timeout(REDIS_TIMEOUT)
+            .response_timeout(REDIS_TIMEOUT)
+            .build()
+            .map(Self)
+            .map_err(|_| Error::Unavailable)
+    }
+}
+
+fn node_info(info: &ConnectionInfo, node: &RedisNode) -> Result<ConnectionInfo, Error> {
+    let addr = match info.addr() {
+        ConnectionAddr::Tcp(..) => ConnectionAddr::Tcp(node.host.clone(), node.port),
+        ConnectionAddr::TcpTls {
+            insecure,
+            tls_params,
+            ..
+        } => ConnectionAddr::TcpTls {
+            host: node.host.clone(),
+            port: node.port,
+            insecure: *insecure,
+            tls_params: tls_params.clone(),
+        },
+        _ => return Err(Error::Unavailable),
+    };
+    Ok(info.clone().set_addr(addr))
+}
+
+impl r2d2::ManageConnection for ClusterConnectionManager {
+    type Connection = PooledConnection<ClusterConnection>;
+    type Error = redis::RedisError;
+
+    fn connect(&self) -> Result<Self::Connection, redis::RedisError> {
+        let connection = self.0.get_connection()?;
+        connection.set_read_timeout(Some(REDIS_TIMEOUT))?;
+        connection.set_write_timeout(Some(REDIS_TIMEOUT))?;
+        Ok(PooledConnection {
+            connection,
+            failed: false,
+        })
+    }
+
+    fn is_valid(&self, connection: &mut Self::Connection) -> Result<(), redis::RedisError> {
+        redis::cmd("PING").query::<String>(&mut connection.connection)?;
+        Ok(())
+    }
+
+    fn has_broken(&self, connection: &mut Self::Connection) -> bool {
+        connection.failed || !redis::ConnectionLike::is_open(&connection.connection)
+    }
+}
+
+pub(crate) enum ConnectionRef<'a> {
+    Node(&'a mut dyn redis::ConnectionLike),
+    Cluster(&'a mut ClusterConnection),
+}
+
+impl redis::ConnectionLike for ConnectionRef<'_> {
+    fn req_packed_command(&mut self, cmd: &[u8]) -> redis::RedisResult<redis::Value> {
+        match self {
+            Self::Node(connection) => connection.req_packed_command(cmd),
+            Self::Cluster(connection) => connection.req_packed_command(cmd),
+        }
+    }
+
+    fn req_packed_commands(
+        &mut self,
+        cmd: &[u8],
+        offset: usize,
+        count: usize,
+    ) -> redis::RedisResult<Vec<redis::Value>> {
+        match self {
+            Self::Node(connection) => connection.req_packed_commands(cmd, offset, count),
+            Self::Cluster(connection) => connection.req_packed_commands(cmd, offset, count),
+        }
+    }
+
+    fn get_db(&self) -> i64 {
+        match self {
+            Self::Node(connection) => connection.get_db(),
+            Self::Cluster(connection) => redis::ConnectionLike::get_db(*connection),
+        }
+    }
+
+    fn supports_pipelining(&self) -> bool {
+        match self {
+            Self::Node(connection) => connection.supports_pipelining(),
+            Self::Cluster(connection) => redis::ConnectionLike::supports_pipelining(*connection),
+        }
+    }
+
+    fn check_connection(&mut self) -> bool {
+        match self {
+            Self::Node(connection) => connection.check_connection(),
+            Self::Cluster(connection) => connection.check_connection(),
+        }
+    }
+
+    fn is_open(&self) -> bool {
+        match self {
+            Self::Node(connection) => connection.is_open(),
+            Self::Cluster(connection) => redis::ConnectionLike::is_open(*connection),
+        }
+    }
+}
+
+impl ConnectionRef<'_> {
+    pub(crate) fn pipeline(
+        &mut self,
+        commands: Vec<redis::Cmd>,
+    ) -> Result<Vec<redis::Value>, Error> {
+        match self {
+            Self::Node(connection) => {
+                let mut pipeline = redis::pipe();
+                for command in &commands {
+                    pipeline.add_command(command.clone());
+                }
+                pipeline
+                    .query::<Vec<redis::Value>>(*connection)
+                    .map_err(|_| Error::Unavailable)
+            }
+            Self::Cluster(connection) => {
+                let mut replies: Vec<Option<redis::Value>> = vec![None; commands.len()];
+                for indices in slot_groups(&commands).into_values() {
+                    let mut pipeline = redis::pipe();
+                    for index in &indices {
+                        pipeline.add_command(commands[*index].clone());
+                    }
+                    let values = connection
+                        .req_packed_commands(&pipeline.get_packed_pipeline(), 0, indices.len())
+                        .map_err(|_| Error::Unavailable)?;
+                    if values.len() != indices.len() {
+                        return Err(Error::Unavailable);
+                    }
+                    for (index, value) in indices.into_iter().zip(values) {
+                        replies[index] = Some(value);
+                    }
+                }
+                replies
+                    .into_iter()
+                    .collect::<Option<Vec<_>>>()
+                    .ok_or(Error::Unavailable)
+            }
+        }
+    }
+
+    pub(crate) fn scan(
+        &mut self,
+        pattern: &str,
+        count: usize,
+        mut visit: impl FnMut(&mut Self, Vec<String>) -> Result<bool, Error>,
+    ) -> Result<(), Error> {
+        let pages = match self {
+            Self::Node(connection) => {
+                let page = scan_command(0, pattern, count)
+                    .query::<ScanPage>(*connection)
+                    .map_err(|_| Error::Unavailable)?;
+                vec![(None, page)]
+            }
+            Self::Cluster(connection) => connection
+                .route_command(
+                    &scan_command(0, pattern, count),
+                    RoutingInfo::MultiNode((
+                        MultipleNodeRoutingInfo::AllMasters,
+                        Some(ResponsePolicy::Special),
+                    )),
+                )
+                .map_err(|_| Error::Unavailable)
+                .and_then(primary_pages)?
+                .into_iter()
+                .map(|(node, page)| (Some(node), page))
+                .collect(),
+        };
+        for (node, (mut cursor, mut keys)) in pages {
+            loop {
+                if !visit(self, keys)? {
+                    return Ok(());
+                }
+                if cursor == 0 {
+                    break;
+                }
+                (cursor, keys) = self.scan_page(node.as_ref(), cursor, pattern, count)?;
+            }
+        }
+        Ok(())
+    }
+
+    pub(crate) fn ping(&mut self) -> Result<bool, redis::RedisError> {
+        let command = redis::cmd("PING");
+        match self {
+            Self::Node(connection) => command
+                .query::<String>(*connection)
+                .map(|response| response == "PONG"),
+            Self::Cluster(connection) => connection
+                .route_command(
+                    &command,
+                    RoutingInfo::MultiNode((
+                        MultipleNodeRoutingInfo::AllNodes,
+                        Some(ResponsePolicy::AllSucceeded),
+                    )),
+                )
+                .map(|_| true),
+        }
+    }
+
+    pub(crate) fn node_text(&mut self, command: &redis::Cmd) -> Result<String, Error> {
+        match self {
+            Self::Node(connection) => command.query(*connection).map_err(|_| Error::Unavailable),
+            Self::Cluster(connection) => {
+                let value = connection
+                    .route_command(
+                        command,
+                        RoutingInfo::MultiNode((
+                            MultipleNodeRoutingInfo::AllNodes,
+                            Some(ResponsePolicy::Special),
+                        )),
+                    )
+                    .map_err(|_| Error::Unavailable)?;
+                let redis::Value::Map(entries) = value else {
+                    return Err(Error::Unavailable);
+                };
+                let mut replies = entries
+                    .into_iter()
+                    .map(|(node, reply)| {
+                        Ok((
+                            redis::from_redis_value::<String>(node)
+                                .map_err(|_| Error::Unavailable)?,
+                            redis::from_redis_value::<String>(reply)
+                                .map_err(|_| Error::Unavailable)?,
+                        ))
+                    })
+                    .collect::<Result<Vec<(String, String)>, Error>>()?;
+                replies.sort();
+                Ok(replies
+                    .into_iter()
+                    .map(|(_, reply)| reply)
+                    .collect::<Vec<_>>()
+                    .join("\n"))
+            }
+        }
+    }
+
+    pub(crate) fn flushall(&mut self) -> Result<(), Error> {
+        let command = redis::cmd("FLUSHALL");
+        match self {
+            Self::Node(connection) => command.query(*connection).map_err(|_| Error::Unavailable),
+            Self::Cluster(connection) => connection
+                .route_command(
+                    &command,
+                    RoutingInfo::MultiNode((
+                        MultipleNodeRoutingInfo::AllMasters,
+                        Some(ResponsePolicy::AllSucceeded),
+                    )),
+                )
+                .map(|_| ())
+                .map_err(|_| Error::Unavailable),
+        }
+    }
+
+    fn scan_page(
+        &mut self,
+        node: Option<&NodeAddress>,
+        cursor: u64,
+        pattern: &str,
+        count: usize,
+    ) -> Result<ScanPage, Error> {
+        let command = scan_command(cursor, pattern, count);
+        match (self, node) {
+            (Self::Node(connection), None) => {
+                command.query(*connection).map_err(|_| Error::Unavailable)
+            }
+            (Self::Cluster(connection), Some(node)) => connection
+                .route_command(
+                    &command,
+                    RoutingInfo::SingleNode(SingleNodeRoutingInfo::ByAddress {
+                        host: node.host().to_string(),
+                        port: node.port(),
+                    }),
+                )
+                .map_err(|_| Error::Unavailable)
+                .and_then(|value| redis::from_redis_value(value).map_err(|_| Error::Unavailable)),
+            _ => Err(Error::Unavailable),
+        }
+    }
+}
+
+type ScanPage = (u64, Vec<String>);
+
+fn primary_pages(value: redis::Value) -> Result<Vec<(NodeAddress, ScanPage)>, Error> {
+    let redis::Value::Map(entries) = value else {
+        return Err(Error::Unavailable);
+    };
+    entries
+        .into_iter()
+        .map(|(node, page)| {
+            let node = redis::from_redis_value::<String>(node).map_err(|_| Error::Unavailable)?;
+            let node = NodeAddress::try_from(node.as_str()).map_err(|_| Error::Unavailable)?;
+            let page = redis::from_redis_value::<ScanPage>(page).map_err(|_| Error::Unavailable)?;
+            Ok((node, page))
+        })
+        .collect()
+}
+
+fn scan_command(cursor: u64, pattern: &str, count: usize) -> redis::Cmd {
+    let mut command = redis::cmd("SCAN");
+    command
+        .cursor_arg(cursor)
+        .arg("MATCH")
+        .arg(pattern)
+        .arg("COUNT")
+        .arg(count);
+    command
+}
+
+fn slot_groups(commands: &[redis::Cmd]) -> HashMap<Slot, Vec<usize>> {
+    let mut groups: HashMap<Slot, Vec<usize>> = HashMap::new();
+    for (index, command) in commands.iter().enumerate() {
+        let key = match command.args_iter().nth(1) {
+            Some(redis::Arg::Simple(key)) => key,
+            _ => b"",
+        };
+        groups.entry(Slot::for_key(key)).or_default().push(index);
+    }
+    groups
+}

litellm-rust/crates/cache-redis/src/cache/operations.rs

@@ -183,20 +183,13 @@ where
     }
 
     pub fn sync_ping(&self) -> Result<bool, Error> {
-        self.connections.execute(|connection| {
-            redis::cmd("PING")
-                .query::<String>(connection)
-                .map(|response| response == "PONG")
-                .map_err(|_| Error::Unavailable)
-        })
+        self.connections
+            .execute(|connection| connection.ping().map_err(|_| Error::Unavailable))
     }
 
     pub async fn ping(&self) -> Result<bool, Error> {
         Self::run_blocking(Arc::clone(&self.connections), |connection| {
-            redis::cmd("PING")
-                .query::<String>(connection)
-                .map(|response| response == "PONG")
-                .map_err(|_| Error::Unavailable)
+            connection.ping().map_err(|_| Error::Unavailable)
         })
         .await
     }
@@ -216,24 +209,13 @@ where
     pub async fn async_scan_iter(&self, pattern: &str, count: usize) -> Result<Vec<String>, Error> {
         let pattern = format!("{}*", self.namespaced_key(pattern));
         Self::run_blocking(Arc::clone(&self.connections), move |connection| {
-            let mut cursor = 0u64;
             let mut matches = Vec::new();
-            loop {
-                let (next_cursor, keys): (u64, Vec<String>) = redis::cmd("SCAN")
-                    .cursor_arg(cursor)
-                    .arg("MATCH")
-                    .arg(&pattern)
-                    .arg("COUNT")
-                    .arg(count)
-                    .query(connection)
-                    .map_err(|_| Error::Unavailable)?;
+            connection.scan(&pattern, count, |_, keys| {
                 matches.extend(keys);
-                if matches.len() >= count || next_cursor == 0 {
-                    matches.truncate(count);
-                    return Ok(matches);
-                }
-                cursor = next_cursor;
-            }
+                Ok(matches.len() < count)
+            })?;
+            matches.truncate(count);
+            Ok(matches)
         })
         .await
     }
@@ -250,13 +232,18 @@ where
         let key = self.namespaced_key(key);
         let ttl = Self::ttl_seconds(ttl.unwrap_or(self.default_ttl));
         Self::run_blocking(Arc::clone(&self.connections), move |connection| {
-            let mut pipeline = redis::pipe();
-            pipeline.cmd("SADD").arg(&key).arg(values);
-            pipeline.cmd("EXPIRE").arg(&key).arg(ttl).ignore();
-            pipeline
-                .query::<(usize,)>(connection)
-                .map(|(added,)| added)
-                .map_err(|_| Error::Unavailable)
+            let mut sadd = redis::cmd("SADD");
+            sadd.arg(&key).arg(values);
+            let mut expire = redis::cmd("EXPIRE");
+            expire.arg(&key).arg(ttl);
+            let replies = connection.pipeline(vec![sadd, expire])?;
+            replies
+                .into_iter()
+                .next()
+                .map(redis::from_redis_value::<usize>)
+                .transpose()
+                .map_err(|_| Error::Unavailable)?
+                .ok_or(Error::Unavailable)
         })
         .await
     }
@@ -293,11 +280,19 @@ where
             return Ok(Vec::new());
         }
         Self::run_blocking(Arc::clone(&self.connections), move |connection| {
-            let mut pipeline = redis::pipe();
-            for (key, values) in operations {
-                pipeline.cmd("RPUSH").arg(key).arg(values);
-            }
-            pipeline.query(connection).map_err(|_| Error::Unavailable)
+            let commands = operations
+                .into_iter()
+                .map(|(key, values)| {
+                    let mut command = redis::cmd("RPUSH");
+                    command.arg(key).arg(values);
+                    command
+                })
+                .collect();
+            connection
+                .pipeline(commands)?
+                .into_iter()
+                .map(|value| redis::from_redis_value(value).map_err(|_| Error::Unavailable))
+                .collect()
         })
         .await
     }
@@ -339,16 +334,18 @@ where
             .map(|(_, count)| count.is_some())
             .collect::<Vec<_>>();
         let values = Self::run_blocking(Arc::clone(&self.connections), move |connection| {
-            let mut pipeline = redis::pipe();
-            for (key, count) in operations {
-                let command = pipeline.cmd("LPOP").arg(key);
-                if let Some(count) = count {
-                    command.arg(count);
-                }
-            }
-            pipeline
-                .query::<Vec<redis::Value>>(connection)
-                .map_err(|_| Error::Unavailable)
+            let commands = operations
+                .into_iter()
+                .map(|(key, count)| {
+                    let mut command = redis::cmd("LPOP");
+                    command.arg(key);
+                    if let Some(count) = count {
+                        command.arg(count);
+                    }
+                    command
+                })
+                .collect();
+            connection.pipeline(commands)
         })
         .await?;
         values
@@ -381,28 +378,17 @@ where
     }
 
     pub fn client_list(&self) -> Result<String, Error> {
-        self.connections.execute(|connection| {
-            redis::cmd("CLIENT")
-                .arg("LIST")
-                .query(connection)
-                .map_err(|_| Error::Unavailable)
-        })
+        self.connections
+            .execute(|connection| connection.node_text(redis::cmd("CLIENT").arg("LIST")))
     }
 
     pub fn info(&self) -> Result<String, Error> {
-        self.connections.execute(|connection| {
-            redis::cmd("INFO")
-                .query(connection)
-                .map_err(|_| Error::Unavailable)
-        })
+        self.connections
+            .execute(|connection| connection.node_text(&redis::cmd("INFO")))
     }
 
     pub fn flushall(&self) -> Result<(), Error> {
-        self.connections.execute(|connection| {
-            redis::cmd("FLUSHALL")
-                .query(connection)
-                .map_err(|_| Error::Unavailable)
-        })
+        self.connections.execute(|connection| connection.flushall())
     }
 }
 
@@ -441,14 +427,27 @@ where
             return Ok(Vec::new());
         }
         Self::run_blocking(Arc::clone(&self.connections), move |connection| {
-            let mut pipeline = redis::pipe();
+            let mut commands = Vec::with_capacity(operations.len() * 2);
+            let mut increments = Vec::with_capacity(operations.len());
             for (key, amount, ttl) in operations {
-                pipeline.cmd("INCRBYFLOAT").arg(&key).arg(amount);
+                let mut increment = redis::cmd("INCRBYFLOAT");
+                increment.arg(&key).arg(amount);
+                increments.push(commands.len());
+                commands.push(increment);
                 if let Some(ttl) = ttl {
-                    pipeline.cmd("EXPIRE").arg(key).arg(ttl).ignore();
+                    let mut expire = redis::cmd("EXPIRE");
+                    expire.arg(key).arg(ttl);
+                    commands.push(expire);
                 }
             }
-            pipeline.query(connection).map_err(|_| Error::Unavailable)
+            let mut replies = connection.pipeline(commands)?;
+            increments
+                .into_iter()
+                .map(|index| {
+                    redis::from_redis_value(std::mem::take(&mut replies[index]))
+                        .map_err(|_| Error::Unavailable)
+                })
+                .collect()
         })
         .await
     }

litellm-rust/crates/cache-redis/tests/cluster.rs

@@ -0,0 +1,492 @@
+//! Contract tests against a real Redis Cluster. Set `LITELLM_TEST_REDIS_CLUSTER_NODES` to a
+//! comma separated `host:port` list (for example `127.0.0.1:7000,127.0.0.1:7001`) to run them.
+
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+use litellm_cache::{
+    BaseCache, BatchCache, BatchEntry, CacheConnectionStatus, CacheScript, ClaimCache,
+    CounterCache, DeleteCache, Error, ExactCacheContext, FlushCache, IncrementOperation, JsonCodec,
+    ScriptCache,
+};
+use litellm_cache_redis::{
+    RedisArg, RedisCache, RedisLpopOperation, RedisLpopResult, RedisNode, RedisRpushOperation,
+    RedisTopology,
+};
+use redis::cluster_routing::Slot;
+
+type Cache = RedisCache<JsonCodec<serde_json::Value>>;
+
+fn topology() -> Option<RedisTopology> {
+    let nodes = std::env::var("LITELLM_TEST_REDIS_CLUSTER_NODES").ok()?;
+    let startup_nodes = nodes
+        .split(',')
+        .map(|node| {
+            let (host, port) = node.trim().rsplit_once(':').expect("host:port");
+            RedisNode {
+                host: host.to_string(),
+                port: port.parse().expect("port"),
+            }
+        })
+        .collect();
+    Some(RedisTopology::Cluster { startup_nodes })
+}
+
+fn namespace(label: &str) -> String {
+    let nanos = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap()
+        .as_nanos();
+    format!("cluster-test:{label}:{nanos}")
+}
+
+fn cluster_url() -> String {
+    std::env::var("LITELLM_TEST_REDIS_CLUSTER_URL")
+        .unwrap_or_else(|_| "redis://127.0.0.1:7000".into())
+}
+
+fn cluster_cache(label: &str) -> Option<Cache> {
+    let topology = topology()?;
+    Some(
+        Cache::connect(
+            &cluster_url(),
+            &topology,
+            Some(Duration::from_secs(120)),
+            JsonCodec::new(),
+        )
+        .expect("cluster connection")
+        .with_namespace(Some(namespace(label))),
+    )
+}
+
+fn counter_cache(label: &str) -> Option<RedisCache<JsonCodec<f64>>> {
+    let topology = topology()?;
+    Some(
+        RedisCache::connect(
+            &cluster_url(),
+            &topology,
+            Some(Duration::from_secs(60)),
+            JsonCodec::new(),
+        )
+        .expect("cluster connection")
+        .with_namespace(Some(namespace(label))),
+    )
+}
+
+fn multi_slot_keys(count: usize) -> Vec<String> {
+    let keys: Vec<String> = (0..count).map(|index| format!("key-{index}")).collect();
+    let slots: std::collections::HashSet<Slot> = keys.iter().map(Slot::for_key).collect();
+    assert!(slots.len() > 1, "keys must span multiple slots");
+    keys
+}
+
+macro_rules! cluster_or_skip {
+    ($label:expr) => {
+        match cluster_cache($label) {
+            Some(cache) => cache,
+            None => return,
+        }
+    };
+}
+
+#[test]
+fn constructor_rejects_clusters_without_startup_nodes() {
+    let error = Cache::connect(
+        "redis://127.0.0.1:7000",
+        &RedisTopology::Cluster {
+            startup_nodes: Vec::new(),
+        },
+        None,
+        JsonCodec::new(),
+    )
+    .err();
+    assert!(matches!(error, Some(Error::Unavailable)));
+}
+
+#[test]
+fn constructor_rejects_unix_socket_urls_for_clusters() {
+    let error = Cache::connect(
+        "redis+unix:///tmp/redis.sock",
+        &RedisTopology::Cluster {
+            startup_nodes: vec![RedisNode {
+                host: "127.0.0.1".into(),
+                port: 7000,
+            }],
+        },
+        None,
+        JsonCodec::new(),
+    )
+    .err();
+    assert!(matches!(error, Some(Error::Unavailable)));
+}
+
+#[test]
+fn single_key_operations_round_trip_with_ttl_rounding() {
+    let cache = cluster_or_skip!("single");
+    let context = ExactCacheContext {
+        ttl: Some(Duration::from_millis(1500)),
+    };
+    let keys = multi_slot_keys(12);
+    for (index, key) in keys.iter().enumerate() {
+        cache
+            .set_cache(key, serde_json::json!({ "index": index }), &context)
+            .unwrap();
+    }
+    for (index, key) in keys.iter().enumerate() {
+        assert_eq!(
+            cache.get_cache(key, &context).unwrap(),
+            Some(serde_json::json!({ "index": index }))
+        );
+    }
+    let runtime = tokio::runtime::Runtime::new().unwrap();
+    let ttl = runtime.block_on(cache.async_get_ttl(&keys[0])).unwrap();
+    assert_eq!(ttl, Some(2));
+    cache.delete_cache(&keys[0]).unwrap();
+    assert_eq!(cache.get_cache(&keys[0], &context).unwrap(), None);
+    assert!(cache.sync_ping().unwrap());
+}
+
+#[tokio::test]
+async fn batch_reads_span_slots_and_preserve_order_with_malformed_entries() {
+    let cache = cluster_or_skip!("batch");
+    let context = ExactCacheContext::default();
+    let keys = multi_slot_keys(40);
+    for (index, key) in keys.iter().enumerate() {
+        if index % 5 == 0 {
+            continue;
+        }
+        cache
+            .async_set_cache(key, serde_json::json!(index), context.clone())
+            .await
+            .unwrap();
+    }
+    let mut raw = redis::cluster::ClusterClient::new(vec![cluster_url()])
+        .unwrap()
+        .get_connection()
+        .unwrap();
+    let malformed = format!("{}:{}", cache.namespace().unwrap(), keys[1]);
+    redis::cmd("SET")
+        .arg(&malformed)
+        .arg("not json")
+        .exec(&mut raw)
+        .unwrap();
+
+    let entries = cache
+        .async_batch_get_cache(keys.clone(), context.clone())
+        .await
+        .unwrap();
+    assert_eq!(entries.len(), keys.len());
+    for (index, entry) in entries.iter().enumerate() {
+        let expected = if index == 1 {
+            BatchEntry::Invalid
+        } else if index % 5 == 0 {
+            BatchEntry::Miss
+        } else {
+            BatchEntry::Hit(serde_json::json!(index))
+        };
+        assert_eq!(*entry, expected, "entry {index}");
+    }
+    let sync_entries = cache.batch_get_cache(&keys, &context).unwrap();
+    assert_eq!(sync_entries, entries);
+
+    cache.delete_cache_keys(keys.clone()).await.unwrap();
+    let entries = cache.async_batch_get_cache(keys, context).await.unwrap();
+    assert!(entries.iter().all(|entry| *entry == BatchEntry::Miss));
+}
+
+#[tokio::test]
+async fn pipelines_group_by_slot_and_return_results_in_submission_order() {
+    let cache = cluster_or_skip!("pipeline");
+    let keys = multi_slot_keys(30);
+    let entries = keys
+        .iter()
+        .enumerate()
+        .map(|(index, key)| (key.clone(), serde_json::json!(index)))
+        .collect();
+    cache
+        .async_set_cache_pipeline(entries, ExactCacheContext::default())
+        .await
+        .unwrap();
+    let hits = cache
+        .async_batch_get_cache(keys.clone(), ExactCacheContext::default())
+        .await
+        .unwrap();
+    assert!(
+        hits.iter()
+            .enumerate()
+            .all(|(index, entry)| *entry == BatchEntry::Hit(serde_json::json!(index)))
+    );
+
+    let queues: Vec<String> = keys.iter().map(|key| format!("queue:{key}")).collect();
+    let pushed = cache
+        .async_rpush_pipeline(
+            queues
+                .iter()
+                .enumerate()
+                .map(|(index, key)| RedisRpushOperation {
+                    key: key.clone(),
+                    values: (0..=index)
+                        .map(|value| RedisArg::Integer(value as i64))
+                        .collect(),
+                })
+                .collect(),
+        )
+        .await
+        .unwrap();
+    assert_eq!(pushed, (1..=keys.len()).collect::<Vec<_>>());
+    let popped = cache
+        .async_lpop_pipeline(
+            queues
+                .iter()
+                .enumerate()
+                .map(|(index, key)| RedisLpopOperation {
+                    key: key.clone(),
+                    count: (index % 2 == 0).then_some(2),
+                })
+                .collect(),
+        )
+        .await
+        .unwrap();
+    for (index, result) in popped.into_iter().enumerate() {
+        match result {
+            RedisLpopResult::Value(value) => {
+                assert_eq!(index % 2, 1, "queue {index}");
+                assert_eq!(value, b"0");
+            }
+            RedisLpopResult::Values(values) => {
+                assert_eq!(index % 2, 0, "queue {index}");
+                let expected: Vec<Vec<u8>> = (0..=index)
+                    .take(2)
+                    .map(|value| value.to_string().into_bytes())
+                    .collect();
+                assert_eq!(values, expected);
+            }
+            other => panic!("queue {index}: {other:?}"),
+        }
+    }
+
+    let counters: Vec<String> = keys.iter().map(|key| format!("counter:{key}")).collect();
+    let Some(counter) = counter_cache("counter") else {
+        return;
+    };
+    let totals = counter
+        .async_increment_pipeline(
+            counters
+                .iter()
+                .enumerate()
+                .map(|(index, key)| IncrementOperation {
+                    key: key.clone(),
+                    amount: index as f64 + 0.5,
+                    ttl: (index % 3 == 0).then_some(Duration::from_secs(30)),
+                })
+                .collect(),
+        )
+        .await
+        .unwrap();
+    let expected: Vec<f64> = (0..keys.len()).map(|index| index as f64 + 0.5).collect();
+    assert_eq!(totals, expected);
+    assert_eq!(counter.async_get_ttl(&counters[0]).await.unwrap(), Some(30));
+    assert_eq!(counter.async_get_ttl(&counters[1]).await.unwrap(), None);
+    counter.async_flush_cache().await.unwrap();
+    cache.async_flush_cache().await.unwrap();
+}
+
+#[tokio::test]
+async fn scan_and_scoped_flush_cover_every_primary() {
+    let cache = cluster_or_skip!("flush");
+    let other = cluster_or_skip!("other");
+    let context = ExactCacheContext::default();
+    let keys = multi_slot_keys(60);
+    for key in &keys {
+        cache
+            .async_set_cache(key, serde_json::json!(true), context.clone())
+            .await
+            .unwrap();
+        other
+            .async_set_cache(key, serde_json::json!(true), context.clone())
+            .await
+            .unwrap();
+    }
+    let mut scanned = cache.async_scan_iter("key-", 1000).await.unwrap();
+    scanned.sort();
+    let mut expected: Vec<String> = keys
+        .iter()
+        .map(|key| format!("{}:{key}", cache.namespace().unwrap()))
+        .collect();
+    expected.sort();
+    assert_eq!(scanned, expected);
+    assert_eq!(cache.async_scan_iter("key-", 7).await.unwrap().len(), 7);
+
+    cache.flush_cache().unwrap();
+    let flushed = cache
+        .async_batch_get_cache(keys.clone(), context.clone())
+        .await
+        .unwrap();
+    assert!(flushed.iter().all(|entry| *entry == BatchEntry::Miss));
+    let kept = other.async_batch_get_cache(keys, context).await.unwrap();
+    assert!(
+        kept.iter()
+            .all(|entry| *entry == BatchEntry::Hit(serde_json::json!(true)))
+    );
+    other.async_flush_cache().await.unwrap();
+}
+
+fn ping_calls_per_node(startup: &redis::Client) -> Vec<(String, u64)> {
+    let mut connection = startup.get_connection().unwrap();
+    let nodes: String = redis::cmd("CLUSTER")
+        .arg("NODES")
+        .query(&mut connection)
+        .unwrap();
+    let mut counts: Vec<(String, u64)> = nodes
+        .lines()
+        .map(|line| {
+            let address = line.split_whitespace().nth(1).unwrap();
+            let address = address.split('@').next().unwrap();
+            let mut node = redis::Client::open(format!("redis://{address}"))
+                .unwrap()
+                .get_connection()
+                .unwrap();
+            let stats: String = redis::cmd("INFO")
+                .arg("commandstats")
+                .query(&mut node)
+                .unwrap();
+            let calls = stats
+                .lines()
+                .find_map(|stat| stat.strip_prefix("cmdstat_ping:calls="))
+                .and_then(|rest| rest.split(',').next())
+                .map_or(0, |calls| calls.parse().unwrap());
+            (address.to_string(), calls)
+        })
+        .collect();
+    counts.sort();
+    counts
+}
+
+#[tokio::test]
+async fn ping_reaches_every_node() {
+    let cache = cluster_or_skip!("ping");
+    let startup = redis::Client::open(cluster_url()).unwrap();
+    let before = ping_calls_per_node(&startup);
+    assert!(before.len() >= 2, "{before:?}");
+    assert!(cache.ping().await.unwrap());
+    let after = ping_calls_per_node(&startup);
+    for ((node, calls_before), (_, calls_after)) in before.iter().zip(&after) {
+        assert!(calls_after > calls_before, "{node} was not pinged");
+    }
+    assert!(cache.sync_ping().unwrap());
+    let result = cache.test_connection().await.unwrap();
+    assert_eq!(result.status, CacheConnectionStatus::Success);
+}
+
+#[tokio::test]
+async fn counters_claims_scripts_and_sets_work_on_the_cluster() {
+    let Some(counter) = counter_cache("counter") else {
+        return;
+    };
+    let context = ExactCacheContext::default();
+    assert_eq!(
+        counter
+            .increment_cache("spend", 1.5, context.clone())
+            .unwrap(),
+        1.5
+    );
+    assert_eq!(
+        counter
+            .async_increment("spend", 2.0, context.clone())
+            .await
+            .unwrap(),
+        3.5
+    );
+    assert_eq!(
+        counter
+            .increment_with_floor("budget", -3, Duration::from_secs(30))
+            .unwrap(),
+        0
+    );
+    assert_eq!(
+        counter
+            .async_increment_with_floor("budget", 7, Duration::from_secs(30))
+            .await
+            .unwrap(),
+        7
+    );
+    assert_eq!(counter.async_set_max("peak", 4.0, None).await.unwrap(), 4.0);
+    assert_eq!(counter.async_set_max("peak", 2.0, None).await.unwrap(), 4.0);
+    counter.flush_cache().unwrap();
+
+    let cache = cluster_or_skip!("claim");
+    let owner = serde_json::json!("owner-a");
+    let rival = serde_json::json!("owner-b");
+    assert_eq!(
+        cache
+            .claim_cache("lock", owner.clone(), &[], context.clone())
+            .unwrap(),
+        owner
+    );
+    assert_eq!(
+        cache
+            .async_claim_cache("lock", rival.clone(), vec![owner.clone()], context.clone())
+            .await
+            .unwrap(),
+        owner
+    );
+    assert_eq!(
+        cache
+            .claim_cache("lock", rival.clone(), &[], context.clone())
+            .unwrap(),
+        owner
+    );
+    assert_eq!(
+        cache
+            .async_claim_cache("lock", rival.clone(), vec![rival.clone()], context.clone())
+            .await
+            .unwrap(),
+        rival
+    );
+
+    let script = cache
+        .async_register_script("return redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2])".into());
+    let reply = script
+        .invoke(
+            vec!["scripted".into()],
+            vec![RedisArg::Bytes(b"payload".to_vec()), RedisArg::Integer(5)],
+        )
+        .await
+        .unwrap();
+    assert_eq!(reply, redis::Value::Okay);
+    assert_eq!(cache.async_get_ttl("scripted").await.unwrap(), Some(5));
+    let evaluated: redis::Value = cache
+        .async_eval(
+            "return redis.call('GET', KEYS[1])".into(),
+            vec!["scripted".into()],
+            Vec::new(),
+        )
+        .await
+        .unwrap();
+    assert_eq!(evaluated, redis::Value::BulkString(b"payload".to_vec()));
+
+    assert_eq!(
+        cache
+            .async_set_cache_sadd(
+                "members",
+                vec![
+                    RedisArg::Bytes(b"a".to_vec()),
+                    RedisArg::Bytes(b"b".to_vec())
+                ],
+                Some(Duration::from_secs(9)),
+            )
+            .await
+            .unwrap(),
+        2
+    );
+    assert_eq!(cache.async_get_ttl("members").await.unwrap(), Some(9));
+
+    let result = cache.test_connection().await.unwrap();
+    assert_eq!(result.status, CacheConnectionStatus::Success);
+    assert!(cache.ping().await.unwrap());
+    let info = cache.info().unwrap();
+    assert!(info.matches("redis_version").count() > 1, "{info}");
+    assert!(cache.client_list().unwrap().contains("id="));
+    cache.async_flush_cache().await.unwrap();
+    assert_eq!(cache.async_get_ttl("members").await.unwrap(), None);
+    assert_eq!(cache.get_cache("lock", &context).unwrap(), None);
+}

litellm-rust/crates/core-utils/src/serde_compat.rs

@@ -1,33 +1,91 @@
-use serde::{Deserialize, Deserializer, de::Error};
-use serde_json::Value;
+use serde::{
+    Deserializer,
+    de::{Error, Visitor},
+};
 use serde_with::DeserializeAs;
 
 pub struct LaxI64;
 pub struct FiniteF64;
 
+pub fn parse_str_bool(value: &str) -> Option<bool> {
+    let token = value.trim_matches(|character: char| {
+        character.is_whitespace() || matches!(character, '\u{1c}'..='\u{1f}')
+    });
+    if token.eq_ignore_ascii_case("true") {
+        return Some(true);
+    }
+    token.eq_ignore_ascii_case("false").then_some(false)
+}
+
 impl<'de> DeserializeAs<'de, i64> for LaxI64 {
     fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<i64, D::Error> {
-        match Value::deserialize(deserializer)? {
-            Value::Number(number) if number.is_f64() => number.as_f64().and_then(integral_float),
-            Value::Number(number) => number.as_i64(),
-            Value::String(value) => integer_string(value.trim()),
-            Value::Bool(value) => Some(i64::from(value)),
-            _ => None,
-        }
-        .ok_or_else(|| D::Error::custom("expected an integer in the i64 range"))
+        deserializer.deserialize_any(Self)
+    }
+}
+
+impl<'de> Visitor<'de> for LaxI64 {
+    type Value = i64;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter.write_str("an integer in the i64 range")
+    }
+
+    fn visit_i64<E: Error>(self, value: i64) -> Result<i64, E> {
+        Ok(value)
+    }
+
+    fn visit_u64<E: Error>(self, value: u64) -> Result<i64, E> {
+        i64::try_from(value).map_err(E::custom)
+    }
+
+    fn visit_f64<E: Error>(self, value: f64) -> Result<i64, E> {
+        integral_float(value).ok_or_else(|| E::custom("expected an integer in the i64 range"))
+    }
+
+    fn visit_str<E: Error>(self, value: &str) -> Result<i64, E> {
+        integer_string(value.trim())
+            .ok_or_else(|| E::custom("expected an integer in the i64 range"))
+    }
+
+    fn visit_bool<E: Error>(self, value: bool) -> Result<i64, E> {
+        Ok(i64::from(value))
     }
 }
 
 impl<'de> DeserializeAs<'de, f64> for FiniteF64 {
     fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<f64, D::Error> {
-        match Value::deserialize(deserializer)? {
-            Value::Number(number) => number.as_f64(),
-            Value::String(value) => value.trim().parse::<f64>().ok(),
-            Value::Bool(value) => Some(f64::from(value)),
-            _ => None,
-        }
-        .filter(|value| value.is_finite())
-        .ok_or_else(|| D::Error::custom("expected a finite number"))
+        deserializer.deserialize_any(Self)
+    }
+}
+
+impl<'de> Visitor<'de> for FiniteF64 {
+    type Value = f64;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter.write_str("a finite number")
+    }
+
+    fn visit_i64<E: Error>(self, value: i64) -> Result<f64, E> {
+        Ok(value as f64)
+    }
+
+    fn visit_u64<E: Error>(self, value: u64) -> Result<f64, E> {
+        Ok(value as f64)
+    }
+
+    fn visit_f64<E: Error>(self, value: f64) -> Result<f64, E> {
+        value
+            .is_finite()
+            .then_some(value)
+            .ok_or_else(|| E::custom("expected a finite number"))
+    }
+
+    fn visit_str<E: Error>(self, value: &str) -> Result<f64, E> {
+        self.visit_f64(value.trim().parse::<f64>().map_err(E::custom)?)
+    }
+
+    fn visit_bool<E: Error>(self, value: bool) -> Result<f64, E> {
+        Ok(f64::from(value))
     }
 }
 
@@ -66,7 +124,7 @@ fn integral_float(value: f64) -> Option<i64> {
 
 #[cfg(test)]
 mod tests {
-    use serde::Serialize;
+    use serde::{Deserialize, Serialize};
     use serde_json::json;
     use serde_with::serde_as;
 
@@ -81,6 +139,22 @@ mod tests {
         float: Option<f64>,
     }
 
+    #[test]
+    fn boolean_tokens_follow_python_string_trimming_without_redis_tokens() {
+        for (input, expected) in [
+            (" True ", Some(true)),
+            ("\u{1c}TRUE\u{1f}", Some(true)),
+            ("\u{a0}False\u{2003}", Some(false)),
+            ("true\u{200b}", None),
+            ("yes", None),
+            ("1", None),
+            ("", None),
+            ("unknown", None),
+        ] {
+            assert_eq!(parse_str_bool(input), expected, "{input:?}");
+        }
+    }
+
     #[test]
     fn adapters_compose_and_serialize_as_numbers() {
         let numbers: Numbers = serde_json::from_value(json!({

litellm-rust/crates/core-utils/src/settings.rs

@@ -1,5 +1,7 @@
 use std::str::FromStr;
 
+use crate::serde_compat::parse_str_bool;
+
 pub trait Lookup {
     fn get(&self, name: &str) -> Option<String>;
 
@@ -9,7 +11,7 @@ pub trait Lookup {
 
     fn enabled(&self, name: &str) -> Option<bool> {
         self.get(name)
-            .is_some_and(|value| value.trim().eq_ignore_ascii_case("true"))
+            .is_some_and(|value| parse_str_bool(&value) == Some(true))
             .then_some(true)
     }
 

litellm-rust/crates/host-python/src/marshal.rs

@@ -45,7 +45,7 @@ where
     fn into_pyobject(self, py: Python<'py>) -> PyResult<Self::Output> {
         catch_unwind(AssertUnwindSafe(|| pythonize::pythonize(py, &self.0)))
             .map_err(panic_to_pyerr)?
-            .map_err(|error| PyValueError::new_err(error.to_string()))
+            .map_err(PyErr::from)
     }
 }
 
@@ -87,6 +87,19 @@ mod tests {
         });
     }
 
+    #[test]
+    fn pythonized_preserves_python_serialization_error_types() {
+        crate::initialize_python();
+        Python::attach(|py| {
+            let value = std::collections::BTreeMap::from([(vec![1], "value")]);
+            let direct = to_py(py, &value).unwrap_err();
+            let wrapped = Pythonized(value).into_pyobject(py).unwrap_err();
+            assert!(direct.is_instance_of::<pyo3::exceptions::PyTypeError>(py));
+            assert!(wrapped.is_instance_of::<pyo3::exceptions::PyTypeError>(py));
+            assert_eq!(wrapped.to_string(), direct.to_string());
+        });
+    }
+
     #[test]
     fn pythonized_maps_serializer_panics_to_a_base_exception() {
         crate::initialize_python();

litellm-rust/crates/http/src/config.rs

@@ -129,6 +129,7 @@ mod tests {
     use rstest::rstest;
 
     use super::*;
+    use crate::TlsSource;
 
     fn settings(ssl_verify: Option<SslVerify>, ssl_cert_file: Option<&str>) -> HttpSettings {
         HttpSettings {
@@ -298,7 +299,11 @@ mod tests {
         };
         assert!(matches!(
             reqwest::ClientBuilder::try_from(&config),
-            Err(Error::Read { path: reported, .. }) if reported == path
+            Err(Error::Read {
+                path: reported,
+                tls_source: TlsSource::CaBundle,
+                ..
+            }) if reported == path
         ));
     }
 
@@ -315,7 +320,11 @@ mod tests {
         std::fs::remove_file(&path).unwrap();
         assert!(matches!(
             result,
-            Err(Error::InvalidPem { path: reported, .. }) if reported == path
+            Err(Error::InvalidPem {
+                path: reported,
+                tls_source: TlsSource::CaBundle,
+                ..
+            }) if reported == path
         ));
     }
 }

litellm-rust/crates/http/src/error.rs

@@ -1,11 +1,25 @@
 use std::path::PathBuf;
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum TlsSource {
+    CaBundle,
+    ClientIdentity,
+}
+
 #[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)]
 pub enum Error {
     #[error("could not read {}: {message}", path.display())]
-    Read { path: PathBuf, message: String },
+    Read {
+        path: PathBuf,
+        message: String,
+        tls_source: TlsSource,
+    },
     #[error("{} is not a PEM file: {message}", path.display())]
-    InvalidPem { path: PathBuf, message: String },
+    InvalidPem {
+        path: PathBuf,
+        message: String,
+        tls_source: TlsSource,
+    },
     #[error("could not build the HTTP client: {0}")]
     Client(String),
     #[error("request body could not be serialized: {0}")]

litellm-rust/crates/http/src/lib.rs

@@ -10,7 +10,7 @@ mod tls;
 pub mod transport;
 
 pub use config::{HttpClientConfig, Resolution, Verify};
-pub use error::Error;
+pub use error::{Error, TlsSource};
 pub use pool::{ClientVariant, HttpClientPool};
 pub use proxy::EnvironmentProxies;
 pub use settings::{HttpSettings, HttpSettingsLayer, SslVerify, TcpKeepalive};

litellm-rust/crates/http/src/media.rs

@@ -54,16 +54,39 @@ impl Default for UrlPolicy {
 impl UrlPolicy {
     fn allows(&self, host: &str, port: u16) -> bool {
         let host = normalize_host(host);
-        let with_port = format!("{host}:{port}");
         self.allowed_hosts
             .iter()
-            .map(|entry| normalize_host(entry))
-            .any(|entry| entry == host || entry == with_port)
+            .filter_map(|entry| parse_allowed_host(entry))
+            .any(|(entry_host, entry_port)| {
+                entry_host == host && entry_port.is_none_or(|entry_port| entry_port == port)
+            })
     }
 }
 
-fn normalize_host(host: &str) -> String {
-    host.to_ascii_lowercase().trim_end_matches('.').to_owned()
+pub fn normalize_host(host: &str) -> String {
+    let host = host.trim().trim_end_matches('.');
+    let host = host
+        .strip_prefix('[')
+        .and_then(|host| host.strip_suffix(']'))
+        .unwrap_or(host);
+    host.to_ascii_lowercase()
+}
+
+fn parse_allowed_host(entry: &str) -> Option<(String, Option<u16>)> {
+    let entry = entry.trim();
+    if let Some(entry) = entry.strip_prefix('[') {
+        let (host, suffix) = entry.split_once(']')?;
+        let port = match suffix {
+            "" => None,
+            suffix => Some(suffix.strip_prefix(':')?.parse().ok()?),
+        };
+        return Some((normalize_host(host), port));
+    }
+    let (host, port) = match entry.rsplit_once(':') {
+        Some((host, port)) if !host.contains(':') => (host, Some(port.parse().ok()?)),
+        _ => (entry, None),
+    };
+    Some((normalize_host(host), port))
 }
 
 type ProxyMatch = Arc<dyn Fn(&Url) -> bool + Send + Sync>;
@@ -670,6 +693,21 @@ mod tests {
         assert!(matches!(result, Err(Error::BlockedUrl)));
     }
 
+    #[test]
+    fn allowlist_matches_bracketed_ipv6_hosts_and_ports() {
+        let policy = UrlPolicy {
+            validate: true,
+            allowed_hosts: vec!["[2001:db8::1]".into(), "[2001:db8::1]:8443".into()],
+        };
+        assert!(policy.allows("2001:db8::1", 443));
+        assert!(policy.allows("2001:db8::1", 8443));
+        let port_specific = UrlPolicy {
+            validate: true,
+            allowed_hosts: vec!["[2001:db8::1]:8443".into()],
+        };
+        assert!(!port_specific.allows("2001:db8::1", 9443));
+    }
+
     #[tokio::test]
     async fn validation_off_fetches_private_hosts_and_follows_redirects() {
         let (url, server, _) = serve_named(

litellm-rust/crates/http/src/settings.rs

@@ -3,7 +3,10 @@ use std::{
     time::Duration,
 };
 
-use litellm_core_utils::settings::{Layer, Lookup, merge};
+use litellm_core_utils::{
+    serde_compat::parse_str_bool,
+    settings::{Layer, Lookup, merge},
+};
 
 use crate::proxy::EnvironmentProxies;
 
@@ -16,9 +19,9 @@ pub enum SslVerify {
 
 impl SslVerify {
     pub fn parse(value: &str) -> Self {
-        match value.trim().to_ascii_lowercase().as_str() {
-            "true" => Self::Enabled,
-            "false" => Self::Disabled,
+        match parse_str_bool(value) {
+            Some(true) => Self::Enabled,
+            Some(false) => Self::Disabled,
             _ => Self::CaBundle(PathBuf::from(value)),
         }
     }
@@ -152,9 +155,7 @@ impl HttpSettings {
         Self {
             ssl_verify: merged.ssl_verify,
             ssl_cert_file: merged.ssl_cert_file,
-            ssl_certificate: merged
-                .ssl_certificate
-                .filter(|path| !path.as_os_str().is_empty()),
+            ssl_certificate: merged.ssl_certificate,
             ssl_security_level: merged.ssl_security_level.filter(|level| !level.is_empty()),
             ssl_ecdh_curve: merged.ssl_ecdh_curve.filter(|curve| !curve.is_empty()),
             force_ipv4: merged.force_ipv4.unwrap_or(defaults.force_ipv4),
@@ -287,7 +288,7 @@ mod tests {
     }
 
     #[test]
-    fn empty_environment_values_clear_the_setting_like_python_truthiness() {
+    fn empty_certificate_is_retained_for_validation_while_empty_tuning_is_absent() {
         let configured = HttpSettingsLayer {
             ssl_certificate: Some("/configured/client.pem".into()),
             ssl_security_level: Some("configured".into()),
@@ -300,7 +301,7 @@ mod tests {
             ("SSL_ECDH_CURVE", ""),
         ]));
         let settings = HttpSettings::from_layers([environment, configured]);
-        assert_eq!(settings.ssl_certificate, None);
+        assert_eq!(settings.ssl_certificate, Some(PathBuf::new()));
         assert_eq!(settings.ssl_security_level, None);
         assert_eq!(settings.ssl_ecdh_curve, None);
     }

litellm-rust/crates/http/src/tls.rs

@@ -9,7 +9,7 @@ use rustls::{
 
 use crate::{
     config::{HttpClientConfig, Verify},
-    error::Error,
+    error::{Error, TlsSource},
 };
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
@@ -197,15 +197,17 @@ impl TryFrom<&HttpClientConfig> for ClientConfig {
             Verify::BuiltInRoots => builder.with_root_certificates(RootCertStore {
                 roots: webpki_roots::TLS_SERVER_ROOTS.to_vec(),
             }),
-            Verify::CaBundle(path) => builder.with_root_certificates(bundle_roots(path)?),
+            Verify::CaBundle(path) => {
+                builder.with_root_certificates(bundle_roots(path, TlsSource::CaBundle)?)
+            }
         };
         let mut tls = match &config.client_certificate {
             None => verified.with_no_client_auth(),
             Some(path) => {
-                let (chain, key) = identity(path)?;
+                let (chain, key) = identity(path, TlsSource::ClientIdentity)?;
                 verified
                     .with_client_auth_cert(chain, key)
-                    .map_err(|error| invalid_pem(path, error))?
+                    .map_err(|error| invalid_pem(path, TlsSource::ClientIdentity, error))?
             }
         };
         tls.alpn_protocols = if config.http2 {
@@ -217,47 +219,52 @@ impl TryFrom<&HttpClientConfig> for ClientConfig {
     }
 }
 
-fn bundle_roots(path: &Path) -> Result<RootCertStore, Error> {
-    let certificates = certificates(path)?;
+fn bundle_roots(path: &Path, source: TlsSource) -> Result<RootCertStore, Error> {
+    let certificates = certificates(path, source)?;
     if certificates.is_empty() {
-        return Err(invalid_pem(path, "no certificates found"));
+        return Err(invalid_pem(path, source, "no certificates found"));
     }
     let mut store = RootCertStore::empty();
     for certificate in certificates {
         store
             .add(certificate)
-            .map_err(|error| invalid_pem(path, error))?;
+            .map_err(|error| invalid_pem(path, source, error))?;
     }
     Ok(store)
 }
 
-fn identity(path: &Path) -> Result<(Vec<CertificateDer<'static>>, PrivateKeyDer<'static>), Error> {
-    let chain = certificates(path)?;
+fn identity(
+    path: &Path,
+    source: TlsSource,
+) -> Result<(Vec<CertificateDer<'static>>, PrivateKeyDer<'static>), Error> {
+    let chain = certificates(path, source)?;
     if chain.is_empty() {
-        return Err(invalid_pem(path, "no certificates found"));
+        return Err(invalid_pem(path, source, "no certificates found"));
     }
-    let key =
-        PrivateKeyDer::from_pem_slice(&read(path)?).map_err(|error| invalid_pem(path, error))?;
+    let key = PrivateKeyDer::from_pem_slice(&read(path, source)?)
+        .map_err(|error| invalid_pem(path, source, error))?;
     Ok((chain, key))
 }
 
-fn certificates(path: &Path) -> Result<Vec<CertificateDer<'static>>, Error> {
-    CertificateDer::pem_slice_iter(&read(path)?)
+fn certificates(path: &Path, source: TlsSource) -> Result<Vec<CertificateDer<'static>>, Error> {
+    CertificateDer::pem_slice_iter(&read(path, source)?)
         .collect::<Result<_, _>>()
-        .map_err(|error| invalid_pem(path, error))
+        .map_err(|error| invalid_pem(path, source, error))
 }
 
-fn read(path: &Path) -> Result<Vec<u8>, Error> {
+fn read(path: &Path, source: TlsSource) -> Result<Vec<u8>, Error> {
     std::fs::read(path).map_err(|error| Error::Read {
         path: path.to_path_buf(),
         message: error.to_string(),
+        tls_source: source,
     })
 }
 
-fn invalid_pem(path: &Path, message: impl fmt::Display) -> Error {
+fn invalid_pem(path: &Path, source: TlsSource, message: impl fmt::Display) -> Error {
     Error::InvalidPem {
         path: path.to_path_buf(),
         message: message.to_string(),
+        tls_source: source,
     }
 }
 
@@ -405,7 +412,11 @@ mod tests {
         std::fs::remove_file(&path).unwrap();
         assert!(matches!(
             result,
-            Err(Error::InvalidPem { path: reported, .. }) if reported == path
+            Err(Error::InvalidPem {
+                path: reported,
+                tls_source: TlsSource::ClientIdentity,
+                ..
+            }) if reported == path
         ));
     }
 }

litellm-rust/crates/python-bridge/Cargo.toml

@@ -21,8 +21,10 @@ tiktoken = ["litellm-token-counter/tiktoken"]
 [dependencies]
 bytes.workspace = true
 litellm-cache.workspace = true
+litellm-cache-azure-blob.workspace = true
 litellm-cache-memory.workspace = true
 litellm-cache-redis.workspace = true
+litellm-cache-disk.workspace = true
 litellm-cache-response.workspace = true
 serde.workspace = true
 litellm-auth.workspace = true
@@ -41,6 +43,8 @@ serde_json.workspace = true
 tokio = { workspace = true, features = ["sync"] }
 
 [dev-dependencies]
+serde.workspace = true
+serde_with.workspace = true
 criterion.workspace = true
 futures-util.workspace = true
 rstest.workspace = true

litellm-rust/crates/python-bridge/python_settings.json

@@ -1,26 +1,154 @@
 {
-  "http_settings": [
-    "ssl_verify",
-    "ssl_certificate",
-    "ssl_security_level",
-    "ssl_ecdh_curve",
-    "force_ipv4",
-    "http2",
-    "aiohttp_trust_env",
-    "disable_aiohttp_trust_env",
-    "disable_aiohttp_transport",
-    "user_agent"
-  ],
-  "url_policy": [
-    "user_url_validation",
-    "user_url_allowed_hosts"
-  ],
-  "provider_defaults": [
-    "vertex_project",
-    "vertex_location",
-    "enable_azure_ad_token_refresh"
-  ],
-  "secret_manager": [
-    "readable"
-  ]
+  "http_settings": {
+    "version": 1,
+    "fields": {
+      "ssl_verify": {
+        "adapter": "SslVerifyInput",
+        "required": true,
+        "precedence": "module_global",
+        "sensitive": false,
+        "shapes": [
+          "none",
+          "bool",
+          "str"
+        ],
+        "unsupported_live": "configuration_error"
+      },
+      "ssl_certificate": {
+        "adapter": "OptionalStrictString",
+        "required": true,
+        "precedence": "module_global",
+        "sensitive": false,
+        "shapes": [],
+        "unsupported_live": null
+      },
+      "ssl_security_level": {
+        "adapter": "TuningString",
+        "required": true,
+        "precedence": "module_global",
+        "sensitive": false,
+        "shapes": [],
+        "unsupported_live": null
+      },
+      "ssl_ecdh_curve": {
+        "adapter": "TuningString",
+        "required": true,
+        "precedence": "module_global",
+        "sensitive": false,
+        "shapes": [],
+        "unsupported_live": null
+      },
+      "force_ipv4": {
+        "adapter": "Truthy",
+        "required": true,
+        "precedence": "module_global",
+        "sensitive": false,
+        "shapes": [],
+        "unsupported_live": null
+      },
+      "http2": {
+        "adapter": "ExactTrue",
+        "required": true,
+        "precedence": "module_global",
+        "sensitive": false,
+        "shapes": [],
+        "unsupported_live": null
+      },
+      "aiohttp_trust_env": {
+        "adapter": "Truthy",
+        "required": true,
+        "precedence": "module_global",
+        "sensitive": false,
+        "shapes": [],
+        "unsupported_live": null
+      },
+      "disable_aiohttp_trust_env": {
+        "adapter": "Truthy",
+        "required": true,
+        "precedence": "module_global",
+        "sensitive": false,
+        "shapes": [],
+        "unsupported_live": null
+      },
+      "disable_aiohttp_transport": {
+        "adapter": "ExactTrue",
+        "required": true,
+        "precedence": "module_global",
+        "sensitive": false,
+        "shapes": [],
+        "unsupported_live": null
+      },
+      "user_agent": {
+        "adapter": "StrictString",
+        "required": true,
+        "precedence": "accessor",
+        "sensitive": false,
+        "shapes": [],
+        "unsupported_live": null
+      }
+    }
+  },
+  "url_policy": {
+    "version": 1,
+    "fields": {
+      "user_url_validation": {
+        "adapter": "Truthy",
+        "required": true,
+        "precedence": "module_global",
+        "sensitive": false,
+        "shapes": [],
+        "unsupported_live": null
+      },
+      "user_url_allowed_hosts": {
+        "adapter": "HostCollection",
+        "required": true,
+        "precedence": "module_global",
+        "sensitive": false,
+        "shapes": [],
+        "unsupported_live": null
+      }
+    }
+  },
+  "provider_defaults": {
+    "version": 1,
+    "fields": {
+      "vertex_project": {
+        "adapter": "FalsyOptionalString",
+        "required": true,
+        "precedence": "module_global",
+        "sensitive": true,
+        "shapes": [],
+        "unsupported_live": null
+      },
+      "vertex_location": {
+        "adapter": "FalsyOptionalString",
+        "required": true,
+        "precedence": "module_global",
+        "sensitive": true,
+        "shapes": [],
+        "unsupported_live": null
+      },
+      "enable_azure_ad_token_refresh": {
+        "adapter": "ExactTrue",
+        "required": true,
+        "precedence": "module_global",
+        "sensitive": false,
+        "shapes": [],
+        "unsupported_live": null
+      }
+    }
+  },
+  "secret_manager": {
+    "version": 1,
+    "fields": {
+      "readable": {
+        "adapter": "StrictBool",
+        "required": true,
+        "precedence": "accessor",
+        "sensitive": false,
+        "shapes": [],
+        "unsupported_live": null
+      }
+    }
+  }
 }

litellm-rust/crates/python-bridge/src/cache/config.rs

@@ -1,10 +1,11 @@
-use std::time::Duration;
+use std::{path::PathBuf, time::Duration};
 
 use litellm_cache::CacheType;
+use litellm_cache_redis::{RedisNode, RedisTopology};
 use pyo3::{
     exceptions::{PyTypeError, PyValueError},
     prelude::*,
-    types::{PyAny, PyDict, PyString},
+    types::{PyAny, PyDict, PyList, PyString},
 };
 
 use super::{native::NativeResponseCache, request::duration};
@@ -25,6 +26,10 @@ pub(super) struct MemoryCacheConfig {
     pub(super) max_entry_bytes: usize,
 }
 
+pub(super) struct DiskCacheConfig {
+    pub(super) directory: PathBuf,
+}
+
 #[derive(Debug, PartialEq)]
 pub(super) enum RedisProtocol {
     Resp2,
@@ -70,12 +75,31 @@ pub(super) struct RedisCacheConfig {
     pub(super) default_ttl: Duration,
     pub(super) namespace: Option<String>,
     pub(super) flush_size: usize,
+    pub(super) topology: RedisTopology,
     pub(super) connection: RedisConnectionConfig,
 }
 
+pub(super) struct AzureBlobCacheConfig {
+    pub(super) account_url: String,
+    pub(super) container: String,
+}
+
+struct RedisClientProjection<'py> {
+    topology: RedisTopology,
+    host: String,
+    port: u16,
+    pool_size: usize,
+    resolved: Bound<'py, PyDict>,
+    tls: Option<RedisTlsConfig>,
+}
+
+const REDIS_PY_DEFAULT_MAX_CONNECTIONS: usize = 1 << 31;
+
 pub(super) enum CacheBackendConfig {
     Memory(MemoryCacheConfig),
     Redis(Box<RedisCacheConfig>),
+    Disk(DiskCacheConfig),
+    AzureBlob(AzureBlobCacheConfig),
 }
 
 #[allow(dead_code, reason = "consumed by the cache activation follow-up")]
@@ -90,6 +114,7 @@ pub(super) enum UnsupportedCacheConfig {
     RedisCredentials,
     RedisConnection,
     RedisOption,
+    DiskStore,
 }
 
 impl UnsupportedCacheConfig {
@@ -100,6 +125,7 @@ impl UnsupportedCacheConfig {
             Self::RedisCredentials => "native Redis credentials require Python",
             Self::RedisConnection => "native Redis connection type is not implemented",
             Self::RedisOption => "native Redis configuration requires Python",
+            Self::DiskStore => "native disk cache requires the built-in diskcache store",
         }
     }
 }
@@ -142,13 +168,24 @@ impl NativeCacheConfig {
                 }))),
                 Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)),
             },
+            Some(CacheType::Disk) => match project_disk(&backend)? {
+                Ok(backend) => Ok(CacheConfigProjection::Native(Box::new(Self {
+                    policy,
+                    backend: CacheBackendConfig::Disk(backend),
+                }))),
+                Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)),
+            },
+            Some(CacheType::AzureBlob) => project_azure_blob(&backend).map(|backend| {
+                CacheConfigProjection::Native(Box::new(Self {
+                    policy,
+                    backend: CacheBackendConfig::AzureBlob(backend),
+                }))
+            }),
             Some(
                 CacheType::RedisSemantic
                 | CacheType::ValkeySemantic
                 | CacheType::S3
-                | CacheType::Disk
                 | CacheType::QdrantSemantic
-                | CacheType::AzureBlob
                 | CacheType::Gcs,
             )
             | None => Ok(CacheConfigProjection::Unsupported(
@@ -158,12 +195,13 @@ impl NativeCacheConfig {
     }
 
     pub(super) fn service_mismatch(&self, service: &NativeResponseCache) -> Option<&'static str> {
-        if service.default_ttl()
-            != Some(match &self.backend {
-                CacheBackendConfig::Memory(config) => config.default_ttl,
-                CacheBackendConfig::Redis(config) => config.default_ttl,
-            })
-        {
+        let default_ttl = match &self.backend {
+            CacheBackendConfig::Memory(config) => Some(config.default_ttl),
+            CacheBackendConfig::Redis(config) => Some(config.default_ttl),
+            CacheBackendConfig::Disk(_) => None,
+            CacheBackendConfig::AzureBlob(_) => None,
+        };
+        if service.default_ttl() != default_ttl {
             return Some("facade and native backend default TTLs must match");
         }
         match &self.backend {
@@ -182,13 +220,51 @@ impl NativeCacheConfig {
             CacheBackendConfig::Redis(_) if service.kind() != "redis" => {
                 Some("facade and native backend types must match")
             }
+            CacheBackendConfig::Redis(config) if service.topology() != Some(&config.topology) => {
+                Some("facade and native backend topologies must match")
+            }
             CacheBackendConfig::Redis(config) => (service.namespace()
                 != config.namespace.as_deref())
             .then_some("facade and native backend namespaces must match"),
+            CacheBackendConfig::Disk(_) if service.kind() != "disk" => {
+                Some("facade and native backend types must match")
+            }
+            CacheBackendConfig::Disk(config) => {
+                let Some(directory) = service.directory() else {
+                    return Some("facade and native backend types must match");
+                };
+                let native = std::fs::canonicalize(directory).ok();
+                let facade = std::fs::canonicalize(&config.directory).ok();
+                (native != facade).then_some("facade and native backend directories must match")
+            }
+            CacheBackendConfig::AzureBlob(config) => match service.azure_blob_identity() {
+                None => Some("facade and native backend types must match"),
+                Some((account_url, container))
+                    if account_url != config.account_url || container != config.container =>
+                {
+                    Some("facade and native backend containers must match")
+                }
+                Some(_) => None,
+            },
         }
     }
 }
 
+#[inline(never)]
+fn project_azure_blob(backend: &Bound<'_, PyAny>) -> PyResult<AzureBlobCacheConfig> {
+    let client = backend.getattr("container_client")?;
+    let container = client.getattr("container_name")?.extract::<String>()?;
+    let url = client.getattr("url")?.extract::<String>()?;
+    let account_url = url
+        .strip_suffix(container.as_str())
+        .and_then(|url| url.strip_suffix('/'))
+        .ok_or_else(|| PyValueError::new_err("Azure Blob container URL is malformed"))?;
+    Ok(AzureBlobCacheConfig {
+        account_url: account_url.to_string(),
+        container,
+    })
+}
+
 #[inline(never)]
 fn project_memory(backend: &Bound<'_, PyAny>) -> PyResult<MemoryCacheConfig> {
     let max_size_kib = backend.getattr("max_size_per_item")?.extract::<usize>()?;
@@ -201,14 +277,26 @@ fn project_memory(backend: &Bound<'_, PyAny>) -> PyResult<MemoryCacheConfig> {
     })
 }
 
+#[inline(never)]
+fn project_disk(
+    backend: &Bound<'_, PyAny>,
+) -> PyResult<Result<DiskCacheConfig, UnsupportedCacheConfig>> {
+    let store = backend.getattr("disk_cache")?;
+    if !instance_class_is(&store, "diskcache.core", "Cache")?
+        || !instance_class_is(&store.getattr("_disk")?, "diskcache.core", "Disk")?
+    {
+        return Ok(Err(UnsupportedCacheConfig::DiskStore));
+    }
+    Ok(Ok(DiskCacheConfig {
+        directory: PathBuf::from(store.getattr("directory")?.extract::<String>()?),
+    }))
+}
+
 #[inline(never)]
 fn project_redis(
     backend: &Bound<'_, PyAny>,
 ) -> PyResult<Result<RedisCacheConfig, UnsupportedCacheConfig>> {
     let source = backend.getattr("redis_kwargs")?.cast_into::<PyDict>()?;
-    if has_value(&source, "startup_nodes")? {
-        return Ok(Err(UnsupportedCacheConfig::RedisTopology));
-    }
     if has_value(&source, "sentinel_nodes")? {
         return Ok(Err(UnsupportedCacheConfig::RedisTopology));
     }
@@ -248,26 +336,25 @@ fn project_redis(
     }
 
     let client = backend.getattr("redis_client")?;
-    let pool = client.getattr("connection_pool")?;
-    if !instance_class_is(&pool, "redis.connection", "ConnectionPool")? {
-        return Ok(Err(UnsupportedCacheConfig::RedisConnection));
-    }
-    let resolved = pool.getattr("connection_kwargs")?.cast_into::<PyDict>()?;
-    for key in ["credential_provider", "redis_connect_func"] {
-        if has_value(&resolved, key)? {
-            return Ok(Err(UnsupportedCacheConfig::RedisCredentials));
-        }
-    }
-    let connection_class = resolved
-        .get_item("connection_class")?
-        .unwrap_or(pool.getattr("connection_class")?);
-    let tls = if class_is(&connection_class, "redis.connection", "Connection")? {
-        None
-    } else if class_is(&connection_class, "redis.connection", "SSLConnection")? {
-        Some(project_tls(&resolved)?)
+    let projection = if has_value(&source, "startup_nodes")? {
+        project_cluster_client(&source, &client)?
     } else {
-        return Ok(Err(UnsupportedCacheConfig::RedisConnection));
+        project_standalone_client(&client)?
     };
+    let RedisClientProjection {
+        topology,
+        host,
+        port,
+        pool_size,
+        resolved,
+        tls,
+    } = match projection {
+        Ok(projection) => projection,
+        Err(reason) => return Ok(Err(reason)),
+    };
+    if has_value(&resolved, "credential_provider")? {
+        return Ok(Err(UnsupportedCacheConfig::RedisCredentials));
+    }
 
     let protocol = match optional_i64(&resolved, "protocol")?.unwrap_or(2) {
         2 => RedisProtocol::Resp2,
@@ -280,15 +367,15 @@ fn project_redis(
         default_ttl: duration(backend.getattr("default_ttl")?.extract::<f64>()?)?,
         namespace: optional_attribute_string(backend, "namespace")?,
         flush_size: backend.getattr("redis_flush_size")?.extract::<usize>()?,
+        topology,
         connection: RedisConnectionConfig {
-            host: required_string(&resolved, "host")?,
-            port: u16::try_from(required_i64(&resolved, "port")?)
-                .map_err(|_| PyValueError::new_err("invalid Redis port"))?,
+            host,
+            port,
             database: optional_i64(&resolved, "db")?.unwrap_or(0),
             username: optional_dict_string(&resolved, "username")?,
             password: optional_dict_string(&resolved, "password")?,
             protocol,
-            pool_size: pool.getattr("max_connections")?.extract::<usize>()?,
+            pool_size,
             read_timeout: optional_dict_duration(&resolved, "socket_timeout")?,
             connect_timeout: optional_dict_duration(&resolved, "socket_connect_timeout")?,
             socket_keepalive: optional_bool(&resolved, "socket_keepalive")?,
@@ -299,6 +386,128 @@ fn project_redis(
     }))
 }
 
+#[inline(never)]
+fn project_standalone_client<'py>(
+    client: &Bound<'py, PyAny>,
+) -> PyResult<Result<RedisClientProjection<'py>, UnsupportedCacheConfig>> {
+    let pool = client.getattr("connection_pool")?;
+    if !instance_class_is(&pool, "redis.connection", "ConnectionPool")? {
+        return Ok(Err(UnsupportedCacheConfig::RedisConnection));
+    }
+    let resolved = pool.getattr("connection_kwargs")?.cast_into::<PyDict>()?;
+    if has_value(&resolved, "redis_connect_func")? {
+        return Ok(Err(UnsupportedCacheConfig::RedisCredentials));
+    }
+    let connection_class = resolved
+        .get_item("connection_class")?
+        .unwrap_or(pool.getattr("connection_class")?);
+    let tls = if class_is(&connection_class, "redis.connection", "Connection")? {
+        None
+    } else if class_is(&connection_class, "redis.connection", "SSLConnection")? {
+        Some(project_tls(&resolved)?)
+    } else {
+        return Ok(Err(UnsupportedCacheConfig::RedisConnection));
+    };
+    Ok(Ok(RedisClientProjection {
+        topology: RedisTopology::Standalone,
+        host: required_string(&resolved, "host")?,
+        port: port(required_i64(&resolved, "port")?)?,
+        pool_size: pool.getattr("max_connections")?.extract::<usize>()?,
+        resolved,
+        tls,
+    }))
+}
+
+#[inline(never)]
+fn project_cluster_client<'py>(
+    source: &Bound<'py, PyDict>,
+    client: &Bound<'py, PyAny>,
+) -> PyResult<Result<RedisClientProjection<'py>, UnsupportedCacheConfig>> {
+    let Some(startup_nodes) = startup_nodes(source)? else {
+        return Ok(Err(UnsupportedCacheConfig::RedisTopology));
+    };
+    if !instance_class_is(client, "redis.cluster", "RedisCluster")? {
+        return Ok(Err(UnsupportedCacheConfig::RedisConnection));
+    }
+    let nodes = client.getattr("nodes_manager")?;
+    if !class_is(
+        &nodes.getattr("connection_pool_class")?,
+        "redis.connection",
+        "ConnectionPool",
+    )? {
+        return Ok(Err(UnsupportedCacheConfig::RedisConnection));
+    }
+    let resolved = nodes.getattr("connection_kwargs")?.cast_into::<PyDict>()?;
+    if let Some(connect) = resolved.get_item("redis_connect_func")?
+        && !connect.is_none()
+    {
+        let own_hook = connect
+            .getattr("__self__")
+            .is_ok_and(|owner| owner.is(client))
+            && connect
+                .getattr("__func__")
+                .and_then(|function| Ok(function.is(&client.get_type().getattr("on_connect")?)))
+                .unwrap_or(false);
+        if !own_hook {
+            return Ok(Err(UnsupportedCacheConfig::RedisCredentials));
+        }
+    }
+    let tls = if optional_bool(&resolved, "ssl")?.unwrap_or(false) {
+        Some(project_tls(&resolved)?)
+    } else {
+        None
+    };
+    let first = &startup_nodes[0];
+    Ok(Ok(RedisClientProjection {
+        host: first.host.clone(),
+        port: first.port,
+        pool_size: optional_i64(&resolved, "max_connections")?
+            .map(|value| {
+                usize::try_from(value).map_err(|_| PyValueError::new_err("invalid Redis pool size"))
+            })
+            .transpose()?
+            .unwrap_or(REDIS_PY_DEFAULT_MAX_CONNECTIONS),
+        topology: RedisTopology::Cluster { startup_nodes },
+        resolved,
+        tls,
+    }))
+}
+
+#[inline(never)]
+fn startup_nodes(source: &Bound<'_, PyDict>) -> PyResult<Option<Vec<RedisNode>>> {
+    let Some(nodes) = source.get_item("startup_nodes")? else {
+        return Ok(None);
+    };
+    let Ok(nodes) = nodes.cast_into::<PyList>() else {
+        return Ok(None);
+    };
+    if nodes.is_empty() {
+        return Ok(None);
+    }
+    let mut parsed = Vec::with_capacity(nodes.len());
+    for node in nodes.iter() {
+        let Ok(node) = node.cast_into::<PyDict>() else {
+            return Ok(None);
+        };
+        if node.len() != 2 || !has_value(&node, "host")? || !has_value(&node, "port")? {
+            return Ok(None);
+        }
+        let (Ok(host), Ok(port)) = (
+            required_string(&node, "host"),
+            required_i64(&node, "port").and_then(port),
+        ) else {
+            return Ok(None);
+        };
+        parsed.push(RedisNode { host, port });
+    }
+    Ok(Some(parsed))
+}
+
+#[inline(never)]
+fn port(value: i64) -> PyResult<u16> {
+    u16::try_from(value).map_err(|_| PyValueError::new_err("invalid Redis port"))
+}
+
 #[inline(never)]
 fn project_tls(values: &Bound<'_, PyDict>) -> PyResult<RedisTlsConfig> {
     Ok(RedisTlsConfig {
@@ -467,12 +676,27 @@ mod tests {
 
     use pyo3::{prelude::*, types::PyDict};
 
+    use litellm_cache_redis::{RedisNode, RedisTopology};
+
     use super::{
-        CacheBackendConfig, CacheConfigProjection, CertificateRequirement, NativeCacheConfig,
-        RedisProtocol,
+        CacheBackendConfig, CacheConfigProjection, CachePolicy, CertificateRequirement,
+        DiskCacheConfig, NativeCacheConfig, RedisProtocol,
     };
     use crate::cache::native::NativeResponseCache;
 
+    fn cluster_facade<'py>(py: Python<'py>, startup_nodes: &str, hook: &str) -> Bound<'py, PyAny> {
+        facade(
+            py,
+            &format!(
+                "RedisCluster = type('RedisCluster', (), {{'__module__': 'redis.cluster', 'on_connect': lambda self, connection: None}})\n\
+                 client = RedisCluster()\n\
+                 client.nodes_manager = SimpleNamespace(connection_pool_class=ConnectionPool, connection_kwargs={{'password': 'secret', 'redis_connect_func': {hook}, 'protocol': 3, 'ssl': True, 'ssl_cert_reqs': 'none'}})\n\
+                 backend = SimpleNamespace(default_ttl=120, namespace='team', redis_flush_size=100, redis_kwargs={{'startup_nodes': {startup_nodes}, 'password': 'secret'}}, redis_client=client)\n\
+                 facade = SimpleNamespace(type='redis', mode='default-on', ttl=None, namespace='team', supported_call_types=None, redis_flush_size=100, semantic_cache_scope='key', cache=backend)"
+            ),
+        )
+    }
+
     fn facade<'py>(py: Python<'py>, body: &str) -> Bound<'py, PyAny> {
         let locals = PyDict::new(py);
         py.run(
@@ -591,4 +815,166 @@ mod tests {
             assert_eq!(reason.message(), "native Redis credentials require Python");
         });
     }
+    #[test]
+    fn projects_builtin_disk_configuration_and_rejects_custom_stores() {
+        Python::initialize();
+        Python::attach(|py| {
+            let root =
+                std::env::temp_dir().join(format!("litellm-disk-config-{}", std::process::id()));
+            let directory = root.to_string_lossy();
+            let disk_facade = facade(
+                py,
+                &format!(
+                    "Cache = type('Cache', (), {{'__module__': 'diskcache.core'}})\n\
+                     Disk = type('Disk', (), {{'__module__': 'diskcache.core'}})\n\
+                     store = Cache()\n\
+                     store._disk = Disk()\n\
+                     store.directory = {directory:?}\n\
+                     backend = SimpleNamespace(disk_cache=store)\n\
+                     facade = SimpleNamespace(type='disk', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)"
+                ),
+            );
+            let CacheConfigProjection::Native(config) =
+                NativeCacheConfig::project(&disk_facade).unwrap()
+            else {
+                panic!("disk cache should be supported");
+            };
+            let CacheBackendConfig::Disk(disk) = config.backend else {
+                panic!("expected disk configuration");
+            };
+            assert_eq!(disk.directory, root);
+            let matching = NativeResponseCache::disk(&directory).unwrap();
+            assert_eq!(
+                (NativeCacheConfig {
+                    policy: config.policy,
+                    backend: CacheBackendConfig::Disk(disk),
+                })
+                .service_mismatch(&matching),
+                None
+            );
+            let other = NativeResponseCache::disk(&root.join("other").to_string_lossy()).unwrap();
+            let mismatch = NativeCacheConfig {
+                policy: CachePolicy {
+                    mode: "default-on".into(),
+                    ttl: None,
+                    namespace: None,
+                    supported_call_types: None,
+                    redis_flush_size: None,
+                    semantic_cache_scope: "key".into(),
+                },
+                backend: CacheBackendConfig::Disk(DiskCacheConfig {
+                    directory: root.clone(),
+                }),
+            };
+            assert_eq!(
+                mismatch.service_mismatch(&other),
+                Some("facade and native backend directories must match")
+            );
+
+            let custom = facade(
+                py,
+                &format!(
+                    "CustomCache = type('CustomCache', (), {{'__module__': 'mypkg'}})\n\
+                     CustomDisk = type('CustomDisk', (), {{'__module__': 'mypkg'}})\n\
+                     store = CustomCache()\n\
+                     store._disk = CustomDisk()\n\
+                     store.directory = {directory:?}\n\
+                     backend = SimpleNamespace(disk_cache=store)\n\
+                     facade = SimpleNamespace(type='disk', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)"
+                ),
+            );
+            let CacheConfigProjection::Unsupported(reason) =
+                NativeCacheConfig::project(&custom).unwrap()
+            else {
+                panic!("custom disk store must stay on Python");
+            };
+            assert_eq!(
+                reason.message(),
+                "native disk cache requires the built-in diskcache store"
+            );
+        });
+    }
+
+    #[test]
+    fn projects_cluster_startup_nodes_as_redis_topology() {
+        Python::initialize();
+        Python::attach(|py| {
+            let facade = cluster_facade(
+                py,
+                "[{'host': 'node-a', 'port': 7000}, {'host': 'node-b', 'port': 7001}]",
+                "client.on_connect",
+            );
+            let CacheConfigProjection::Native(config) =
+                NativeCacheConfig::project(&facade).unwrap()
+            else {
+                panic!("cluster startup nodes should project natively");
+            };
+            let CacheBackendConfig::Redis(redis) = &config.backend else {
+                panic!("expected Redis configuration");
+            };
+            let expected = RedisTopology::Cluster {
+                startup_nodes: vec![
+                    RedisNode {
+                        host: "node-a".into(),
+                        port: 7000,
+                    },
+                    RedisNode {
+                        host: "node-b".into(),
+                        port: 7001,
+                    },
+                ],
+            };
+            assert_eq!(redis.topology, expected);
+            assert_eq!(redis.connection.host, "node-a");
+            assert_eq!(redis.connection.port, 7000);
+            assert_eq!(redis.connection.password.as_deref(), Some("secret"));
+            assert_eq!(redis.connection.protocol, RedisProtocol::Resp3);
+            assert_eq!(
+                redis
+                    .connection
+                    .tls
+                    .as_ref()
+                    .unwrap()
+                    .certificate_requirement,
+                CertificateRequirement::None
+            );
+        });
+    }
+
+    #[test]
+    fn malformed_startup_nodes_and_foreign_connect_hooks_stay_on_python() {
+        Python::initialize();
+        Python::attach(|py| {
+            for (startup_nodes, hook, message) in [
+                (
+                    "[{'host': 'node-a', 'port': 7000, 'server_type': 'primary'}]",
+                    "client.on_connect",
+                    "native Redis topology is not implemented",
+                ),
+                (
+                    "[{'host': 'node-a', 'port': 'seven'}]",
+                    "client.on_connect",
+                    "native Redis topology is not implemented",
+                ),
+                (
+                    "[]",
+                    "client.on_connect",
+                    "native Redis topology is not implemented",
+                ),
+                (
+                    "[{'host': 'node-a', 'port': 7000}]",
+                    "lambda connection: None",
+                    "native Redis credentials require Python",
+                ),
+            ] {
+                let facade = cluster_facade(py, startup_nodes, hook);
+                let CacheConfigProjection::Unsupported(reason) =
+                    NativeCacheConfig::project(&facade).unwrap()
+                else {
+                    panic!("{startup_nodes} with {hook} must stay on Python");
+                };
+                assert_eq!(reason.message(), message, "{startup_nodes} with {hook}");
+            }
+        });
+    }
 }

litellm-rust/crates/python-bridge/src/cache/facade.rs

@@ -1,3 +1,4 @@
+use litellm_cache_redis::RedisTopology;
 use litellm_host_python::from_py;
 use pyo3::{
     PyTraverseError, PyVisit,
@@ -29,13 +30,50 @@ struct RedisPoolGuard {
     reference: Py<PyAny>,
     connection_class: Py<PyAny>,
     connection_kwargs: Py<PyAny>,
-    max_connections: usize,
+    max_connections: Option<usize>,
+    attributes: RedisPoolAttributes,
 }
 
+struct DiskStoreGuard {
+    reference: Py<PyAny>,
+    directory: String,
+}
+
+struct AzureBlobClientGuard {
+    sync_client: Py<PyAny>,
+    async_client: Py<PyAny>,
+    url: String,
+    container_name: String,
+}
+
+enum ConnectionGuard {
+    None,
+    RedisPool(RedisPoolGuard),
+    AzureBlob(AzureBlobClientGuard),
+}
+struct RedisPoolAttributes {
+    pool: &'static str,
+    connection_class: &'static str,
+    max_connections: Option<&'static str>,
+}
+
+const STANDALONE_POOL: RedisPoolAttributes = RedisPoolAttributes {
+    pool: "connection_pool",
+    connection_class: "connection_class",
+    max_connections: Some("max_connections"),
+};
+
+const CLUSTER_POOL: RedisPoolAttributes = RedisPoolAttributes {
+    pool: "nodes_manager",
+    connection_class: "connection_pool_class",
+    max_connections: None,
+};
+
 pub(super) struct FacadeGuard {
     outer: ObjectGuard,
     backend: ObjectGuard,
-    redis_pool: Option<RedisPoolGuard>,
+    disk_store: Option<DiskStoreGuard>,
+    connection: ConnectionGuard,
 }
 
 impl ObjectGuard {
@@ -138,31 +176,40 @@ impl ObjectGuard {
 }
 
 impl RedisPoolGuard {
-    fn capture(backend: &Bound<'_, PyAny>) -> PyResult<Self> {
-        let pool = backend
-            .getattr("redis_client")?
-            .getattr("connection_pool")?;
+    fn capture(backend: &Bound<'_, PyAny>, attributes: RedisPoolAttributes) -> PyResult<Self> {
+        let pool = backend.getattr("redis_client")?.getattr(attributes.pool)?;
         Ok(Self {
             reference: pool.clone().unbind(),
-            connection_class: pool.getattr("connection_class")?.unbind(),
+            connection_class: pool.getattr(attributes.connection_class)?.unbind(),
             connection_kwargs: pool
                 .getattr("connection_kwargs")?
                 .call_method0("copy")?
                 .unbind(),
-            max_connections: pool.getattr("max_connections")?.extract::<usize>()?,
+            max_connections: Self::max_connections(&pool, &attributes)?,
+            attributes,
         })
     }
 
+    fn max_connections(
+        pool: &Bound<'_, PyAny>,
+        attributes: &RedisPoolAttributes,
+    ) -> PyResult<Option<usize>> {
+        attributes
+            .max_connections
+            .map(|name| pool.getattr(name)?.extract::<usize>())
+            .transpose()
+    }
+
     fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult<bool> {
         let pool = backend
             .getattr("redis_client")?
-            .getattr("connection_pool")?;
+            .getattr(self.attributes.pool)?;
         Ok(self.reference.bind(py).is(&pool)
             && self
                 .connection_class
                 .bind(py)
-                .is(&pool.getattr("connection_class")?)
-            && self.max_connections == pool.getattr("max_connections")?.extract::<usize>()?
+                .is(&pool.getattr(self.attributes.connection_class)?)
+            && self.max_connections == Self::max_connections(&pool, &self.attributes)?
             && self
                 .connection_kwargs
                 .bind(py)
@@ -176,6 +223,81 @@ impl RedisPoolGuard {
     }
 }
 
+impl DiskStoreGuard {
+    fn capture(backend: &Bound<'_, PyAny>) -> PyResult<Self> {
+        let store = backend.getattr("disk_cache")?;
+        Ok(Self {
+            reference: store.clone().unbind(),
+            directory: store.getattr("directory")?.extract()?,
+        })
+    }
+
+    fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult<bool> {
+        let store = backend.getattr("disk_cache")?;
+        Ok(self.reference.bind(py).is(&store)
+            && self.directory == store.getattr("directory")?.extract::<String>()?)
+    }
+
+    fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
+        visit.call(&self.reference)
+    }
+}
+
+impl AzureBlobClientGuard {
+    fn capture(backend: &Bound<'_, PyAny>) -> PyResult<Self> {
+        let sync_client = backend.getattr("container_client")?;
+        Ok(Self {
+            url: sync_client.getattr("url")?.extract::<String>()?,
+            container_name: sync_client.getattr("container_name")?.extract::<String>()?,
+            sync_client: sync_client.unbind(),
+            async_client: backend.getattr("async_container_client")?.unbind(),
+        })
+    }
+
+    fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult<bool> {
+        let sync_client = backend.getattr("container_client")?;
+        Ok(self.sync_client.bind(py).is(&sync_client)
+            && self
+                .async_client
+                .bind(py)
+                .is(&backend.getattr("async_container_client")?)
+            && self.url == sync_client.getattr("url")?.extract::<String>()?
+            && self.container_name == sync_client.getattr("container_name")?.extract::<String>()?)
+    }
+
+    fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
+        visit.call(&self.sync_client)?;
+        visit.call(&self.async_client)
+    }
+}
+
+impl ConnectionGuard {
+    fn capture(kind: &str, cluster: bool, backend: &Bound<'_, PyAny>) -> PyResult<Self> {
+        Ok(match (kind, cluster) {
+            ("redis", false) => Self::RedisPool(RedisPoolGuard::capture(backend, STANDALONE_POOL)?),
+            ("redis", true) => Self::RedisPool(RedisPoolGuard::capture(backend, CLUSTER_POOL)?),
+            ("azure-blob", _) => Self::AzureBlob(AzureBlobClientGuard::capture(backend)?),
+            _ => Self::None,
+        })
+    }
+
+    fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult<bool> {
+        match self {
+            Self::None => Ok(true),
+            Self::RedisPool(guard) => guard.matches(py, backend),
+            Self::AzureBlob(guard) => guard.matches(py, backend),
+        }
+    }
+
+    fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
+        match self {
+            Self::None => Ok(()),
+            Self::RedisPool(guard) => guard.traverse(visit),
+            Self::AzureBlob(guard) => guard.traverse(visit),
+        }
+    }
+}
+
 impl FacadeGuard {
     pub(super) fn capture(
         py: Python<'_>,
@@ -189,9 +311,21 @@ impl FacadeGuard {
                 "only exact built-in Cache facades can be registered",
             ));
         }
-        let (module, name, cache_kind) = match kind {
-            "memory" => ("litellm.caching.in_memory_cache", "InMemoryCache", "local"),
-            "redis" => ("litellm.caching.redis_cache", "RedisCache", "redis"),
+        let cluster = matches!(service.topology(), Some(RedisTopology::Cluster { .. }));
+        let (module, name, cache_kind) = match (kind, cluster) {
+            ("memory", _) => ("litellm.caching.in_memory_cache", "InMemoryCache", "local"),
+            ("redis", false) => ("litellm.caching.redis_cache", "RedisCache", "redis"),
+            ("redis", true) => (
+                "litellm.caching.redis_cluster_cache",
+                "RedisClusterCache",
+                "redis",
+            ),
+            ("disk", _) => ("litellm.caching.disk_cache", "DiskCache", "disk"),
+            ("azure-blob", _) => (
+                "litellm.caching.azure_blob_cache",
+                "AzureBlobCache",
+                "azure-blob",
+            ),
             _ => unreachable!(),
         };
         let backend = facade.getattr("cache")?;
@@ -237,9 +371,10 @@ impl FacadeGuard {
                     "redis_flush_size",
                 ],
             )?,
-            redis_pool: (kind == "redis")
-                .then(|| RedisPoolGuard::capture(&backend))
+            disk_store: (kind == "disk")
+                .then(|| DiskStoreGuard::capture(&backend))
                 .transpose()?,
+            connection: ConnectionGuard::capture(kind, cluster, &backend)?,
         })
     }
 
@@ -251,19 +386,21 @@ impl FacadeGuard {
         if !self.backend.matches(py, &backend)? {
             return Ok(false);
         }
-        match &self.redis_pool {
-            Some(guard) => guard.matches(py, &backend),
-            None => Ok(true),
+        if let Some(guard) = &self.disk_store
+            && !guard.matches(py, &backend)?
+        {
+            return Ok(false);
         }
+        self.connection.matches(py, &backend)
     }
 
     pub(super) fn traverse(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> {
         self.outer.traverse(&visit)?;
         self.backend.traverse(&visit)?;
-        if let Some(guard) = &self.redis_pool {
+        if let Some(guard) = &self.disk_store {
             guard.traverse(&visit)?;
         }
-        Ok(())
+        self.connection.traverse(&visit)
     }
 }
 

litellm-rust/crates/python-bridge/src/cache/handle.rs

@@ -1,4 +1,5 @@
-use litellm_host_python::release_gil;
+use litellm_cache_redis::{RedisNode, RedisTopology};
+use litellm_host_python::{release_gil, run_sync_value};
 use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*};
 
 use super::{cache_error, facade::FacadeGuard, native::NativeResponseCache, request::duration};
@@ -34,16 +35,55 @@ impl CacheTestHandle {
     }
 
     #[staticmethod]
-    #[pyo3(signature = (url, *, ttl_seconds=60.0, namespace=None))]
+    #[pyo3(signature = (url, *, ttl_seconds=60.0, namespace=None, startup_nodes=None))]
     fn redis(
         py: Python<'_>,
         url: String,
         ttl_seconds: f64,
         namespace: Option<String>,
+        startup_nodes: Option<Vec<(String, u16)>>,
     ) -> PyResult<Self> {
         let ttl = Some(duration(ttl_seconds)?);
-        let service = release_gil(py, move || NativeResponseCache::redis(&url, ttl, namespace))
-            .map_err(cache_error)?;
+        let topology = match startup_nodes {
+            None => RedisTopology::Standalone,
+            Some(nodes) => RedisTopology::Cluster {
+                startup_nodes: nodes
+                    .into_iter()
+                    .map(|(host, port)| RedisNode { host, port })
+                    .collect(),
+            },
+        };
+        let service = release_gil(py, move || {
+            NativeResponseCache::redis(&url, &topology, ttl, namespace)
+        })
+        .map_err(cache_error)?;
+        Ok(Self {
+            service,
+            guard: None,
+            pid: std::process::id(),
+        })
+    }
+
+    #[staticmethod]
+    #[pyo3(signature = (directory))]
+    fn disk(py: Python<'_>, directory: String) -> PyResult<Self> {
+        let service =
+            release_gil(py, move || NativeResponseCache::disk(&directory)).map_err(cache_error)?;
+        Ok(Self {
+            service,
+            guard: None,
+            pid: std::process::id(),
+        })
+    }
+
+    #[staticmethod]
+    #[pyo3(signature = (account_url, container))]
+    fn azure_blob(py: Python<'_>, account_url: String, container: String) -> PyResult<Self> {
+        let service = run_sync_value(py, async move {
+            NativeResponseCache::azure_blob(&account_url, &container)
+                .await
+                .map_err(cache_error)
+        })?;
         Ok(Self {
             service,
             guard: None,

litellm-rust/crates/python-bridge/src/cache/native.rs

@@ -1,8 +1,10 @@
-use std::{sync::Arc, time::Duration};
+use std::{path::Path, sync::Arc, time::Duration};
 
 use litellm_cache::{CacheCodec, CacheConnectionResult, Error};
+use litellm_cache_azure_blob::AzureBlobCache;
+use litellm_cache_disk::DiskCache;
 use litellm_cache_memory::InMemoryCache;
-use litellm_cache_redis::RedisCache;
+use litellm_cache_redis::{RedisCache, RedisTopology};
 use litellm_cache_response::{
     CacheEntry, PartialHits, ResponseCache, ResponseCacheCodec, ResponseCacheRequest, WriteBuffer,
 };
@@ -15,6 +17,8 @@ pub(super) enum NativeResponseCache {
         cache: Arc<ResponseCache<RedisCache<ResponseCacheCodec>>>,
         buffer: Option<Arc<WriteBuffer>>,
     },
+    Disk(Arc<ResponseCache<DiskCache<ResponseCacheCodec>>>),
+    AzureBlob(Arc<ResponseCache<AzureBlobCache<ResponseCacheCodec>>>),
 }
 
 impl NativeResponseCache {
@@ -34,15 +38,44 @@ impl NativeResponseCache {
 
     pub fn redis(
         url: &str,
+        topology: &RedisTopology,
         ttl: Option<Duration>,
         namespace: Option<String>,
     ) -> Result<Self, Error> {
-        let backend = RedisCache::new(url, ttl, ResponseCacheCodec)?.with_namespace(namespace);
+        let backend =
+            RedisCache::connect(url, topology, ttl, ResponseCacheCodec)?.with_namespace(namespace);
         Ok(Self::Redis {
             cache: Arc::new(ResponseCache::new(Arc::new(backend))),
             buffer: None,
         })
     }
+    pub fn disk(directory: &str) -> Result<Self, Error> {
+        let cache = DiskCache::open(directory, ResponseCacheCodec)?;
+        Ok(Self::Disk(Arc::new(ResponseCache::new(Arc::new(cache)))))
+    }
+
+    pub async fn azure_blob(account_url: &str, container: &str) -> Result<Self, Error> {
+        let backend = AzureBlobCache::connect(
+            account_url,
+            container,
+            ResponseCacheCodec,
+            tokio::runtime::Handle::current(),
+        )
+        .await?;
+        Ok(Self::AzureBlob(Arc::new(ResponseCache::new(Arc::new(
+            backend,
+        )))))
+    }
+
+    pub fn azure_blob_identity(&self) -> Option<(&str, &str)> {
+        match self {
+            Self::AzureBlob(cache) => Some((
+                cache.backend().account_url(),
+                cache.backend().container_name(),
+            )),
+            Self::Memory(_) | Self::Redis { .. } | Self::Disk(_) => None,
+        }
+    }
 }
 
 impl NativeResponseCache {
@@ -50,6 +83,8 @@ impl NativeResponseCache {
         match self {
             Self::Memory(_) => "memory",
             Self::Redis { .. } => "redis",
+            Self::Disk(_) => "disk",
+            Self::AzureBlob(_) => "azure-blob",
         }
     }
 
@@ -57,27 +92,36 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.default_ttl(),
             Self::Redis { cache, .. } => cache.default_ttl(),
+            Self::Disk(cache) => cache.default_ttl(),
+            Self::AzureBlob(cache) => cache.default_ttl(),
         }
     }
 
     pub fn namespace(&self) -> Option<&str> {
         match self {
-            Self::Memory(_) => None,
+            Self::Memory(_) | Self::Disk(_) | Self::AzureBlob(_) => None,
             Self::Redis { cache, .. } => cache.backend().namespace(),
         }
     }
 
+    pub fn topology(&self) -> Option<&RedisTopology> {
+        match self {
+            Self::Memory(_) | Self::Disk(_) | Self::AzureBlob(_) => None,
+            Self::Redis { cache, .. } => Some(cache.backend().topology()),
+        }
+    }
+
     pub fn capacity(&self) -> Option<usize> {
         match self {
             Self::Memory(cache) => Some(cache.backend().max_size_in_memory()),
-            Self::Redis { .. } => None,
+            Self::Redis { .. } | Self::Disk(_) | Self::AzureBlob(_) => None,
         }
     }
 
     pub fn max_entry_bytes(&self) -> Option<usize> {
         match self {
             Self::Memory(cache) => cache.backend().max_entry_bytes(),
-            Self::Redis { .. } => None,
+            Self::Redis { .. } | Self::Disk(_) | Self::AzureBlob(_) => None,
         }
     }
 
@@ -87,7 +131,14 @@ impl NativeResponseCache {
                 cache,
                 buffer: flush_size.map(|flush_size| Arc::new(WriteBuffer::new(flush_size))),
             },
-            memory => memory,
+            other => other,
+        }
+    }
+
+    pub fn directory(&self) -> Option<&Path> {
+        match self {
+            Self::Disk(cache) => Some(cache.backend().directory()),
+            Self::Memory(_) | Self::Redis { .. } | Self::AzureBlob(_) => None,
         }
     }
 
@@ -99,6 +150,8 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.lookup(request, now),
             Self::Redis { cache, .. } => cache.lookup(request, now),
+            Self::Disk(cache) => cache.lookup(request, now),
+            Self::AzureBlob(cache) => cache.lookup(request, now),
         }
     }
 
@@ -111,6 +164,8 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.store(request, response, now),
             Self::Redis { cache, .. } => cache.store(request, response, now),
+            Self::Disk(cache) => cache.store(request, response, now),
+            Self::AzureBlob(cache) => cache.store(request, response, now),
         }
     }
 
@@ -122,6 +177,8 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.lookup_batch(requests, now),
             Self::Redis { cache, .. } => cache.lookup_batch(requests, now),
+            Self::Disk(cache) => cache.lookup_batch(requests, now),
+            Self::AzureBlob(cache) => cache.lookup_batch(requests, now),
         }
     }
 
@@ -133,6 +190,8 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.async_lookup(request, now).await,
             Self::Redis { cache, .. } => cache.async_lookup(request, now).await,
+            Self::Disk(cache) => cache.async_lookup(request, now).await,
+            Self::AzureBlob(cache) => cache.async_lookup(request, now).await,
         }
     }
 
@@ -152,6 +211,8 @@ impl NativeResponseCache {
                 cache,
                 buffer: Some(buffer),
             } => buffer.async_store(cache, request, response, now).await,
+            Self::Disk(cache) => cache.async_store(request, response, now).await,
+            Self::AzureBlob(cache) => cache.async_store(request, response, now).await,
         }
     }
 
@@ -163,6 +224,8 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.async_lookup_batch(requests, now).await,
             Self::Redis { cache, .. } => cache.async_lookup_batch(requests, now).await,
+            Self::Disk(cache) => cache.async_lookup_batch(requests, now).await,
+            Self::AzureBlob(cache) => cache.async_lookup_batch(requests, now).await,
         }
     }
 
@@ -174,6 +237,8 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.async_store_batch(entries, now).await,
             Self::Redis { cache, .. } => cache.async_store_batch(entries, now).await,
+            Self::Disk(cache) => cache.async_store_batch(entries, now).await,
+            Self::AzureBlob(cache) => cache.async_store_batch(entries, now).await,
         }
     }
 
@@ -186,6 +251,8 @@ impl NativeResponseCache {
                 }
                 cache.async_flush().await
             }
+            Self::Disk(cache) => cache.async_flush().await,
+            Self::AzureBlob(cache) => cache.async_flush().await,
         }
     }
 
@@ -193,6 +260,8 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.test_connection().await,
             Self::Redis { cache, .. } => cache.test_connection().await,
+            Self::Disk(cache) => cache.test_connection().await,
+            Self::AzureBlob(cache) => cache.test_connection().await,
         }
     }
 }

litellm-rust/crates/python-bridge/src/coercion.rs

@@ -0,0 +1,231 @@
+use std::collections::BTreeSet;
+
+use litellm_core_utils::serde_compat::parse_str_bool;
+use litellm_http::SslVerify;
+use pyo3::{
+    exceptions::{PyAttributeError, PyRuntimeError, PyValueError},
+    prelude::*,
+    types::{PyBool, PyString},
+};
+
+#[derive(Debug)]
+pub(crate) enum ProjectionError {
+    Python(PyErr),
+    InvalidConfiguration(String),
+    UnsupportedLiveObject(String),
+    InternalSchemaFailure(String),
+}
+
+impl From<PyErr> for ProjectionError {
+    fn from(error: PyErr) -> Self {
+        Self::Python(error)
+    }
+}
+
+impl From<ProjectionError> for PyErr {
+    fn from(error: ProjectionError) -> Self {
+        match error {
+            ProjectionError::Python(error) => error,
+            ProjectionError::InvalidConfiguration(message)
+            | ProjectionError::UnsupportedLiveObject(message) => PyValueError::new_err(message),
+            ProjectionError::InternalSchemaFailure(message) => PyRuntimeError::new_err(message),
+        }
+    }
+}
+
+pub(crate) struct Truthy(pub bool);
+pub(crate) struct ExactTrue(pub bool);
+pub(crate) struct StrBool(pub Option<bool>);
+pub(crate) struct OptionalStrictString(pub Option<String>);
+pub(crate) struct FalsyOptionalString(pub Option<String>);
+pub(crate) struct TuningString(pub Option<String>);
+pub(crate) struct StringCollection(pub Vec<String>);
+pub(crate) struct SslVerifyInput(pub Option<SslVerify>);
+
+pub(crate) struct Field<'py> {
+    path: &'static str,
+    value: Bound<'py, PyAny>,
+}
+
+impl<'py> Field<'py> {
+    pub(crate) fn new(path: &'static str, value: Bound<'py, PyAny>) -> Self {
+        Self { path, value }
+    }
+
+    pub(crate) fn read(
+        snapshot: &Bound<'py, PyAny>,
+        path: &'static str,
+    ) -> Result<Self, ProjectionError> {
+        let name = path.rsplit('.').next().unwrap_or(path);
+        match snapshot.getattr(name) {
+            Ok(value) => Ok(Self::new(path, value)),
+            Err(error) if error.is_instance_of::<PyAttributeError>(snapshot.py()) => {
+                match Self::missing_field(snapshot, name) {
+                    Ok(true) => Err(ProjectionError::InternalSchemaFailure(format!(
+                        "{path}: missing snapshot field"
+                    ))),
+                    _ => Err(error.into()),
+                }
+            }
+            Err(error) => Err(error.into()),
+        }
+    }
+
+    fn missing_field(snapshot: &Bound<'_, PyAny>, name: &str) -> PyResult<bool> {
+        let py = snapshot.py();
+        let object = py.import("builtins")?.getattr("object")?;
+        let missing = object.call0()?;
+        let lookup = py.import("inspect")?.getattr("getattr_static")?;
+        let declared = lookup.call1((snapshot, name, &missing))?;
+        let fallback = lookup.call1((snapshot.get_type(), "__getattr__", &missing))?;
+        let getter = lookup.call1((snapshot.get_type(), "__getattribute__"))?;
+        Ok(declared.is(&missing)
+            && fallback.is(&missing)
+            && getter.is(object.getattr("__getattribute__")?))
+    }
+
+    fn expected(&self, expected: &'static str) -> Result<String, ProjectionError> {
+        Ok(format!(
+            "{}: expected {expected}, got {}",
+            self.path,
+            self.value.get_type().name()?
+        ))
+    }
+
+    fn invalid(&self, expected: &'static str) -> ProjectionError {
+        match self.expected(expected) {
+            Ok(message) => ProjectionError::InvalidConfiguration(message),
+            Err(error) => error,
+        }
+    }
+
+    pub(crate) fn truthy(&self) -> Result<Truthy, ProjectionError> {
+        Ok(Truthy(self.value.is_truthy()?))
+    }
+
+    pub(crate) fn exact_true(&self) -> ExactTrue {
+        ExactTrue(self.value.is(PyBool::new(self.value.py(), true)))
+    }
+
+    pub(crate) fn strict_string(&self) -> Result<String, ProjectionError> {
+        let value = self
+            .value
+            .cast::<PyString>()
+            .map_err(|_| self.invalid("a string"))?;
+        Ok(value.to_str()?.to_owned())
+    }
+
+    pub(crate) fn schema_string(&self) -> Result<String, ProjectionError> {
+        if !self.value.is_instance_of::<PyString>() {
+            return Err(ProjectionError::InternalSchemaFailure(
+                self.expected("a string")?,
+            ));
+        }
+        self.strict_string()
+    }
+
+    pub(crate) fn schema_bool(&self) -> Result<bool, ProjectionError> {
+        if !self.value.is_instance_of::<PyBool>() {
+            return Err(ProjectionError::InternalSchemaFailure(
+                self.expected("a Boolean")?,
+            ));
+        }
+        Ok(self.exact_true().0)
+    }
+
+    pub(crate) fn str_bool(&self) -> Result<StrBool, ProjectionError> {
+        if self.value.is_none() {
+            return Ok(StrBool(None));
+        }
+        Ok(StrBool(parse_str_bool(&self.strict_string()?)))
+    }
+
+    pub(crate) fn optional_strict_string(&self) -> Result<OptionalStrictString, ProjectionError> {
+        if self.value.is_none() {
+            return Ok(OptionalStrictString(None));
+        }
+        self.strict_string().map(Some).map(OptionalStrictString)
+    }
+
+    pub(crate) fn falsy_optional_string(&self) -> Result<FalsyOptionalString, ProjectionError> {
+        if !self.truthy()?.0 {
+            return Ok(FalsyOptionalString(None));
+        }
+        self.strict_string().map(Some).map(FalsyOptionalString)
+    }
+
+    pub(crate) fn tuning_string(&self) -> Result<TuningString, ProjectionError> {
+        if !self.truthy()?.0 || !self.value.is_instance_of::<PyString>() {
+            return Ok(TuningString(None));
+        }
+        self.strict_string().map(Some).map(TuningString)
+    }
+
+    pub(crate) fn string_collection(&self) -> Result<StringCollection, ProjectionError> {
+        if !self.truthy()?.0 {
+            return Ok(StringCollection(Vec::new()));
+        }
+        if self.value.is_instance_of::<PyString>() {
+            return self
+                .strict_string()
+                .map(|value| StringCollection(vec![value]));
+        }
+        let values = self
+            .value
+            .try_iter()?
+            .filter_map(|item| {
+                let member = match item {
+                    Ok(value) => Self::new(self.path, value),
+                    Err(error) => return Some(Err(error.into())),
+                };
+                match member.truthy() {
+                    Ok(Truthy(false)) => None,
+                    Ok(Truthy(true)) => Some(member.strict_string()),
+                    Err(error) => Some(Err(error)),
+                }
+            })
+            .collect::<Result<Vec<_>, ProjectionError>>()?;
+        Ok(StringCollection(values))
+    }
+
+    pub(crate) fn host_collection(&self) -> Result<StringCollection, ProjectionError> {
+        let values = self
+            .string_collection()?
+            .0
+            .into_iter()
+            .map(|host| litellm_http::media::normalize_host(&host))
+            .collect::<BTreeSet<_>>();
+        Ok(StringCollection(values.into_iter().collect()))
+    }
+
+    pub(crate) fn ssl_verify(&self) -> Result<SslVerifyInput, ProjectionError> {
+        if self.value.is_none() {
+            return Ok(SslVerifyInput(None));
+        }
+        if self.value.is_instance_of::<PyBool>() {
+            return Ok(SslVerifyInput(Some(if self.exact_true().0 {
+                SslVerify::Enabled
+            } else {
+                SslVerify::Disabled
+            })));
+        }
+        if self.value.is_instance_of::<PyString>() {
+            let parsed = match self.str_bool()?.0 {
+                Some(true) => SslVerify::Enabled,
+                Some(false) => SslVerify::Disabled,
+                None => SslVerify::CaBundle(self.strict_string()?.into()),
+            };
+            return Ok(SslVerifyInput(Some(parsed)));
+        }
+        let context = self.value.py().import("ssl")?.getattr("SSLContext")?;
+        if self.value.is_instance(&context)? {
+            return Err(ProjectionError::UnsupportedLiveObject(self.expected(
+                "a Boolean, Boolean string, CA path, or None; live SSLContext is unsupported",
+            )?));
+        }
+        Err(self.invalid("a Boolean, Boolean string, CA path, or None"))
+    }
+}
+
+#[cfg(test)]
+mod tests;

litellm-rust/crates/python-bridge/src/coercion/tests.rs

@@ -0,0 +1,372 @@
+use std::ffi::CString;
+
+use pyo3::{
+    exceptions::{PyLookupError, PyRuntimeError, PyValueError},
+    types::PyDict,
+};
+use rstest::rstest;
+
+use super::*;
+
+fn evaluate<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyAny> {
+    py.eval(&CString::new(source).unwrap(), None, None).unwrap()
+}
+
+#[rstest]
+#[case("None", false, false)]
+#[case("False", false, false)]
+#[case("True", true, true)]
+#[case("0", false, false)]
+#[case("1", true, false)]
+#[case("''", false, false)]
+#[case("'false'", true, false)]
+#[case("[]", false, false)]
+#[case("[0]", true, false)]
+#[case("{}", false, false)]
+#[case("object()", true, false)]
+fn boolean_operations_have_distinct_python_semantics(
+    #[case] source: &str,
+    #[case] truth: bool,
+    #[case] exact: bool,
+) {
+    Python::initialize();
+    Python::attach(|py| {
+        let value = evaluate(py, source);
+        let field = Field::new("test.flag", value.clone());
+        assert_eq!(field.truthy().unwrap().0, truth);
+        assert_eq!(field.exact_true().0, exact);
+        assert_eq!(
+            field.truthy().unwrap().0,
+            py.import("builtins")
+                .unwrap()
+                .getattr("bool")
+                .unwrap()
+                .call1((value,))
+                .unwrap()
+                .extract::<bool>()
+                .unwrap()
+        );
+    });
+}
+
+#[rstest]
+#[case("None", Ok(None), Ok(None), Ok(None))]
+#[case("''", Ok(Some("")), Ok(None), Ok(None))]
+#[case(
+    "' value '",
+    Ok(Some(" value ")),
+    Ok(Some(" value ")),
+    Ok(Some(" value "))
+)]
+#[case("[]", Err(()), Ok(None), Ok(None))]
+#[case("0", Err(()), Ok(None), Ok(None))]
+#[case("1", Err(()), Err(()), Ok(None))]
+#[case("object()", Err(()), Err(()), Ok(None))]
+fn string_operations_do_not_conflate_absence_and_type_checks(
+    #[case] source: &str,
+    #[case] strict: Result<Option<&str>, ()>,
+    #[case] fallback: Result<Option<&str>, ()>,
+    #[case] tuning: Result<Option<&str>, ()>,
+) {
+    Python::initialize();
+    Python::attach(|py| {
+        let field = Field::new("test.string", evaluate(py, source));
+        let owned =
+            |expected: Result<Option<&str>, ()>| expected.map(|value| value.map(str::to_owned));
+        assert_eq!(
+            field
+                .optional_strict_string()
+                .map(|value| value.0)
+                .map_err(|_| ()),
+            owned(strict)
+        );
+        assert_eq!(
+            field
+                .falsy_optional_string()
+                .map(|value| value.0)
+                .map_err(|_| ()),
+            owned(fallback)
+        );
+        assert_eq!(
+            field.tuning_string().map(|value| value.0).map_err(|_| ()),
+            owned(tuning)
+        );
+    });
+}
+
+#[rstest]
+#[case("None", None)]
+#[case("' True '", Some(true))]
+#[case("' fAlSe '", Some(false))]
+#[case("'yes'", None)]
+#[case("'1'", None)]
+#[case("'unknown'", None)]
+fn string_boolean_tokens_remain_separate_from_truthiness(
+    #[case] source: &str,
+    #[case] expected: Option<bool>,
+) {
+    Python::initialize();
+    Python::attach(|py| {
+        assert_eq!(
+            Field::new("test.flag", evaluate(py, source))
+                .str_bool()
+                .unwrap()
+                .0,
+            expected
+        );
+    });
+}
+
+#[rstest]
+#[case("'EXAMPLE.TEST.'", vec!["example.test"])]
+#[case("['B.test', '', None, 0, [], 'A.test.', 'b.test']", vec!["a.test", "b.test"])]
+#[case("('B.test', 'a.test')", vec!["a.test", "b.test"])]
+#[case("{'B.test', 'a.test'}", vec!["a.test", "b.test"])]
+#[case("(host for host in ['B.test', 'a.test'])", vec!["a.test", "b.test"])]
+#[case("None", vec![])]
+#[case("False", vec![])]
+fn host_collection_is_owned_normalized_and_deterministic(
+    #[case] source: &str,
+    #[case] expected: Vec<&str>,
+) {
+    Python::initialize();
+    Python::attach(|py| {
+        assert_eq!(
+            Field::new("url_policy.user_url_allowed_hosts", evaluate(py, source))
+                .host_collection()
+                .unwrap()
+                .0,
+            expected
+        );
+    });
+}
+
+#[test]
+fn protocol_errors_preserve_exception_identity_traceback_cause_and_context() {
+    Python::initialize();
+    Python::attach(|py| {
+        let locals = PyDict::new(py);
+        py.run(
+            c"
+failure = LookupError('protocol failed')
+cause = ValueError('cause')
+context = RuntimeError('context')
+def fail():
+    try:
+        raise context
+    except RuntimeError:
+        raise failure from cause
+class Bool:
+    def __bool__(self): return fail()
+class Length:
+    def __len__(self): return fail()
+class Iter:
+    def __iter__(self): return fail()
+class Next:
+    def __iter__(self): return self
+    def __next__(self): return fail()
+class Descriptor:
+    @property
+    def flag(self): return fail()
+values = (Bool(), Length(), Iter(), Next(), [Bool()])
+descriptor = Descriptor()
+",
+            Some(&locals),
+            Some(&locals),
+        )
+        .unwrap();
+        let values = locals.get_item("values").unwrap().unwrap();
+        for value in values.try_iter().unwrap() {
+            let error = Field::new("test.flag", value.unwrap())
+                .host_collection()
+                .err()
+                .unwrap();
+            let error = PyErr::from(error);
+            assert!(
+                error
+                    .value(py)
+                    .is(locals.get_item("failure").unwrap().unwrap())
+            );
+            assert!(error.is_instance_of::<PyLookupError>(py));
+            assert!(error.traceback(py).is_some());
+            assert!(
+                error
+                    .value(py)
+                    .getattr("__cause__")
+                    .unwrap()
+                    .is(locals.get_item("cause").unwrap().unwrap())
+            );
+            assert!(
+                error
+                    .value(py)
+                    .getattr("__context__")
+                    .unwrap()
+                    .is(locals.get_item("context").unwrap().unwrap())
+            );
+        }
+        let error = Field::read(
+            &locals.get_item("descriptor").unwrap().unwrap(),
+            "test.flag",
+        )
+        .err()
+        .unwrap();
+        assert!(
+            PyErr::from(error)
+                .value(py)
+                .is(locals.get_item("failure").unwrap().unwrap())
+        );
+    });
+}
+
+#[test]
+fn identity_and_string_contents_do_not_invoke_unrelated_protocols() {
+    Python::initialize();
+    Python::attach(|py| {
+        let locals = PyDict::new(py);
+        py.run(
+            c"
+class Hostile:
+    def __bool__(self): raise AssertionError('bool called')
+    def __eq__(self, other): raise AssertionError('eq called')
+    def __str__(self): raise AssertionError('str called')
+class Text(str):
+    def __str__(self): raise AssertionError('str called')
+    def strip(self): raise AssertionError('strip called')
+    def lower(self): raise AssertionError('lower called')
+hostile = Hostile()
+text = Text(' False ')
+",
+            Some(&locals),
+            Some(&locals),
+        )
+        .unwrap();
+        let hostile = Field::new("test.flag", locals.get_item("hostile").unwrap().unwrap());
+        assert!(!hostile.exact_true().0);
+        assert!(matches!(
+            hostile.strict_string(),
+            Err(ProjectionError::InvalidConfiguration(_))
+        ));
+        let text = Field::new("test.flag", locals.get_item("text").unwrap().unwrap());
+        assert_eq!(text.strict_string().unwrap(), " False ");
+        assert_eq!(text.str_bool().unwrap().0, Some(false));
+    });
+}
+
+#[test]
+fn missing_snapshot_fields_and_descriptor_attribute_errors_are_distinct() {
+    Python::initialize();
+    Python::attach(|py| {
+        let locals = PyDict::new(py);
+        py.run(
+            c"
+failure = AttributeError('descriptor failed')
+class Snapshot:
+    @property
+    def flag(self): raise failure
+snapshot = Snapshot()
+class Dynamic:
+    def __getattr__(self, name): raise failure
+class Intercepted:
+    def __getattribute__(self, name): raise failure
+dynamic = Dynamic()
+intercepted = Intercepted()
+",
+            Some(&locals),
+            Some(&locals),
+        )
+        .unwrap();
+        let snapshot = locals.get_item("snapshot").unwrap().unwrap();
+        let descriptor = PyErr::from(Field::read(&snapshot, "test.flag").err().unwrap());
+        assert!(
+            descriptor
+                .value(py)
+                .is(locals.get_item("failure").unwrap().unwrap())
+        );
+        for name in ["dynamic", "intercepted"] {
+            let value = locals.get_item(name).unwrap().unwrap();
+            let error = PyErr::from(Field::read(&value, "test.flag").err().unwrap());
+            assert!(
+                error
+                    .value(py)
+                    .is(locals.get_item("failure").unwrap().unwrap())
+            );
+        }
+        let missing = PyErr::from(Field::read(&snapshot, "test.missing").err().unwrap());
+        assert!(missing.is_instance_of::<PyRuntimeError>(py));
+        assert!(missing.to_string().contains("test.missing"));
+    });
+}
+
+#[test]
+fn configuration_errors_name_fields_without_exposing_values() {
+    Python::initialize();
+    Python::attach(|py| {
+        for source in [
+            "{'secret': 'do-not-print'}",
+            "['host.test', {'secret': 'do-not-print'}]",
+        ] {
+            let field = Field::new("test.setting", evaluate(py, source));
+            let error = PyErr::from(field.falsy_optional_string().err().unwrap());
+            assert!(error.is_instance_of::<PyValueError>(py));
+            assert!(error.to_string().contains("test.setting"));
+            assert!(!error.to_string().contains("do-not-print"));
+        }
+        let hosts = Field::new(
+            "url_policy.user_url_allowed_hosts",
+            evaluate(py, "['host.test', 1]"),
+        );
+        assert!(matches!(
+            hosts.host_collection(),
+            Err(ProjectionError::InvalidConfiguration(_))
+        ));
+        assert!(matches!(
+            Field::new("test.flag", evaluate(py, "1")).str_bool(),
+            Err(ProjectionError::InvalidConfiguration(_))
+        ));
+    });
+}
+
+#[test]
+fn projection_releases_the_source_collection() {
+    Python::initialize();
+    Python::attach(|py| {
+        let source = evaluate(py, "['A.test']");
+        let projected = Field::new("test.hosts", source.clone())
+            .host_collection()
+            .unwrap()
+            .0;
+        source.call_method1("append", ("b.test",)).unwrap();
+        assert_eq!(projected, ["a.test"]);
+        assert_eq!(
+            Field::new("test.hosts", source)
+                .host_collection()
+                .unwrap()
+                .0,
+            ["a.test", "b.test"]
+        );
+    });
+}
+
+#[rstest]
+#[case("True", Some(true))]
+#[case("False", Some(false))]
+#[case("1", None)]
+#[case("None", None)]
+#[case("[]", None)]
+fn accessor_booleans_are_strict_schema_values(
+    #[case] source: &str,
+    #[case] expected: Option<bool>,
+) {
+    Python::initialize();
+    Python::attach(|py| {
+        let result = Field::new("secret_manager.readable", evaluate(py, source)).schema_bool();
+        match expected {
+            Some(expected) => assert_eq!(result.unwrap(), expected),
+            None => {
+                let error = PyErr::from(result.unwrap_err());
+                assert!(error.is_instance_of::<PyRuntimeError>(py));
+                assert!(error.to_string().contains("secret_manager.readable"));
+            }
+        }
+    });
+}

litellm-rust/crates/python-bridge/src/http.rs

@@ -7,12 +7,12 @@ use std::{
 use litellm_core_utils::settings::ProcessEnvironment;
 use litellm_http::{
     HttpClientConfig, HttpClientPool, HttpSettings, HttpSettingsLayer, Resolution, SslVerify,
-    Unsupported,
+    TlsSource, Unsupported,
     media::{PublicDnsResolver, UrlPolicy},
 };
-use pyo3::{prelude::*, types::PyDict};
+use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict};
 
-use crate::{errors::RustBridgeDeclined, python_settings::PythonSettings};
+use crate::{coercion::Field, python_settings::PythonSettings};
 
 static POOL: LazyLock<HttpClientPool> =
     LazyLock::new(|| HttpClientPool::new(Arc::new(PublicDnsResolver)));
@@ -41,6 +41,30 @@ pub(crate) fn call_config(
     Ok(resolution.config)
 }
 
+pub(crate) fn client_error(error: litellm_http::Error) -> PyErr {
+    match error {
+        litellm_http::Error::Read {
+            tls_source: TlsSource::ClientIdentity,
+            ..
+        }
+        | litellm_http::Error::InvalidPem {
+            tls_source: TlsSource::ClientIdentity,
+            ..
+        } => PyValueError::new_err(
+            "http_settings.ssl_certificate: expected a readable PEM certificate and private key",
+        ),
+        litellm_http::Error::Read {
+            tls_source: TlsSource::CaBundle,
+            ..
+        }
+        | litellm_http::Error::InvalidPem {
+            tls_source: TlsSource::CaBundle,
+            ..
+        } => PyValueError::new_err("http_settings.ssl_verify: expected a readable PEM CA bundle"),
+        _ => PyValueError::new_err("http_settings: native HTTP client configuration is invalid"),
+    }
+}
+
 fn unreported(
     reported: &Mutex<HashSet<Unsupported>>,
     unsupported: Vec<Unsupported>,
@@ -53,25 +77,25 @@ fn unreported(
 }
 
 pub(crate) fn url_policy(py: Python<'_>) -> PyResult<UrlPolicy> {
-    let policy: PythonUrlPolicy =
-        PythonSettings::UrlPolicy
-            .read(py)?
-            .extract()
-            .map_err(|error: PyErr| {
-                RustBridgeDeclined::new_err(format!(
-                    "litellm URL policy cannot be used by the Rust route: {error}"
-                ))
-            })?;
+    project_url_policy(&PythonSettings::UrlPolicy.read(py)?)
+}
+
+fn project_url_policy(value: &Bound<'_, PyAny>) -> PyResult<UrlPolicy> {
     Ok(UrlPolicy {
-        validate: policy.user_url_validation,
-        allowed_hosts: policy.user_url_allowed_hosts,
+        validate: Field::read(value, "url_policy.user_url_validation")?
+            .truthy()?
+            .0,
+        allowed_hosts: Field::read(value, "url_policy.user_url_allowed_hosts")?
+            .host_collection()?
+            .0,
     })
 }
 
 fn call_ssl_verify(kwargs: &Bound<'_, PyDict>) -> PyResult<Option<SslVerify>> {
-    Ok(kwargs
-        .get_item("ssl_verify")?
-        .and_then(|value| ssl_verify(&value)))
+    match kwargs.get_item("ssl_verify")? {
+        Some(value) => Ok(Field::new("request.ssl_verify", value).ssl_verify()?.0),
+        None => Ok(None),
+    }
 }
 
 fn for_call(call_ssl_verify: Option<SslVerify>, asynchronous: bool) -> HttpSettingsLayer {
@@ -82,64 +106,47 @@ fn for_call(call_ssl_verify: Option<SslVerify>, asynchronous: bool) -> HttpSetti
     }
 }
 
-#[derive(FromPyObject)]
-struct PythonUrlPolicy {
-    user_url_validation: bool,
-    user_url_allowed_hosts: Vec<String>,
-}
-
-#[derive(FromPyObject)]
-struct PythonHttpSettings<'py> {
-    ssl_verify: Bound<'py, PyAny>,
-    ssl_certificate: Option<String>,
-    ssl_security_level: Option<String>,
-    ssl_ecdh_curve: Option<String>,
-    force_ipv4: bool,
-    http2: bool,
-    aiohttp_trust_env: bool,
-    disable_aiohttp_trust_env: bool,
-    disable_aiohttp_transport: bool,
-    user_agent: String,
-}
-
 fn configured(value: &Bound<'_, PyAny>) -> PyResult<HttpSettingsLayer> {
-    let python: PythonHttpSettings = value.extract().map_err(|error: PyErr| {
-        RustBridgeDeclined::new_err(format!(
-            "litellm HTTP settings cannot be used by the Rust route: {error}"
-        ))
-    })?;
     Ok(HttpSettingsLayer {
-        ssl_verify: ssl_verify(&python.ssl_verify),
-        ssl_certificate: python.ssl_certificate.map(PathBuf::from),
-        ssl_security_level: python.ssl_security_level,
-        ssl_ecdh_curve: python.ssl_ecdh_curve,
-        force_ipv4: Some(python.force_ipv4),
-        http2: Some(python.http2),
-        aiohttp_trust_env: Some(python.aiohttp_trust_env),
-        disable_aiohttp_trust_env: Some(python.disable_aiohttp_trust_env),
-        disable_aiohttp_transport: Some(python.disable_aiohttp_transport),
-        user_agent: Some(python.user_agent),
+        ssl_verify: Field::read(value, "http_settings.ssl_verify")?
+            .ssl_verify()?
+            .0,
+        ssl_certificate: Field::read(value, "http_settings.ssl_certificate")?
+            .optional_strict_string()?
+            .0
+            .map(PathBuf::from),
+        ssl_security_level: Field::read(value, "http_settings.ssl_security_level")?
+            .tuning_string()?
+            .0,
+        ssl_ecdh_curve: Field::read(value, "http_settings.ssl_ecdh_curve")?
+            .tuning_string()?
+            .0,
+        force_ipv4: Some(Field::read(value, "http_settings.force_ipv4")?.truthy()?.0),
+        http2: Some(Field::read(value, "http_settings.http2")?.exact_true().0),
+        aiohttp_trust_env: Some(
+            Field::read(value, "http_settings.aiohttp_trust_env")?
+                .truthy()?
+                .0,
+        ),
+        disable_aiohttp_trust_env: Some(
+            Field::read(value, "http_settings.disable_aiohttp_trust_env")?
+                .truthy()?
+                .0,
+        ),
+        disable_aiohttp_transport: Some(
+            Field::read(value, "http_settings.disable_aiohttp_transport")?
+                .exact_true()
+                .0,
+        ),
+        user_agent: Some(Field::read(value, "http_settings.user_agent")?.schema_string()?),
         ..HttpSettingsLayer::default()
     })
 }
 
-fn ssl_verify(value: &Bound<'_, PyAny>) -> Option<SslVerify> {
-    if let Ok(enabled) = value.extract::<bool>() {
-        return Some(if enabled {
-            SslVerify::Enabled
-        } else {
-            SslVerify::Disabled
-        });
-    }
-    value
-        .extract::<String>()
-        .ok()
-        .map(|path| SslVerify::parse(&path))
-}
-
 #[cfg(test)]
 mod tests {
     use litellm_http::Verify;
+    use pyo3::exceptions::PyRuntimeError;
     use rstest::rstest;
 
     use super::*;
@@ -163,7 +170,7 @@ defaults = dict(
     user_agent='litellm/test',
 )
 defaults.update(dict({overrides}))
-settings = types.SimpleNamespace(**{{name: defaults[name] for name in json.loads(contract)['http_settings']}})
+settings = types.SimpleNamespace(**{{name: defaults[name] for name in json.loads(contract)['http_settings']['fields']}})
 "
         );
         let locals = PyDict::new(py);
@@ -189,6 +196,33 @@ settings = types.SimpleNamespace(**{{name: defaults[name] for name in json.loads
         });
     }
 
+    #[test]
+    fn client_error_uses_tls_source_when_paths_match() {
+        Python::initialize();
+        Python::attach(|py| {
+            let path = PathBuf::from("/shared.pem");
+            let ca_error = client_error(litellm_http::Error::InvalidPem {
+                path: path.clone(),
+                message: "invalid".into(),
+                tls_source: TlsSource::CaBundle,
+            });
+            assert_eq!(
+                ca_error.to_string(),
+                "ValueError: http_settings.ssl_verify: expected a readable PEM CA bundle"
+            );
+            let client_error = client_error(litellm_http::Error::InvalidPem {
+                path,
+                message: "invalid".into(),
+                tls_source: TlsSource::ClientIdentity,
+            });
+            assert!(client_error.is_instance_of::<PyValueError>(py));
+            assert_eq!(
+                client_error.to_string(),
+                "ValueError: http_settings.ssl_certificate: expected a readable PEM certificate and private key"
+            );
+        });
+    }
+
     #[test]
     fn python_settings_flow_into_the_configured_layer() {
         Python::initialize();
@@ -259,12 +293,16 @@ user_agent='litellm/9.9.9',
         });
     }
 
-    #[test]
-    fn ssl_context_global_is_ignored_so_environment_and_defaults_apply() {
+    #[rstest]
+    #[case("ssl_verify=object()")]
+    #[case("ssl_verify=__import__('ssl').SSLContext(__import__('ssl').PROTOCOL_TLS_CLIENT)")]
+    #[case("ssl_certificate=1")]
+    fn invalid_http_configuration_is_terminal(#[case] overrides: &str) {
         Python::initialize();
         Python::attach(|py| {
-            let layer = configured(&python_settings(py, "ssl_verify=object()")).unwrap();
-            assert_eq!(layer.ssl_verify, None);
+            let error = configured(&python_settings(py, overrides)).unwrap_err();
+            assert!(error.is_instance_of::<PyValueError>(py));
+            assert!(error.to_string().contains("http_settings.ssl_"));
         });
     }
 
@@ -281,11 +319,21 @@ user_agent='litellm/9.9.9',
     }
 
     #[test]
-    fn mistyped_python_settings_decline_instead_of_raising() {
+    fn mutable_globals_use_their_consumer_operations() {
         Python::initialize();
         Python::attach(|py| {
-            let error = configured(&python_settings(py, "force_ipv4='yes'")).unwrap_err();
-            assert!(error.is_instance_of::<RustBridgeDeclined>(py));
+            let layer = configured(&python_settings(py,
+                "force_ipv4='yes', http2=1, disable_aiohttp_transport=1, aiohttp_trust_env=[1], disable_aiohttp_trust_env=[], ssl_security_level=1, ssl_ecdh_curve=[]"
+            )).unwrap();
+            assert_eq!(layer.force_ipv4, Some(true));
+            assert_eq!(layer.http2, Some(false));
+            assert_eq!(layer.disable_aiohttp_transport, Some(false));
+            assert_eq!(layer.aiohttp_trust_env, Some(true));
+            assert_eq!(layer.disable_aiohttp_trust_env, Some(false));
+            assert_eq!(layer.ssl_security_level, None);
+            assert_eq!(layer.ssl_ecdh_curve, None);
+            let error = configured(&python_settings(py, "user_agent=1")).unwrap_err();
+            assert!(error.is_instance_of::<PyRuntimeError>(py));
         });
     }
 
@@ -323,17 +371,36 @@ user_agent='litellm/9.9.9',
     }
 
     #[test]
-    fn live_ssl_context_argument_is_ignored_so_the_configured_value_applies() {
+    fn live_ssl_context_argument_raises_instead_of_using_another_layer() {
         Python::initialize();
         Python::attach(|py| {
             let kwargs = PyDict::new(py);
-            kwargs
-                .set_item("ssl_verify", py.eval(c"object()", None, None).unwrap())
+            let ssl = py.import("ssl").unwrap();
+            let context = ssl
+                .getattr("SSLContext")
+                .unwrap()
+                .call1((ssl.getattr("PROTOCOL_TLS_CLIENT").unwrap(),))
                 .unwrap();
-            let call = for_call(call_ssl_verify(&kwargs).unwrap(), true);
-            let settings =
-                HttpSettings::from_layers([call, configured_ssl_verify(SslVerify::Disabled)]);
-            assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled));
+            kwargs.set_item("ssl_verify", context).unwrap();
+            let error = call_ssl_verify(&kwargs).unwrap_err();
+            assert!(error.is_instance_of::<PyValueError>(py));
+            assert!(error.to_string().contains("request.ssl_verify"));
+            assert!(error.to_string().contains("SSLContext"));
+        });
+    }
+
+    #[test]
+    fn url_policy_uses_truthiness_and_normalized_owned_hosts() {
+        Python::initialize();
+        Python::attach(|py| {
+            let value = py.eval(c"__import__('types').SimpleNamespace(user_url_validation=[], user_url_allowed_hosts=['B.test', 'a.test.', 'b.test'])", None, None).unwrap();
+            assert_eq!(
+                project_url_policy(&value).unwrap(),
+                UrlPolicy {
+                    validate: false,
+                    allowed_hosts: vec!["a.test".into(), "b.test".into()],
+                }
+            );
         });
     }
 

litellm-rust/crates/python-bridge/src/lib.rs

@@ -1,4 +1,5 @@
 mod cache;
+mod coercion;
 mod credentials;
 mod diagnostics;
 mod errors;

litellm-rust/crates/python-bridge/src/marshal.rs

@@ -172,6 +172,60 @@ mod tests {
         request_input_sources(&kwargs, names.iter().copied())
     }
 
+    #[serde_with::serde_as]
+    #[derive(Debug, serde::Deserialize, serde::Serialize, PartialEq)]
+    struct Numbers {
+        #[serde_as(deserialize_as = "Option<Vec<litellm_core_utils::serde_compat::LaxI64>>")]
+        integers: Option<Vec<i64>>,
+        #[serde_as(deserialize_as = "Option<litellm_core_utils::serde_compat::FiniteF64>")]
+        float: Option<f64>,
+    }
+
+    #[test]
+    fn numeric_adapters_agree_across_json_and_python_boundaries() {
+        Python::initialize();
+        Python::attach(|py| {
+            for input in [
+                json!({}),
+                json!({"integers": null, "float": null}),
+                json!({"integers": [i64::MIN, i64::MAX, "9007199254740993.0", " +1_000.00 ", true, 3.0], "float": " 1.25 "}),
+                json!({"integers": [u64::MAX]}),
+                json!({"integers": ["1.0000000000000001"]}),
+                json!({"integers": [2.5]}),
+                json!({"float": "NaN"}),
+                json!({"float": "inf"}),
+                json!({"float": "1e999"}),
+                json!({"float": true}),
+                json!({"float": u64::MAX}),
+            ] {
+                let expected = serde_json::from_value::<Numbers>(input.clone());
+                let python = litellm_host_python::to_py(py, &input).unwrap();
+                let actual = from_py::<Numbers>(python.bind(py));
+                match (expected, actual) {
+                    (Ok(expected), Ok(actual)) => {
+                        assert_eq!(actual, expected);
+                        let serialized = litellm_host_python::to_py(py, &actual).unwrap();
+                        assert_eq!(
+                            from_py::<Value>(serialized.bind(py)).unwrap(),
+                            serde_json::to_value(expected).unwrap()
+                        );
+                    }
+                    (Err(_), Err(_)) => {}
+                    mismatch => panic!("boundary mismatch for {input}: {mismatch:?}"),
+                }
+            }
+            for source in [
+                c"{'float': float('nan')}",
+                c"{'float': float('inf')}",
+                c"{'integers': [float('inf')]}",
+                c"{'integers': [2 ** 100]}",
+            ] {
+                let value = py.eval(source, None, None).unwrap();
+                assert!(from_py::<Numbers>(&value).is_err());
+            }
+        });
+    }
+
     #[test]
     fn argument_converters_keep_nested_values_and_accept_explicit_none() {
         Python::initialize();

litellm-rust/crates/python-bridge/src/python_settings.rs

@@ -43,32 +43,204 @@ pub(crate) const CONTRACT: &str = include_str!("../python_settings.json");
 
 #[cfg(test)]
 mod tests {
-    use std::{collections::BTreeSet, ffi::CString};
-
-    use pyo3::{prelude::*, types::PyDict};
-
     use super::{CONTRACT, PythonSettings};
+    use pyo3::prelude::*;
+    use serde_json::{Value, json};
+
+    struct SettingSpec {
+        group: &'static str,
+        name: &'static str,
+        adapter: &'static str,
+        precedence: &'static str,
+        sensitive: bool,
+        shapes: &'static [&'static str],
+        unsupported_live: Option<&'static str>,
+    }
+
+    const SETTINGS: &[SettingSpec] = &[
+        SettingSpec {
+            group: "http_settings",
+            name: "ssl_verify",
+            adapter: "SslVerifyInput",
+            precedence: "module_global",
+            sensitive: false,
+            shapes: &["none", "bool", "str"],
+            unsupported_live: Some("configuration_error"),
+        },
+        SettingSpec {
+            group: "http_settings",
+            name: "ssl_certificate",
+            adapter: "OptionalStrictString",
+            precedence: "module_global",
+            sensitive: false,
+            shapes: &[],
+            unsupported_live: None,
+        },
+        SettingSpec {
+            group: "http_settings",
+            name: "ssl_security_level",
+            adapter: "TuningString",
+            precedence: "module_global",
+            sensitive: false,
+            shapes: &[],
+            unsupported_live: None,
+        },
+        SettingSpec {
+            group: "http_settings",
+            name: "ssl_ecdh_curve",
+            adapter: "TuningString",
+            precedence: "module_global",
+            sensitive: false,
+            shapes: &[],
+            unsupported_live: None,
+        },
+        SettingSpec {
+            group: "http_settings",
+            name: "force_ipv4",
+            adapter: "Truthy",
+            precedence: "module_global",
+            sensitive: false,
+            shapes: &[],
+            unsupported_live: None,
+        },
+        SettingSpec {
+            group: "http_settings",
+            name: "http2",
+            adapter: "ExactTrue",
+            precedence: "module_global",
+            sensitive: false,
+            shapes: &[],
+            unsupported_live: None,
+        },
+        SettingSpec {
+            group: "http_settings",
+            name: "aiohttp_trust_env",
+            adapter: "Truthy",
+            precedence: "module_global",
+            sensitive: false,
+            shapes: &[],
+            unsupported_live: None,
+        },
+        SettingSpec {
+            group: "http_settings",
+            name: "disable_aiohttp_trust_env",
+            adapter: "Truthy",
+            precedence: "module_global",
+            sensitive: false,
+            shapes: &[],
+            unsupported_live: None,
+        },
+        SettingSpec {
+            group: "http_settings",
+            name: "disable_aiohttp_transport",
+            adapter: "ExactTrue",
+            precedence: "module_global",
+            sensitive: false,
+            shapes: &[],
+            unsupported_live: None,
+        },
+        SettingSpec {
+            group: "http_settings",
+            name: "user_agent",
+            adapter: "StrictString",
+            precedence: "accessor",
+            sensitive: false,
+            shapes: &[],
+            unsupported_live: None,
+        },
+        SettingSpec {
+            group: "url_policy",
+            name: "user_url_validation",
+            adapter: "Truthy",
+            precedence: "module_global",
+            sensitive: false,
+            shapes: &[],
+            unsupported_live: None,
+        },
+        SettingSpec {
+            group: "url_policy",
+            name: "user_url_allowed_hosts",
+            adapter: "HostCollection",
+            precedence: "module_global",
+            sensitive: false,
+            shapes: &[],
+            unsupported_live: None,
+        },
+        SettingSpec {
+            group: "provider_defaults",
+            name: "vertex_project",
+            adapter: "FalsyOptionalString",
+            precedence: "module_global",
+            sensitive: true,
+            shapes: &[],
+            unsupported_live: None,
+        },
+        SettingSpec {
+            group: "provider_defaults",
+            name: "vertex_location",
+            adapter: "FalsyOptionalString",
+            precedence: "module_global",
+            sensitive: true,
+            shapes: &[],
+            unsupported_live: None,
+        },
+        SettingSpec {
+            group: "provider_defaults",
+            name: "enable_azure_ad_token_refresh",
+            adapter: "ExactTrue",
+            precedence: "module_global",
+            sensitive: false,
+            shapes: &[],
+            unsupported_live: None,
+        },
+        SettingSpec {
+            group: "secret_manager",
+            name: "readable",
+            adapter: "StrictBool",
+            precedence: "accessor",
+            sensitive: false,
+            shapes: &[],
+            unsupported_live: None,
+        },
+    ];
 
     #[test]
-    fn every_settings_group_is_in_the_python_contract() {
-        Python::initialize();
-        Python::attach(|py| {
-            let locals = PyDict::new(py);
-            locals.set_item("contract", CONTRACT).unwrap();
-            let source = CString::new("import json\nkeys = list(json.loads(contract))").unwrap();
-            py.run(&source, Some(&locals), Some(&locals)).unwrap();
-            let declared: BTreeSet<String> = locals
-                .get_item("keys")
+    fn settings_manifest_matches_the_semantic_contract() {
+        pyo3::Python::initialize();
+        let manifest: Value = pyo3::Python::attach(|py| {
+            let value = py
+                .import("json")
                 .unwrap()
-                .unwrap()
-                .extract::<Vec<String>>()
-                .unwrap()
-                .into_iter()
-                .collect();
-            let read: BTreeSet<String> = PythonSettings::ALL
-                .map(|group| group.name().to_owned())
-                .into();
-            assert_eq!(read, declared);
+                .call_method1("loads", (CONTRACT,))
+                .unwrap();
+            litellm_host_python::from_py(&value).unwrap()
         });
+        let expected: serde_json::Map<String, Value> = PythonSettings::ALL
+            .into_iter()
+            .map(|group| {
+                let fields: serde_json::Map<String, Value> = SETTINGS
+                    .iter()
+                    .filter(|spec| spec.group == group.name())
+                    .map(|spec| {
+                        (
+                            spec.name.to_owned(),
+                            json!({
+                                "adapter": spec.adapter,
+                                "required": true,
+                                "precedence": spec.precedence,
+                                "sensitive": spec.sensitive,
+                                "shapes": spec.shapes,
+                                "unsupported_live": spec.unsupported_live,
+                            }),
+                        )
+                    })
+                    .collect();
+                (
+                    group.name().to_owned(),
+                    json!({"version": 1, "fields": fields}),
+                )
+            })
+            .collect();
+        assert_eq!(manifest, Value::Object(expected));
     }
 }

litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs

@@ -19,7 +19,7 @@ use pyo3::{
     types::{PyDict, PyTuple},
 };
 
-use crate::{errors::RustBridgeDeclined, http, python_settings::PythonSettings};
+use crate::{coercion::Field, errors::RustBridgeDeclined, http, python_settings::PythonSettings};
 
 const SURFACE: LegacySurface = LegacySurface {
     call_type: "ocr",
@@ -51,7 +51,7 @@ fn run_ocr(
         ocr_settings(py)?,
         secrets,
     )
-    .map_err(|error| RustBridgeDeclined::new_err(error.to_string()))?;
+    .map_err(http::client_error)?;
     run_legacy_call(
         py,
         if asynchronous { ASYNC_SURFACE } else { SURFACE },
@@ -62,14 +62,8 @@ fn run_ocr(
     )
 }
 
-#[derive(FromPyObject)]
-struct PythonSecretManager {
-    readable: bool,
-}
-
 fn process_environment_secrets(secret_manager: &Bound<'_, PyAny>) -> PyResult<Secrets> {
-    let manager: PythonSecretManager = secret_manager.extract()?;
-    if manager.readable {
+    if Field::read(secret_manager, "secret_manager.readable")?.schema_bool()? {
         return Err(RustBridgeDeclined::new_err(
             "a readable secret manager is configured and the Rust route only reads the process environment",
         ));
@@ -77,26 +71,24 @@ fn process_environment_secrets(secret_manager: &Bound<'_, PyAny>) -> PyResult<Se
     Ok(Arc::new(ProcessEnvironment))
 }
 
-#[derive(FromPyObject)]
-struct PythonProviderDefaults {
-    vertex_project: Option<String>,
-    vertex_location: Option<String>,
-    enable_azure_ad_token_refresh: Option<bool>,
+fn ocr_settings(py: Python<'_>) -> PyResult<OcrSettings> {
+    project_provider_defaults(&PythonSettings::ProviderDefaults.read(py)?)
 }
 
-fn ocr_settings(py: Python<'_>) -> PyResult<OcrSettings> {
-    let defaults: PythonProviderDefaults = PythonSettings::ProviderDefaults
-        .read(py)?
-        .extract()
-        .map_err(|error: PyErr| {
-            RustBridgeDeclined::new_err(format!(
-                "litellm provider defaults cannot be used by the Rust route: {error}"
-            ))
-        })?;
+fn project_provider_defaults(value: &Bound<'_, PyAny>) -> PyResult<OcrSettings> {
     Ok(OcrSettings {
-        vertex_project: defaults.vertex_project,
-        vertex_location: defaults.vertex_location,
-        enable_azure_ad_token_refresh: defaults.enable_azure_ad_token_refresh == Some(true),
+        vertex_project: Field::read(value, "provider_defaults.vertex_project")?
+            .falsy_optional_string()?
+            .0,
+        vertex_location: Field::read(value, "provider_defaults.vertex_location")?
+            .falsy_optional_string()?
+            .0,
+        enable_azure_ad_token_refresh: Field::read(
+            value,
+            "provider_defaults.enable_azure_ad_token_refresh",
+        )?
+        .exact_true()
+        .0,
         ..OcrSettings::from_environment(&ProcessEnvironment)
     })
 }
@@ -140,6 +132,35 @@ mod tests {
         locals.get_item("manager").unwrap().unwrap()
     }
 
+    #[test]
+    fn provider_defaults_distinguish_falsey_values_and_exact_true() {
+        Python::initialize();
+        Python::attach(|py| {
+            let value = py.eval(c"__import__('types').SimpleNamespace(vertex_project=[], vertex_location=0, enable_azure_ad_token_refresh=1)", None, None).unwrap();
+            let projected = super::project_provider_defaults(&value).unwrap();
+            assert_eq!(projected.vertex_project, None);
+            assert_eq!(projected.vertex_location, None);
+            assert!(!projected.enable_azure_ad_token_refresh);
+            value.setattr("vertex_project", "project").unwrap();
+            value.setattr("vertex_location", "region").unwrap();
+            value
+                .setattr("enable_azure_ad_token_refresh", true)
+                .unwrap();
+            let next = super::project_provider_defaults(&value).unwrap();
+            assert_eq!(next.vertex_project.as_deref(), Some("project"));
+            assert_eq!(next.vertex_location.as_deref(), Some("region"));
+            assert!(next.enable_azure_ad_token_refresh);
+            value.setattr("vertex_project", 1).unwrap();
+            let error = super::project_provider_defaults(&value).err().unwrap();
+            assert!(error.is_instance_of::<pyo3::exceptions::PyValueError>(py));
+            assert!(
+                error
+                    .to_string()
+                    .contains("provider_defaults.vertex_project")
+            );
+        });
+    }
+
     #[test]
     fn a_readable_secret_manager_sends_the_call_back_to_python() {
         Python::initialize();

litellm-rust/crates/secrets-azure/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "litellm-secrets-azure"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-auth-azure.workspace = true
+litellm-auth-types.workspace = true
+litellm-secrets-types.workspace = true
+litellm-core-utils.workspace = true
+reqwest.workspace = true
+serde.workspace = true
+thiserror.workspace = true
+veil.workspace = true
+percent-encoding = "2.3"
+
+[dev-dependencies]
+tokio.workspace = true
+wiremock = "0.6.5"
+rstest.workspace = true
+serde_json.workspace = true
+sha2.workspace = true

litellm-rust/crates/secrets-azure/src/error.rs

@@ -0,0 +1,25 @@
+#[derive(thiserror::Error, veil::Redact)]
+pub enum Error {
+    #[error("{0} environment variable is missing")]
+    MissingEnvironment(&'static str),
+    #[error("AZURE_KEY_VAULT_URI is not a valid https vault URL")]
+    VaultUri,
+    #[error("Azure Key Vault credentials are not configured")]
+    MissingCredentials,
+    #[error(transparent)]
+    Auth(
+        #[from]
+        #[redact]
+        litellm_auth_types::Error,
+    ),
+    #[error("Azure Key Vault request failed")]
+    Http(
+        #[source]
+        #[redact]
+        reqwest::Error,
+    ),
+    #[error("Azure Key Vault returned HTTP {0}")]
+    Status(u16),
+    #[error("Azure Key Vault response is missing the secret value")]
+    MissingValue,
+}

litellm-rust/crates/secrets-azure/src/key_vault.rs

@@ -0,0 +1,118 @@
+use std::sync::Arc;
+
+use litellm_auth_azure::{AzureAuthInputs, AzureAuthService, ConfigValue};
+use litellm_auth_types::{InputSource, Sourced};
+use litellm_core_utils::settings::Lookup;
+use litellm_secrets_types::{Secret, SecretValue};
+use percent_encoding::{AsciiSet, NON_ALPHANUMERIC};
+use serde::Deserialize;
+
+use crate::Error;
+
+const AZURE_KEY_VAULT_URI: &str = "AZURE_KEY_VAULT_URI";
+const API_VERSION: &str = "7.4";
+const PATH_SEGMENT: &AsciiSet = &NON_ALPHANUMERIC
+    .remove(b'-')
+    .remove(b'.')
+    .remove(b'_')
+    .remove(b'~');
+
+#[derive(Clone)]
+pub struct AzureKeyVault {
+    client: reqwest::Client,
+    vault: reqwest::Url,
+    auth: Arc<AzureAuthService>,
+    inputs: Arc<AzureAuthInputs>,
+    environment: Arc<dyn Lookup + Send + Sync>,
+}
+
+#[derive(Deserialize)]
+struct SecretResponse {
+    value: Option<String>,
+}
+
+impl AzureKeyVault {
+    pub fn with_client(
+        client: reqwest::Client,
+        vault: reqwest::Url,
+        environment: Arc<dyn Lookup + Send + Sync>,
+    ) -> Result<Self, Error> {
+        if vault.host_str().is_none() {
+            return Err(Error::VaultUri);
+        }
+        let inputs = AzureAuthInputs {
+            azure_scope: ConfigValue::Value(Sourced::new(
+                scope_for(&vault),
+                InputSource::Deployment,
+            )),
+            enable_azure_ad_token_refresh: Sourced::new(true, InputSource::Deployment),
+            ..AzureAuthInputs::default()
+        };
+        Ok(Self {
+            client,
+            vault,
+            auth: Arc::new(AzureAuthService::default()),
+            inputs: Arc::new(inputs),
+            environment,
+        })
+    }
+
+    pub fn new(environment: Arc<dyn Lookup + Send + Sync>) -> Result<Self, Error> {
+        let value = environment
+            .get(AZURE_KEY_VAULT_URI)
+            .ok_or(Error::MissingEnvironment(AZURE_KEY_VAULT_URI))?;
+        let vault = reqwest::Url::parse(&value).map_err(|_| Error::VaultUri)?;
+        if vault.scheme() != "https" || vault.host_str().is_none() {
+            return Err(Error::VaultUri);
+        }
+        Self::with_client(reqwest::Client::new(), vault, environment)
+    }
+
+    pub fn scope(&self) -> &str {
+        self.inputs
+            .azure_scope
+            .as_value()
+            .map(|value| value.value().as_str())
+            .unwrap_or_default()
+    }
+
+    pub async fn get_secret_from_azure_key_vault(
+        &self,
+        name: &str,
+    ) -> Result<Option<Secret>, Error> {
+        let token = self
+            .auth
+            .get_azure_ad_token(&self.inputs, &|key| self.environment.get(key))
+            .await?
+            .ok_or(Error::MissingCredentials)?;
+        let encoded_name = percent_encoding::utf8_percent_encode(name, PATH_SEGMENT);
+        let url = self
+            .vault
+            .join(&format!("secrets/{encoded_name}?api-version={API_VERSION}"))
+            .map_err(|_| Error::VaultUri)?;
+        let response = self
+            .client
+            .get(url)
+            .bearer_auth(token.value().secret().expose())
+            .send()
+            .await
+            .map_err(Error::Http)?;
+        if response.status() == reqwest::StatusCode::NOT_FOUND {
+            return Ok(None);
+        }
+        if response.status() != reqwest::StatusCode::OK {
+            return Err(Error::Status(response.status().as_u16()));
+        }
+        let payload: SecretResponse = response.json().await.map_err(Error::Http)?;
+        let value = payload.value.ok_or(Error::MissingValue)?;
+        Ok(Some(Secret::String(SecretValue::new(value))))
+    }
+}
+
+fn scope_for(vault: &reqwest::Url) -> String {
+    let host = vault.host_str().unwrap_or_default();
+    let resource = host
+        .split_once('.')
+        .map_or(host, |(_, remainder)| remainder);
+    format!("https://{resource}/.default")
+}

litellm-rust/crates/secrets-azure/src/lib.rs

@@ -0,0 +1,7 @@
+#![forbid(unsafe_code)]
+
+mod error;
+mod key_vault;
+
+pub use error::Error;
+pub use key_vault::AzureKeyVault;

litellm-rust/crates/secrets-azure/tests/fixtures/key_vault_parity.json

@@ -0,0 +1,8 @@
+{
+  "cases": [
+    {"name": "plain_value", "secret_name": "OPENAI-API-KEY", "response": {"status": 200, "body": {"value": "sk-parity-1", "id": "https://example.vault.azure.net/secrets/OPENAI-API-KEY/abc"}}, "expected": {"value": "sk-parity-1"}},
+    {"name": "json_value_is_kept_as_string", "secret_name": "JSON-SECRET", "response": {"status": 200, "body": {"value": "{\"api_key\": \"nested\"}", "id": "https://example.vault.azure.net/secrets/JSON-SECRET/abc"}}, "expected": {"value": "{\"api_key\": \"nested\"}"}},
+    {"name": "missing_secret", "secret_name": "MISSING", "response": {"status": 404, "body": {"error": {"code": "SecretNotFound", "message": "not found"}}}, "expected": {"missing": true}},
+    {"name": "forbidden", "secret_name": "FORBIDDEN", "response": {"status": 403, "body": {"error": {"code": "Forbidden", "message": "denied"}}}, "expected": {"error": true}}
+  ]
+}

litellm-rust/crates/secrets-azure/tests/key_vault.rs

@@ -0,0 +1,222 @@
+use std::sync::Arc;
+
+use litellm_secrets_azure::{AzureKeyVault, Error};
+use litellm_secrets_types::{Secret, SecretValue};
+use serde::Deserialize;
+use wiremock::{
+    Mock, MockServer, ResponseTemplate,
+    matchers::{header, path, query_param},
+};
+
+fn manager(server: &MockServer) -> AzureKeyVault {
+    AzureKeyVault::with_client(
+        reqwest::Client::new(),
+        server.uri().parse().unwrap(),
+        Arc::new(|name: &str| (name == "AZURE_AD_TOKEN").then(|| "fake".to_owned())),
+    )
+    .unwrap()
+}
+
+#[tokio::test]
+async fn reads_secret_with_bearer_token_and_api_version() {
+    let server = MockServer::start().await;
+    Mock::given(path("/secrets/OPENAI-API-KEY"))
+        .and(query_param("api-version", "7.4"))
+        .and(header("authorization", "Bearer fake"))
+        .respond_with(
+            ResponseTemplate::new(200)
+                .set_body_json(serde_json::json!({"value": "s3cret", "id": "secret-id"})),
+        )
+        .expect(1)
+        .mount(&server)
+        .await;
+
+    let secret = manager(&server)
+        .get_secret_from_azure_key_vault("OPENAI-API-KEY")
+        .await
+        .unwrap()
+        .unwrap();
+
+    assert_eq!(secret, Secret::String(SecretValue::new("s3cret")));
+}
+
+#[tokio::test]
+async fn percent_encodes_secret_name_path_segment() {
+    let server = MockServer::start().await;
+    Mock::given(path("/secrets/name%2Fwith%20spaces"))
+        .and(query_param("api-version", "7.4"))
+        .respond_with(
+            ResponseTemplate::new(200).set_body_json(serde_json::json!({"value": "value"})),
+        )
+        .expect(1)
+        .mount(&server)
+        .await;
+
+    let secret = manager(&server)
+        .get_secret_from_azure_key_vault("name/with spaces")
+        .await
+        .unwrap()
+        .unwrap();
+
+    assert_eq!(secret.as_str(), Some("value"));
+}
+
+#[rstest::rstest]
+#[case::not_found(404, None)]
+#[case::forbidden(403, Some(403))]
+#[tokio::test]
+async fn handles_statuses(#[case] status: u16, #[case] expected_status: Option<u16>) {
+    let server = MockServer::start().await;
+    Mock::given(path("/secrets/NAME"))
+        .respond_with(ResponseTemplate::new(status))
+        .expect(1)
+        .mount(&server)
+        .await;
+
+    let result = manager(&server)
+        .get_secret_from_azure_key_vault("NAME")
+        .await;
+
+    match expected_status {
+        None => assert_eq!(result.unwrap(), None),
+        Some(status) => assert!(matches!(result, Err(Error::Status(actual)) if actual == status)),
+    }
+}
+
+#[tokio::test]
+async fn missing_value_is_an_error() {
+    let server = MockServer::start().await;
+    Mock::given(path("/secrets/NAME"))
+        .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({})))
+        .expect(1)
+        .mount(&server)
+        .await;
+
+    assert!(matches!(
+        manager(&server)
+            .get_secret_from_azure_key_vault("NAME")
+            .await,
+        Err(Error::MissingValue)
+    ));
+}
+
+#[test]
+fn new_validates_vault_environment() {
+    assert!(matches!(
+        AzureKeyVault::new(Arc::new(|_: &str| None)),
+        Err(Error::MissingEnvironment("AZURE_KEY_VAULT_URI"))
+    ));
+    assert!(matches!(
+        AzureKeyVault::new(Arc::new(|name: &str| {
+            (name == "AZURE_KEY_VAULT_URI").then(|| "http://vault.example".to_owned())
+        })),
+        Err(Error::VaultUri)
+    ));
+    assert!(matches!(
+        AzureKeyVault::new(Arc::new(|name: &str| {
+            (name == "AZURE_KEY_VAULT_URI").then(|| "vault.example".to_owned())
+        })),
+        Err(Error::VaultUri)
+    ));
+}
+
+#[rstest::rstest]
+#[case("https://myvault.vault.azure.net", "https://vault.azure.net/.default")]
+#[case(
+    "https://v.vault.usgovcloudapi.net/",
+    "https://vault.usgovcloudapi.net/.default"
+)]
+#[case("http://localhost:8080", "https://localhost/.default")]
+#[test]
+fn derives_scope_from_vault_host(#[case] uri: &str, #[case] expected: &str) {
+    let manager = AzureKeyVault::with_client(
+        reqwest::Client::new(),
+        uri.parse().unwrap(),
+        Arc::new(|_: &str| None),
+    )
+    .unwrap();
+
+    assert_eq!(manager.scope(), expected);
+}
+
+#[tokio::test]
+async fn missing_credentials_do_not_request_vault() {
+    let server = MockServer::start().await;
+    Mock::given(path("/secrets/NAME"))
+        .respond_with(ResponseTemplate::new(200))
+        .expect(0)
+        .mount(&server)
+        .await;
+
+    assert!(
+        manager_without_credentials(&server)
+            .get_secret_from_azure_key_vault("NAME")
+            .await
+            .is_err()
+    );
+}
+
+fn manager_without_credentials(server: &MockServer) -> AzureKeyVault {
+    AzureKeyVault::with_client(
+        reqwest::Client::new(),
+        server.uri().parse().unwrap(),
+        Arc::new(|name: &str| {
+            (name == "AZURE_CREDENTIAL").then(|| "ClientSecretCredential".to_owned())
+        }),
+    )
+    .unwrap()
+}
+
+#[derive(Deserialize)]
+struct Fixture {
+    cases: Vec<FixtureCase>,
+}
+
+#[derive(Deserialize)]
+struct FixtureCase {
+    secret_name: String,
+    response: FixtureResponse,
+    expected: FixtureExpected,
+}
+
+#[derive(Deserialize)]
+struct FixtureResponse {
+    status: u16,
+    body: serde_json::Value,
+}
+
+#[derive(Deserialize)]
+struct FixtureExpected {
+    value: Option<String>,
+    missing: Option<bool>,
+    error: Option<bool>,
+}
+
+#[tokio::test]
+async fn parity_fixture_matches_python_backend_contract() {
+    let fixture: Fixture =
+        serde_json::from_str(include_str!("fixtures/key_vault_parity.json")).unwrap();
+    for case in fixture.cases {
+        let server = MockServer::start().await;
+        Mock::given(path(format!("/secrets/{}", case.secret_name)))
+            .respond_with(
+                ResponseTemplate::new(case.response.status).set_body_json(case.response.body),
+            )
+            .expect(1)
+            .mount(&server)
+            .await;
+        let result = manager(&server)
+            .get_secret_from_azure_key_vault(&case.secret_name)
+            .await;
+        if case.expected.missing == Some(true) {
+            assert_eq!(result.unwrap(), None);
+        } else if case.expected.error == Some(true) {
+            assert!(result.is_err());
+        } else {
+            assert_eq!(
+                result.unwrap().unwrap().as_str(),
+                case.expected.value.as_deref()
+            );
+        }
+    }
+}

litellm-rust/crates/secrets-azure/tests/live.rs

@@ -0,0 +1,30 @@
+use std::sync::Arc;
+
+use litellm_core_utils::settings::ProcessEnvironment;
+use litellm_secrets_azure::AzureKeyVault;
+use litellm_secrets_types::Secret;
+
+#[tokio::test]
+#[ignore]
+async fn reads_a_real_secret() {
+    let environment = Arc::new(ProcessEnvironment);
+    let manager = AzureKeyVault::new(environment).unwrap();
+    let name = std::env::var("AZURE_KEY_VAULT_LIVE_SECRET_NAME").unwrap();
+    let secret = manager
+        .get_secret_from_azure_key_vault(&name)
+        .await
+        .unwrap()
+        .unwrap();
+    assert!(matches!(&secret, Secret::String(_)));
+    let host = std::env::var("AZURE_KEY_VAULT_URI")
+        .unwrap()
+        .parse::<reqwest::Url>()
+        .unwrap()
+        .host_str()
+        .unwrap()
+        .to_owned();
+    let value_len = secret.as_str().unwrap().len();
+    println!(
+        "native provider=litellm-secrets-azure vault_host={host} secret={name} value_len={value_len}"
+    );
+}

litellm-rust/crates/secrets-cyberark/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "litellm-secrets-cyberark"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-secrets-types.workspace = true
+litellm-core-utils.workspace = true
+base64.workspace = true
+moka.workspace = true
+reqwest.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+veil.workspace = true
+tracing = "0.1"
+percent-encoding = "2.3"
+tokio = { workspace = true, features = ["sync"] }
+
+[dev-dependencies]
+rstest.workspace = true
+tokio.workspace = true
+wiremock = "0.6.5"
+serde.workspace = true
+serde_json.workspace = true

litellm-rust/crates/secrets-cyberark/src/error.rs

@@ -0,0 +1,27 @@
+#[derive(thiserror::Error, veil::Redact)]
+pub enum Error {
+    #[error("CyberArk Conjur HTTP request failed")]
+    Http(
+        #[from]
+        #[redact]
+        reqwest::Error,
+    ),
+    #[error("CyberArk Conjur authentication returned HTTP {0}")]
+    AuthStatus(u16),
+    #[error("CyberArk Conjur returned HTTP {0}")]
+    Status(u16),
+    #[error(
+        "CyberArk credentials are missing: set CYBERARK_API_KEY or both CYBERARK_CLIENT_CERT and CYBERARK_CLIENT_KEY"
+    )]
+    MissingCredentials,
+    #[error("CyberArk client certificate could not be loaded")]
+    ClientCertificate,
+    #[error("invalid refresh interval")]
+    RefreshInterval,
+    #[error("invalid CyberArk Conjur endpoint")]
+    Endpoint,
+    #[error("CyberArk secret manager requires an enterprise license")]
+    EnterpriseRequired,
+    #[error(transparent)]
+    Operation(#[from] litellm_secrets_types::Error),
+}

litellm-rust/crates/secrets-cyberark/src/lib.rs

@@ -0,0 +1,7 @@
+#![forbid(unsafe_code)]
+
+mod error;
+mod secret_manager;
+
+pub use error::Error;
+pub use secret_manager::{CyberArkSecretManager, DeleteOutcome};

litellm-rust/crates/secrets-cyberark/src/secret_manager.rs

@@ -0,0 +1,317 @@
+use std::{fs, sync::Arc, time::Duration};
+
+use base64::{Engine, engine::general_purpose::STANDARD};
+use litellm_core_utils::settings::Lookup;
+use litellm_secrets_types::{BaseSecretManager, SecretValue, validate_secret_name};
+use moka::future::Cache;
+use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode};
+
+use crate::Error;
+
+const CYBERARK_API_BASE: &str = "CYBERARK_API_BASE";
+const CYBERARK_ACCOUNT: &str = "CYBERARK_ACCOUNT";
+const CYBERARK_USERNAME: &str = "CYBERARK_USERNAME";
+const CYBERARK_API_KEY: &str = "CYBERARK_API_KEY";
+const CYBERARK_CLIENT_CERT: &str = "CYBERARK_CLIENT_CERT";
+const CYBERARK_CLIENT_KEY: &str = "CYBERARK_CLIENT_KEY";
+const CYBERARK_SSL_VERIFY: &str = "CYBERARK_SSL_VERIFY";
+const CYBERARK_REFRESH_INTERVAL: &str = "CYBERARK_REFRESH_INTERVAL";
+const DEFAULT_API_BASE: &str = "http://127.0.0.1:8080";
+const DEFAULT_ACCOUNT: &str = "default";
+const DEFAULT_USERNAME: &str = "admin";
+const DEFAULT_REFRESH_INTERVAL: Duration = Duration::from_secs(300);
+const SECRET_NAME_SAFE: &AsciiSet = &NON_ALPHANUMERIC
+    .remove(b'-')
+    .remove(b'_')
+    .remove(b'.')
+    .remove(b'~');
+
+#[derive(Clone)]
+pub struct CyberArkSecretManager {
+    client: reqwest::Client,
+    endpoint: reqwest::Url,
+    account: String,
+    username: String,
+    api_key: SecretValue,
+    token: Cache<(), SecretValue>,
+    secrets: Cache<String, SecretValue>,
+    authentication_lock: Arc<tokio::sync::Mutex<()>>,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum DeleteOutcome {
+    NotSupported,
+}
+
+impl CyberArkSecretManager {
+    pub fn with_client(
+        client: reqwest::Client,
+        endpoint: reqwest::Url,
+        account: String,
+        username: String,
+        api_key: SecretValue,
+        refresh_interval: Option<Duration>,
+    ) -> Self {
+        let endpoint = normalize_endpoint(endpoint);
+        let ttl = refresh_interval
+            .filter(|interval| !interval.is_zero())
+            .unwrap_or(DEFAULT_REFRESH_INTERVAL);
+        let token = Cache::builder().time_to_live(ttl).build();
+        let secrets = Cache::builder().time_to_live(ttl).build();
+        Self {
+            client,
+            endpoint,
+            account,
+            username,
+            api_key,
+            token,
+            secrets,
+            authentication_lock: Arc::new(tokio::sync::Mutex::new(())),
+        }
+    }
+
+    pub fn new(
+        environment: Arc<dyn Lookup + Send + Sync>,
+        enterprise_enabled: bool,
+    ) -> Result<Self, Error> {
+        let api_key = environment.get(CYBERARK_API_KEY).unwrap_or_default();
+        let cert = environment.get(CYBERARK_CLIENT_CERT).unwrap_or_default();
+        let key = environment.get(CYBERARK_CLIENT_KEY).unwrap_or_default();
+        if api_key.is_empty() && (cert.is_empty() || key.is_empty()) {
+            return Err(Error::MissingCredentials);
+        }
+        if !enterprise_enabled {
+            return Err(Error::EnterpriseRequired);
+        }
+        let verify = environment
+            .get(CYBERARK_SSL_VERIFY)
+            .map(|value| !value.trim().eq_ignore_ascii_case("false"))
+            .unwrap_or(true);
+        let mut builder = reqwest::Client::builder();
+        if !verify {
+            tracing::warn!(
+                "CyberArk SSL verification is disabled. This is insecure and should only be used for testing with self-signed certificates."
+            );
+            builder = builder.danger_accept_invalid_certs(true);
+        }
+        if !cert.is_empty() && !key.is_empty() {
+            let certificate = fs::read(cert).map_err(|_| Error::ClientCertificate)?;
+            let private_key = fs::read(key).map_err(|_| Error::ClientCertificate)?;
+            let identity = reqwest::Identity::from_pem(&[certificate, private_key].concat())
+                .map_err(|_| Error::ClientCertificate)?;
+            builder = builder.identity(identity);
+        }
+        let client = builder.build()?;
+        let endpoint = reqwest::Url::parse(
+            &environment
+                .get(CYBERARK_API_BASE)
+                .unwrap_or_else(|| DEFAULT_API_BASE.to_owned()),
+        )
+        .map_err(|_| Error::Endpoint)?;
+        let account = environment
+            .get(CYBERARK_ACCOUNT)
+            .unwrap_or_else(|| DEFAULT_ACCOUNT.to_owned());
+        let username = environment
+            .get(CYBERARK_USERNAME)
+            .unwrap_or_else(|| DEFAULT_USERNAME.to_owned());
+        let refresh_interval = environment
+            .get(CYBERARK_REFRESH_INTERVAL)
+            .map(|value| {
+                value
+                    .parse::<u64>()
+                    .map(Duration::from_secs)
+                    .map_err(|_| Error::RefreshInterval)
+            })
+            .transpose()?;
+        Ok(Self::with_client(
+            client,
+            endpoint,
+            account,
+            username,
+            SecretValue::new(api_key),
+            refresh_interval,
+        ))
+    }
+
+    fn secret_url(&self, name: &str) -> Result<reqwest::Url, Error> {
+        let encoded = utf8_percent_encode(name, SECRET_NAME_SAFE);
+        self.endpoint
+            .join(&format!("secrets/{}/variable/{}", self.account, encoded))
+            .map_err(|_| Error::Endpoint)
+    }
+
+    async fn authenticate(&self) -> Result<SecretValue, Error> {
+        if let Some(token) = self.token.get(&()).await {
+            return Ok(token);
+        }
+        let _guard = self.authentication_lock.lock().await;
+        if let Some(token) = self.token.get(&()).await {
+            return Ok(token);
+        }
+        let url = self
+            .endpoint
+            .join(&format!(
+                "authn/{}/{}/authenticate",
+                self.account, self.username
+            ))
+            .map_err(|_| Error::Endpoint)?;
+        let response = self
+            .client
+            .post(url)
+            .body(self.api_key.expose().to_owned())
+            .send()
+            .await?;
+        if !response.status().is_success() {
+            return Err(Error::AuthStatus(response.status().as_u16()));
+        }
+        let token = SecretValue::new(STANDARD.encode(response.text().await?));
+        self.token.insert((), token.clone()).await;
+        Ok(token)
+    }
+
+    async fn authorization_header(&self) -> Result<String, Error> {
+        Ok(format!(
+            "Token token=\"{}\"",
+            self.authenticate().await?.expose()
+        ))
+    }
+
+    pub async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
+        if let Some(value) = self.secrets.get(name).await {
+            return Ok(Some(value));
+        }
+        let response = self
+            .client
+            .get(self.secret_url(name)?)
+            .header("Authorization", self.authorization_header().await?)
+            .send()
+            .await?;
+        if response.status() == reqwest::StatusCode::NOT_FOUND {
+            return Ok(None);
+        }
+        if !response.status().is_success() {
+            return Err(Error::Status(response.status().as_u16()));
+        }
+        let value = SecretValue::new(response.text().await?);
+        self.secrets.insert(name.to_owned(), value.clone()).await;
+        Ok(Some(value))
+    }
+
+    pub async fn async_write_secret(
+        &self,
+        name: &str,
+        value: &SecretValue,
+        _description: Option<&str>,
+    ) -> Result<(), Error> {
+        validate_secret_name(name)?;
+        self.ensure_variable_exists(name).await;
+        let response = self
+            .client
+            .post(self.secret_url(name)?)
+            .header("Authorization", self.authorization_header().await?)
+            .body(value.expose().to_owned())
+            .send()
+            .await?;
+        if !response.status().is_success() {
+            return Err(Error::Status(response.status().as_u16()));
+        }
+        self.secrets.insert(name.to_owned(), value.clone()).await;
+        Ok(())
+    }
+
+    async fn ensure_variable_exists(&self, name: &str) {
+        let policy_url = self
+            .endpoint
+            .join(&format!("policies/{}/policy/root", self.account));
+        let Ok(policy_url) = policy_url else {
+            tracing::warn!("Could not build CyberArk policy endpoint");
+            return;
+        };
+        let Ok(authorization) = self.authorization_header().await else {
+            tracing::warn!("Could not authenticate while ensuring CyberArk variable exists");
+            return;
+        };
+        let body = format!(
+            "- !variable {}\n",
+            serde_json::to_string(name).expect("serializing a string cannot fail")
+        );
+        let response = self
+            .client
+            .post(policy_url)
+            .header("Authorization", authorization)
+            .header("Content-Type", "application/x-yaml")
+            .body(body)
+            .send()
+            .await;
+        match response {
+            Ok(response) if response.status().is_success() => {}
+            Ok(response)
+                if matches!(
+                    response.status(),
+                    reqwest::StatusCode::CONFLICT | reqwest::StatusCode::UNPROCESSABLE_ENTITY
+                ) =>
+            {
+                tracing::debug!(
+                    "CyberArk variable policy already exists or conflicts: {}",
+                    response.status()
+                );
+            }
+            Ok(response) => {
+                tracing::warn!(
+                    "Could not ensure CyberArk variable exists: {}",
+                    response.status()
+                );
+            }
+            Err(error) => {
+                tracing::warn!("Error ensuring CyberArk variable exists: {error}");
+            }
+        }
+    }
+
+    pub async fn async_delete_secret(
+        &self,
+        name: &str,
+        _recovery_window_in_days: i64,
+    ) -> Result<DeleteOutcome, Error> {
+        tracing::warn!(
+            "CyberArk Conjur does not support direct secret deletion. Secrets must be removed through policy updates."
+        );
+        self.secrets.invalidate(name).await;
+        Ok(DeleteOutcome::NotSupported)
+    }
+}
+
+impl BaseSecretManager for CyberArkSecretManager {
+    type Error = Error;
+    type WriteResponse = ();
+    type DeleteResponse = DeleteOutcome;
+
+    async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
+        self.async_read_secret(name).await
+    }
+
+    async fn async_write_secret(
+        &self,
+        name: &str,
+        value: &SecretValue,
+        description: Option<&str>,
+    ) -> Result<(), Error> {
+        self.async_write_secret(name, value, description).await
+    }
+
+    async fn async_delete_secret(
+        &self,
+        name: &str,
+        recovery_window_in_days: i64,
+    ) -> Result<DeleteOutcome, Error> {
+        self.async_delete_secret(name, recovery_window_in_days)
+            .await
+    }
+}
+
+fn normalize_endpoint(mut endpoint: reqwest::Url) -> reqwest::Url {
+    if !endpoint.path().ends_with('/') {
+        endpoint.set_path(&format!("{}/", endpoint.path()));
+    }
+    endpoint
+}

litellm-rust/crates/secrets-cyberark/tests/fixtures/parity.json

@@ -0,0 +1,32 @@
+{
+  "endpoint": "http://conjur.test:8080",
+  "account": "acct",
+  "username": "admin",
+  "api_key": "k3y",
+  "authenticate_path": "/authn/acct/admin/authenticate",
+  "token_json": "{\"protected\":\"p\",\"payload\":\"q\",\"signature\":\"s\"}",
+  "authorization_header": "Token token=\"eyJwcm90ZWN0ZWQiOiJwIiwicGF5bG9hZCI6InEiLCJzaWduYXR1cmUiOiJzIn0=\"",
+  "policy_path": "/policies/acct/policy/root",
+  "secrets": [
+    {
+      "name": "OPENAI_API_KEY",
+      "path": "/secrets/acct/variable/OPENAI_API_KEY",
+      "policy_body": "- !variable \"OPENAI_API_KEY\"\n"
+    },
+    {
+      "name": "team/app/key",
+      "path": "/secrets/acct/variable/team%2Fapp%2Fkey",
+      "policy_body": "- !variable \"team/app/key\"\n"
+    },
+    {
+      "name": "a b+c.d-e_f~g",
+      "path": "/secrets/acct/variable/a%20b%2Bc.d-e_f~g",
+      "policy_body": "- !variable \"a b+c.d-e_f~g\"\n"
+    },
+    {
+      "name": "needs \"quote\"",
+      "path": "/secrets/acct/variable/needs%20%22quote%22",
+      "policy_body": "- !variable \"needs \\\"quote\\\"\"\n"
+    }
+  ]
+}

litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs

@@ -0,0 +1,516 @@
+use std::{sync::Arc, time::Duration};
+
+use base64::{Engine, engine::general_purpose::STANDARD};
+use litellm_secrets_cyberark::{CyberArkSecretManager, DeleteOutcome, Error};
+use litellm_secrets_types::SecretValue;
+use serde::Deserialize;
+use wiremock::{
+    Match, Mock, MockServer, Request, ResponseTemplate,
+    matchers::{body_string, header, method, path},
+};
+
+const TOKEN_JSON: &str = r#"{"protected":"p","payload":"q","signature":"s"}"#;
+
+#[derive(Deserialize)]
+struct ParityFixture {
+    endpoint: String,
+    account: String,
+    username: String,
+    api_key: String,
+    authenticate_path: String,
+    token_json: String,
+    authorization_header: String,
+    policy_path: String,
+    secrets: Vec<ParitySecret>,
+}
+
+#[derive(Deserialize)]
+struct ParitySecret {
+    name: String,
+    path: String,
+    policy_body: String,
+}
+
+#[derive(Debug)]
+struct RawPath(String);
+
+impl Match for RawPath {
+    fn matches(&self, request: &Request) -> bool {
+        request.url.path() == self.0
+    }
+}
+
+fn fixture() -> ParityFixture {
+    serde_json::from_str(include_str!("fixtures/parity.json")).unwrap()
+}
+
+fn manager(server: &MockServer, ttl: Duration) -> CyberArkSecretManager {
+    CyberArkSecretManager::with_client(
+        reqwest::Client::new(),
+        server.uri().parse().unwrap(),
+        "acct".into(),
+        "admin".into(),
+        SecretValue::new("k3y"),
+        Some(ttl),
+    )
+}
+
+async fn mount_auth(server: &MockServer, expected: u64) {
+    Mock::given(method("POST"))
+        .and(path("/authn/acct/admin/authenticate"))
+        .and(body_string("k3y"))
+        .respond_with(ResponseTemplate::new(200).set_body_string(TOKEN_JSON))
+        .expect(expected)
+        .mount(server)
+        .await;
+}
+
+#[tokio::test]
+async fn successful_reads_cache_auth_secret_and_redact_values() {
+    let server = MockServer::start().await;
+    mount_auth(&server, 1).await;
+    let token = STANDARD.encode(TOKEN_JSON);
+    Mock::given(path("/secrets/acct/variable/OPENAI_API_KEY"))
+        .and(header("authorization", format!("Token token=\"{token}\"")))
+        .respond_with(ResponseTemplate::new(200).set_body_string("sk-live"))
+        .expect(1)
+        .mount(&server)
+        .await;
+    let manager = manager(&server, Duration::from_secs(60));
+
+    for _ in 0..2 {
+        let value = manager
+            .async_read_secret("OPENAI_API_KEY")
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(value.expose(), "sk-live");
+        assert!(!format!("{value:?}").contains("sk-live"));
+    }
+}
+
+#[tokio::test]
+async fn concurrent_reads_share_authentication_request() {
+    let server = MockServer::start().await;
+    Mock::given(path("/authn/acct/admin/authenticate"))
+        .and(body_string("k3y"))
+        .respond_with(
+            ResponseTemplate::new(200)
+                .set_body_string(TOKEN_JSON)
+                .set_delay(Duration::from_millis(20)),
+        )
+        .expect(1)
+        .mount(&server)
+        .await;
+    Mock::given(path("/secrets/acct/variable/key"))
+        .and(header(
+            "authorization",
+            format!("Token token=\"{}\"", STANDARD.encode(TOKEN_JSON)),
+        ))
+        .respond_with(ResponseTemplate::new(200).set_body_string("value"))
+        .expect(2)
+        .mount(&server)
+        .await;
+    let manager = manager(&server, Duration::from_secs(60));
+
+    let (first, second) = tokio::join!(
+        manager.async_read_secret("key"),
+        manager.async_read_secret("key")
+    );
+
+    assert_eq!(first.unwrap().unwrap().expose(), "value");
+    assert_eq!(second.unwrap().unwrap().expose(), "value");
+}
+
+#[rstest::rstest]
+#[case::not_found(404)]
+#[case::unauthorized(401)]
+#[case::forbidden(403)]
+#[case::server_error(500)]
+#[tokio::test]
+async fn failed_reads_are_not_cached(#[case] status: u16) {
+    let server = MockServer::start().await;
+    mount_auth(&server, 1).await;
+    let failing = Mock::given(path("/secrets/acct/variable/key"))
+        .respond_with(ResponseTemplate::new(status))
+        .expect(1)
+        .mount_as_scoped(&server)
+        .await;
+    let manager = manager(&server, Duration::from_secs(60));
+    let result = manager.async_read_secret("key").await;
+    if status == 404 {
+        assert_eq!(result.unwrap(), None);
+    } else {
+        assert!(matches!(result, Err(Error::Status(actual)) if actual == status));
+    }
+    drop(failing);
+    Mock::given(path("/secrets/acct/variable/key"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("recovered"))
+        .expect(1)
+        .mount(&server)
+        .await;
+    for _ in 0..2 {
+        assert_eq!(
+            manager
+                .async_read_secret("key")
+                .await
+                .unwrap()
+                .unwrap()
+                .expose(),
+            "recovered"
+        );
+    }
+}
+
+#[tokio::test]
+async fn failed_authentication_is_not_cached_and_does_not_read_secret() {
+    let server = MockServer::start().await;
+    let failing = Mock::given(path("/authn/acct/admin/authenticate"))
+        .respond_with(ResponseTemplate::new(401))
+        .expect(1)
+        .mount_as_scoped(&server)
+        .await;
+    let unused_secret = Mock::given(path("/secrets/acct/variable/key"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("value"))
+        .expect(0)
+        .mount_as_scoped(&server)
+        .await;
+    let manager = manager(&server, Duration::from_secs(60));
+    assert!(matches!(
+        manager.async_read_secret("key").await,
+        Err(Error::AuthStatus(401))
+    ));
+    drop(unused_secret);
+    drop(failing);
+    mount_auth(&server, 1).await;
+    Mock::given(path("/secrets/acct/variable/key"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("value"))
+        .expect(1)
+        .mount(&server)
+        .await;
+    assert_eq!(
+        manager
+            .async_read_secret("key")
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "value"
+    );
+}
+
+#[tokio::test]
+async fn expired_tokens_and_secrets_are_fetched_again() {
+    let server = MockServer::start().await;
+    mount_auth(&server, 2).await;
+    Mock::given(path("/secrets/acct/variable/key"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("value"))
+        .expect(2)
+        .mount(&server)
+        .await;
+    let manager = manager(&server, Duration::from_millis(1));
+    for _ in 0..2 {
+        assert!(manager.async_read_secret("key").await.unwrap().is_some());
+        tokio::time::sleep(Duration::from_millis(5)).await;
+    }
+}
+
+#[rstest::rstest]
+#[tokio::test]
+async fn secret_names_use_python_quote_encoding(
+    #[values("OPENAI_API_KEY", "team/app/key", "a b+c.d-e_f~g", "needs \"quote\"")] name: &str,
+) {
+    let fixture = fixture();
+    let secret = fixture
+        .secrets
+        .iter()
+        .find(|secret| secret.name == name)
+        .unwrap();
+    let server = MockServer::start().await;
+    mount_auth(&server, 1).await;
+    Mock::given(RawPath(secret.path.clone()))
+        .respond_with(ResponseTemplate::new(200).set_body_string("value"))
+        .expect(1)
+        .mount(&server)
+        .await;
+    assert_eq!(
+        manager(&server, Duration::from_secs(60))
+            .async_read_secret(name)
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "value"
+    );
+}
+
+#[rstest::rstest]
+#[case(201)]
+#[case(409)]
+#[case(422)]
+#[case(500)]
+#[tokio::test]
+async fn writes_tolerate_policy_status_and_cache_value(#[case] policy_status: u16) {
+    let server = MockServer::start().await;
+    mount_auth(&server, 1).await;
+    Mock::given(path("/policies/acct/policy/root"))
+        .and(header("content-type", "application/x-yaml"))
+        .and(body_string("- !variable \"team/app\"\n"))
+        .respond_with(ResponseTemplate::new(policy_status))
+        .expect(1)
+        .mount(&server)
+        .await;
+    Mock::given(path("/secrets/acct/variable/team%2Fapp"))
+        .and(body_string("v"))
+        .respond_with(ResponseTemplate::new(200))
+        .expect(1)
+        .mount(&server)
+        .await;
+    let manager = manager(&server, Duration::from_secs(60));
+    manager
+        .async_write_secret("team/app", &SecretValue::new("v"), None)
+        .await
+        .unwrap();
+    assert_eq!(
+        manager
+            .async_read_secret("team/app")
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "v"
+    );
+}
+
+#[tokio::test]
+async fn failed_value_write_is_not_cached() {
+    let server = MockServer::start().await;
+    mount_auth(&server, 1).await;
+    Mock::given(path("/policies/acct/policy/root"))
+        .respond_with(ResponseTemplate::new(409))
+        .mount(&server)
+        .await;
+    Mock::given(path("/secrets/acct/variable/key"))
+        .and(body_string("v"))
+        .respond_with(ResponseTemplate::new(403))
+        .expect(1)
+        .mount(&server)
+        .await;
+    Mock::given(path("/secrets/acct/variable/key"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("recovered"))
+        .expect(1)
+        .mount(&server)
+        .await;
+    let manager = manager(&server, Duration::from_secs(60));
+    assert!(matches!(
+        manager
+            .async_write_secret("key", &SecretValue::new("v"), None)
+            .await,
+        Err(Error::Status(403))
+    ));
+    assert_eq!(
+        manager
+            .async_read_secret("key")
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "recovered"
+    );
+}
+
+#[tokio::test]
+async fn unsafe_names_fail_before_http_calls() {
+    let server = MockServer::start().await;
+    let manager = manager(&server, Duration::from_secs(60));
+    assert!(matches!(
+        manager
+            .async_write_secret("../etc", &SecretValue::new("v"), None)
+            .await,
+        Err(Error::Operation(
+            litellm_secrets_types::Error::UnsafeSecretName
+        ))
+    ));
+}
+
+#[tokio::test]
+async fn delete_invalidates_cache_and_reports_not_supported() {
+    let server = MockServer::start().await;
+    mount_auth(&server, 1).await;
+    Mock::given(path("/secrets/acct/variable/key"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("v"))
+        .expect(2)
+        .mount(&server)
+        .await;
+    let manager = manager(&server, Duration::from_secs(60));
+    assert_eq!(
+        manager
+            .async_read_secret("key")
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "v"
+    );
+    assert_eq!(
+        manager.async_delete_secret("key", 7).await.unwrap(),
+        DeleteOutcome::NotSupported
+    );
+    assert_eq!(
+        manager
+            .async_read_secret("key")
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "v"
+    );
+}
+
+#[test]
+fn new_validates_credentials_before_license_and_configuration() {
+    let empty: Arc<dyn litellm_core_utils::settings::Lookup + Send + Sync> =
+        Arc::new(|_: &str| None);
+    assert!(matches!(
+        CyberArkSecretManager::new(empty, true),
+        Err(Error::MissingCredentials)
+    ));
+    assert!(matches!(
+        CyberArkSecretManager::new(
+            Arc::new(|name: &str| (name == "CYBERARK_API_KEY").then(|| "k3y".into())),
+            false
+        ),
+        Err(Error::EnterpriseRequired)
+    ));
+    assert!(matches!(
+        CyberArkSecretManager::new(
+            Arc::new(|name: &str| (name == "CYBERARK_CLIENT_CERT").then(|| "cert".into())),
+            true
+        ),
+        Err(Error::MissingCredentials)
+    ));
+    assert!(matches!(
+        CyberArkSecretManager::new(
+            Arc::new(|name: &str| match name {
+                "CYBERARK_API_KEY" => Some("k3y".into()),
+                "CYBERARK_REFRESH_INTERVAL" => Some("abc".into()),
+                _ => None,
+            }),
+            true
+        ),
+        Err(Error::RefreshInterval)
+    ));
+    assert!(matches!(
+        CyberArkSecretManager::new(
+            Arc::new(|name: &str| match name {
+                "CYBERARK_API_KEY" => Some("k3y".into()),
+                "CYBERARK_API_BASE" => Some("not a url".into()),
+                _ => None,
+            }),
+            true
+        ),
+        Err(Error::Endpoint)
+    ));
+}
+
+#[tokio::test]
+async fn new_reads_environment_defaults_end_to_end() {
+    let server = MockServer::start().await;
+    Mock::given(path("/authn/default/admin/authenticate"))
+        .and(body_string("k3y"))
+        .respond_with(ResponseTemplate::new(200).set_body_string(TOKEN_JSON))
+        .mount(&server)
+        .await;
+    Mock::given(path("/secrets/default/variable/key"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("value"))
+        .mount(&server)
+        .await;
+    let endpoint = server.uri();
+    let manager = CyberArkSecretManager::new(
+        Arc::new(move |name: &str| match name {
+            "CYBERARK_API_BASE" => Some(endpoint.clone()),
+            "CYBERARK_API_KEY" => Some("k3y".into()),
+            _ => None,
+        }),
+        true,
+    )
+    .unwrap();
+    assert_eq!(
+        manager
+            .async_read_secret("key")
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "value"
+    );
+}
+
+#[test]
+fn new_reports_missing_client_certificate_files() {
+    assert!(matches!(
+        CyberArkSecretManager::new(
+            Arc::new(|name: &str| match name {
+                "CYBERARK_CLIENT_CERT" => Some("/missing/cert".into()),
+                "CYBERARK_CLIENT_KEY" => Some("/missing/key".into()),
+                _ => None,
+            }),
+            true
+        ),
+        Err(Error::ClientCertificate)
+    ));
+}
+
+#[tokio::test]
+async fn trailing_slash_endpoint_preserves_base_path() {
+    let server = MockServer::start().await;
+    Mock::given(path("/prefix/authn/acct/admin/authenticate"))
+        .and(body_string("k3y"))
+        .respond_with(ResponseTemplate::new(200).set_body_string(TOKEN_JSON))
+        .expect(1)
+        .mount(&server)
+        .await;
+    Mock::given(path("/prefix/secrets/acct/variable/key"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("value"))
+        .mount(&server)
+        .await;
+    let endpoint = format!("{}/prefix/", server.uri()).parse().unwrap();
+    let manager = CyberArkSecretManager::with_client(
+        reqwest::Client::new(),
+        endpoint,
+        "acct".into(),
+        "admin".into(),
+        SecretValue::new("k3y"),
+        Some(Duration::from_secs(60)),
+    );
+    assert_eq!(
+        manager
+            .async_read_secret("key")
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "value"
+    );
+}
+
+#[test]
+fn parity_fixture_matches_authentication_contract() {
+    let fixture = fixture();
+    assert_eq!(fixture.endpoint, "http://conjur.test:8080");
+    assert_eq!(fixture.account, "acct");
+    assert_eq!(fixture.username, "admin");
+    assert_eq!(fixture.api_key, "k3y");
+    assert_eq!(fixture.authenticate_path, "/authn/acct/admin/authenticate");
+    assert_eq!(fixture.token_json, TOKEN_JSON);
+    assert_eq!(
+        fixture.authorization_header,
+        format!("Token token=\"{}\"", STANDARD.encode(TOKEN_JSON))
+    );
+    assert_eq!(fixture.policy_path, "/policies/acct/policy/root");
+    assert_eq!(fixture.secrets.len(), 4);
+    assert_eq!(
+        fixture.secrets[1].policy_body,
+        "- !variable \"team/app/key\"\n"
+    );
+}

litellm-rust/crates/secrets/Cargo.toml

@@ -9,11 +9,15 @@ repository.workspace = true
 default = []
 aws = ["dep:litellm-secrets-aws"]
 google = ["dep:litellm-secrets-google"]
+azure = ["dep:litellm-secrets-azure"]
+cyberark = ["dep:litellm-secrets-cyberark"]
 
 [dependencies]
 litellm-secrets-types.workspace = true
 litellm-secrets-aws = { workspace = true, optional = true }
 litellm-secrets-google = { workspace = true, optional = true }
+litellm-secrets-azure = { workspace = true, optional = true }
+litellm-secrets-cyberark = { workspace = true, optional = true }
 litellm-core-utils.workspace = true
 base64.workspace = true
 serde.workspace = true

litellm-rust/crates/secrets/src/error.rs

@@ -30,4 +30,10 @@ pub enum Error {
     #[cfg(feature = "google")]
     #[error(transparent)]
     Google(#[from] litellm_secrets_google::Error),
+    #[cfg(feature = "azure")]
+    #[error(transparent)]
+    Azure(#[from] litellm_secrets_azure::Error),
+    #[cfg(feature = "cyberark")]
+    #[error(transparent)]
+    Cyberark(#[from] litellm_secrets_cyberark::Error),
 }

litellm-rust/crates/secrets/src/handler.rs

@@ -13,6 +13,10 @@ pub enum SecretManager {
     GoogleKms(crate::google::GoogleKms),
     #[cfg(feature = "google")]
     GoogleSecretManager(crate::google::GoogleSecretManager),
+    #[cfg(feature = "azure")]
+    AzureKeyVault(crate::azure::AzureKeyVault),
+    #[cfg(feature = "cyberark")]
+    Cyberark(crate::cyberark::CyberArkSecretManager),
 }
 
 impl SecretManager {
@@ -27,6 +31,10 @@ impl SecretManager {
             Self::GoogleKms(_) => KeyManagementSystem::GoogleKms,
             #[cfg(feature = "google")]
             Self::GoogleSecretManager(_) => KeyManagementSystem::GoogleSecretManager,
+            #[cfg(feature = "azure")]
+            Self::AzureKeyVault(_) => KeyManagementSystem::AzureKeyVault,
+            #[cfg(feature = "cyberark")]
+            Self::Cyberark(_) => KeyManagementSystem::Cyberark,
         }
     }
 }
@@ -78,6 +86,17 @@ pub async fn get_secret_from_manager(
             .get_secret_from_google_secret_manager(secret_name)
             .await
             .map_err(Error::from),
+        #[cfg(feature = "azure")]
+        SecretManager::AzureKeyVault(client) => client
+            .get_secret_from_azure_key_vault(secret_name)
+            .await
+            .map_err(Error::from),
+        #[cfg(feature = "cyberark")]
+        SecretManager::Cyberark(client) => client
+            .async_read_secret(secret_name)
+            .await
+            .map(|value| value.map(Secret::String))
+            .map_err(Error::from),
     }
 }
 

litellm-rust/crates/secrets/src/lib.rs

@@ -17,5 +17,9 @@ pub use state::{SecretManagerState, secret_manager_would_be_consulted};
 
 #[cfg(feature = "aws")]
 pub use litellm_secrets_aws as aws;
+#[cfg(feature = "azure")]
+pub use litellm_secrets_azure as azure;
+#[cfg(feature = "cyberark")]
+pub use litellm_secrets_cyberark as cyberark;
 #[cfg(feature = "google")]
 pub use litellm_secrets_google as google;

litellm-rust/crates/secrets/tests/handler.rs

@@ -105,3 +105,117 @@ async fn google_handler_requires_canonical_base64_and_preserves_plaintext_whites
         Err(Error::MissingCiphertext)
     ));
 }
+
+#[cfg(feature = "azure")]
+#[tokio::test]
+async fn azure_handler_reads_missing_and_failed_secrets() {
+    use litellm_secrets::{
+        Error, KeyManagementSettings, KeyManagementSystem, SecretManager, azure::AzureKeyVault,
+        get_secret_from_manager,
+    };
+    use wiremock::{
+        Mock, MockServer, ResponseTemplate,
+        matchers::{path, query_param},
+    };
+
+    let server = MockServer::start().await;
+    Mock::given(path("/secrets/KEY"))
+        .and(query_param("api-version", "7.4"))
+        .respond_with(
+            ResponseTemplate::new(200).set_body_json(serde_json::json!({"value": "value"})),
+        )
+        .expect(1)
+        .mount(&server)
+        .await;
+    let manager = SecretManager::AzureKeyVault(
+        AzureKeyVault::with_client(
+            reqwest::Client::new(),
+            server.uri().parse().unwrap(),
+            std::sync::Arc::new(|name: &str| (name == "AZURE_AD_TOKEN").then(|| "fake".to_owned())),
+        )
+        .unwrap(),
+    );
+    assert_eq!(manager.system(), KeyManagementSystem::AzureKeyVault);
+    let settings = KeyManagementSettings::default();
+    let value = get_secret_from_manager(&manager, "KEY", &settings, &|_: &str| None)
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(value.as_str(), Some("value"));
+
+    let not_found = Mock::given(path("/secrets/MISSING"))
+        .respond_with(ResponseTemplate::new(404))
+        .expect(1)
+        .mount_as_scoped(&server)
+        .await;
+    assert_eq!(
+        get_secret_from_manager(&manager, "MISSING", &settings, &|_: &str| None)
+            .await
+            .unwrap(),
+        None
+    );
+    drop(not_found);
+
+    Mock::given(path("/secrets/FAILED"))
+        .respond_with(ResponseTemplate::new(500))
+        .expect(1)
+        .mount(&server)
+        .await;
+    assert!(matches!(
+        get_secret_from_manager(&manager, "FAILED", &settings, &|_: &str| None).await,
+        Err(Error::Azure(_))
+    ));
+}
+
+#[cfg(feature = "cyberark")]
+#[tokio::test]
+async fn cyberark_handler_reads_values_and_surfaces_errors() {
+    use std::time::Duration;
+
+    use litellm_secrets::{
+        Error, KeyManagementSettings, SecretManager, SecretValue, cyberark::CyberArkSecretManager,
+        get_secret_from_manager,
+    };
+    use wiremock::{
+        Mock, MockServer, ResponseTemplate,
+        matchers::{body_string, path},
+    };
+
+    let server = MockServer::start().await;
+    Mock::given(path("/authn/acct/admin/authenticate"))
+        .and(body_string("k3y"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("token"))
+        .mount(&server)
+        .await;
+    Mock::given(path("/secrets/acct/variable/KEY"))
+        .respond_with(ResponseTemplate::new(200).set_body_string("value"))
+        .mount(&server)
+        .await;
+    let manager = SecretManager::Cyberark(CyberArkSecretManager::with_client(
+        reqwest::Client::new(),
+        server.uri().parse().unwrap(),
+        "acct".into(),
+        "admin".into(),
+        SecretValue::new("k3y"),
+        Some(Duration::from_secs(60)),
+    ));
+    assert_eq!(
+        manager.system(),
+        litellm_secrets::KeyManagementSystem::Cyberark
+    );
+    let settings = KeyManagementSettings::default();
+    let value = get_secret_from_manager(&manager, "KEY", &settings, &|_: &str| None)
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(value.as_str(), Some("value"));
+
+    Mock::given(path("/secrets/acct/variable/ERROR"))
+        .respond_with(ResponseTemplate::new(500))
+        .mount(&server)
+        .await;
+    assert!(matches!(
+        get_secret_from_manager(&manager, "ERROR", &settings, &|_: &str| None).await,
+        Err(Error::Cyberark(_))
+    ));
+}

litellm/__init__.py

@@ -689,6 +689,7 @@ recraft_models: Set = set()
 cometapi_models: Set = set()
 oci_models: Set = set()
 vercel_ai_gateway_models: Set = set()
+edenai_models: Set = set()  # mutable-ok: filled from the price map at import, like the sibling provider sets
 volcengine_models: Set = set()
 wandb_models: Set = set(WANDB_MODELS)
 ovhcloud_models: Set = set()
@@ -763,6 +764,8 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None:
             openrouter_models.add(key)
         elif value.get("litellm_provider") == "vercel_ai_gateway":
             vercel_ai_gateway_models.add(key)
+        elif value.get("litellm_provider") == "edenai":
+            edenai_models.add(key)
         elif value.get("litellm_provider") == "datarobot":
             datarobot_models.add(key)
         elif value.get("litellm_provider") == "vertex_ai-text-models":
@@ -1111,6 +1114,7 @@ model_list = list(
     | oci_models
     | heroku_models
     | vercel_ai_gateway_models
+    | edenai_models
     | volcengine_models
     | wandb_models
     | ovhcloud_models
@@ -1139,6 +1143,7 @@ def _build_models_by_provider() -> dict:
         "baseten": baseten_models,
         "openrouter": openrouter_models,
         "vercel_ai_gateway": vercel_ai_gateway_models,
+        "edenai": edenai_models,
         "datarobot": datarobot_models,
         "vertex_ai": vertex_chat_models
         | vertex_text_models
@@ -2117,6 +2122,30 @@ if TYPE_CHECKING:
     from .llms.vercel_ai_gateway.chat.transformation import (
         VercelAIGatewayConfig as VercelAIGatewayConfig,
     )
+    from .llms.edenai.chat.transformation import (
+        EdenAIChatConfig as EdenAIChatConfig,
+    )
+    from .llms.edenai.responses.transformation import (
+        EdenAIResponsesAPIConfig as EdenAIResponsesAPIConfig,
+    )
+    from .llms.edenai.messages.transformation import (
+        EdenAIAnthropicMessagesConfig as EdenAIAnthropicMessagesConfig,
+    )
+    from .llms.edenai.embedding.transformation import (
+        EdenAIEmbeddingConfig as EdenAIEmbeddingConfig,
+    )
+    from .llms.edenai.audio_transcription.transformation import (
+        EdenAIAudioTranscriptionConfig as EdenAIAudioTranscriptionConfig,
+    )
+    from .llms.edenai.text_to_speech.transformation import (
+        EdenAITextToSpeechConfig as EdenAITextToSpeechConfig,
+    )
+    from .llms.edenai.image_generation.transformation import (
+        EdenAIImageGenerationConfig as EdenAIImageGenerationConfig,
+    )
+    from .llms.edenai.videos.transformation import (
+        EdenAIVideoConfig as EdenAIVideoConfig,
+    )
     from .llms.ovhcloud.chat.transformation import (
         OVHCloudChatConfig as OVHCloudChatConfig,
     )

litellm/_lazy_imports_registry.py

@@ -327,6 +327,14 @@ LLM_CONFIG_NAMES: Final = (
     "InceptionChatConfig",
     "HyperbolicChatConfig",
     "VercelAIGatewayConfig",
+    "EdenAIChatConfig",
+    "EdenAIResponsesAPIConfig",
+    "EdenAIAnthropicMessagesConfig",
+    "EdenAIEmbeddingConfig",
+    "EdenAIAudioTranscriptionConfig",
+    "EdenAITextToSpeechConfig",
+    "EdenAIImageGenerationConfig",
+    "EdenAIVideoConfig",
     "OVHCloudChatConfig",
     "OVHCloudEmbeddingConfig",
     "CometAPIEmbeddingConfig",
@@ -1232,6 +1240,17 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         ".llms.vercel_ai_gateway.chat.transformation",
         "VercelAIGatewayConfig",
     ),
+    "EdenAIChatConfig": (".llms.edenai.chat.transformation", "EdenAIChatConfig"),
+    "EdenAIResponsesAPIConfig": (".llms.edenai.responses.transformation", "EdenAIResponsesAPIConfig"),
+    "EdenAIAnthropicMessagesConfig": (".llms.edenai.messages.transformation", "EdenAIAnthropicMessagesConfig"),
+    "EdenAIEmbeddingConfig": (".llms.edenai.embedding.transformation", "EdenAIEmbeddingConfig"),
+    "EdenAIAudioTranscriptionConfig": (
+        ".llms.edenai.audio_transcription.transformation",
+        "EdenAIAudioTranscriptionConfig",
+    ),
+    "EdenAITextToSpeechConfig": (".llms.edenai.text_to_speech.transformation", "EdenAITextToSpeechConfig"),
+    "EdenAIImageGenerationConfig": (".llms.edenai.image_generation.transformation", "EdenAIImageGenerationConfig"),
+    "EdenAIVideoConfig": (".llms.edenai.videos.transformation", "EdenAIVideoConfig"),
     "OVHCloudChatConfig": (".llms.ovhcloud.chat.transformation", "OVHCloudChatConfig"),
     "OVHCloudEmbeddingConfig": (
         ".llms.ovhcloud.embedding.transformation",

litellm/a2a_protocol/providers/pydantic_ai_agents/config.py

@@ -23,7 +23,7 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig):
         params: dict[str, Any],
         api_base: str | None = None,
         **kwargs: Any,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """Handle non-streaming request to Pydantic AI agent."""
         if api_base is None:
             raise ValueError("api_base is required for PydanticAIProviderConfig")
@@ -41,7 +41,7 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig):
         params: dict[str, Any],
         api_base: str | None = None,
         **kwargs,
-    ) -> AsyncIterator[dict[str, Any]]:
+    ) -> AsyncIterator[dict[str, object]]:
         """Handle streaming request with fake streaming."""
         if not api_base:
             raise ValueError("api_base is required for Pydantic AI agents")

litellm/anthropic_beta_headers_config.json

@@ -11,6 +11,7 @@
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
+    "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
     "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": "fast-mode-2026-02-01",
     "files-api-2025-04-14": "files-api-2025-04-14",
@@ -44,6 +45,7 @@
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
+    "dangerous-tool-use-2026-09-03": null,
     "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": null,
     "files-api-2025-04-14": "files-api-2025-04-14",
@@ -76,6 +78,7 @@
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": null,
+    "dangerous-tool-use-2026-09-03": null,
     "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": null,
     "files-api-2025-04-14": null,
@@ -109,6 +112,7 @@
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
+    "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
     "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": null,
     "files-api-2025-04-14": null,
@@ -143,6 +147,7 @@
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
+    "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
     "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": null,
     "files-api-2025-04-14": null,
@@ -177,6 +182,7 @@
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
+    "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
     "effort-2025-11-24": null,
     "fast-mode-2026-02-01": null,
     "files-api-2025-04-14": null,
@@ -210,6 +216,7 @@
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
+    "dangerous-tool-use-2026-09-03": null,
     "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": "fast-mode-2026-02-01",
     "files-api-2025-04-14": "files-api-2025-04-14",

litellm/caching/caching.py

@@ -81,7 +81,7 @@ class Cache:
         s3_aws_access_key_id: str | None = None,
         s3_aws_secret_access_key: str | None = None,
         s3_aws_session_token: str | None = None,
-        s3_config: Any | None = None,
+        s3_config: object | None = None,
         s3_path: str | None = None,
         gcs_bucket_name: str | None = None,
         gcs_path_service_account: str | None = None,

litellm/caching/caching_handler.py

@@ -74,7 +74,7 @@ class CachingHandlerResponse(BaseModel):
     For embeddings there can be a cache hit for some of the inputs in the list and a cache miss for others
     """
 
-    cached_result: Any | None = None
+    cached_result: object | None = None
     final_embedding_cached_response: EmbeddingResponse | None = None
     embedding_all_elements_cache_hit: bool = False  # this is set to True when all elements in the list have a cache hit in the embedding cache, if true return the final_embedding_cached_response no need to make an API call
 
@@ -722,7 +722,7 @@ class LLMCachingHandler:
 
     async def _retrieve_from_cache(
         self, call_type: str, kwargs: dict[str, object], args: tuple[object, ...]
-    ) -> Any | None:
+    ) -> object | None:
         """
         Internal method to
         - get cache key
@@ -968,7 +968,7 @@ class LLMCachingHandler:
 
     def _convert_cached_stream_response(
         self,
-        cached_result: Any,
+        cached_result: dict[str, object],
         call_type: str,
         logging_obj: LiteLLMLoggingObj,
         model: str,
@@ -997,7 +997,7 @@ class LLMCachingHandler:
 
     async def async_set_cache(
         self,
-        result: Any,
+        result: object,
         original_function: Callable,
         kwargs: dict[str, Any],
         args: tuple[object, ...] | None = None,
@@ -1065,7 +1065,7 @@ class LLMCachingHandler:
 
     def sync_set_cache(
         self,
-        result: Any,
+        result: object,
         kwargs: dict[str, object],
         args: tuple[object, ...] | None = None,
     ):

litellm/caching/redis_cache.py

@@ -80,6 +80,8 @@ class _AsyncRedisCommands(Protocol):
 
     def ttl(self, name: str) -> Awaitable[int]: ...
 
+    def expire(self, name: str, time: int) -> Awaitable[bool]: ...
+
     def rpush(self, name: str, *values: str | bytes | float) -> Awaitable[int]: ...
 
     def lpop(self, name: str, count: int | None = None) -> Awaitable[object]: ...
@@ -979,7 +981,7 @@ class RedisCache(BaseCache):
             client: object = None,
         ) -> object:
             async def execute() -> object:
-                executor: Callable[..., Awaitable[Any]] | None = litellm.in_memory_llm_clients_cache.get_cache(
+                executor: Callable[..., Awaitable[object]] | None = litellm.in_memory_llm_clients_cache.get_cache(
                     key=script_cache_key
                 )
                 if executor is None:
@@ -991,7 +993,7 @@ class RedisCache(BaseCache):
 
         return run_script
 
-    def _register_script_for_current_loop(self, script: str) -> Callable[..., Awaitable[Any]]:
+    def _register_script_for_current_loop(self, script: str) -> Callable[..., Awaitable[object]]:
         """
         Register the script against the current event loop's Redis client.
 
@@ -1948,6 +1950,14 @@ class RedisCache(BaseCache):
             _record_swallowed_redis_failure(self._circuit_breaker, e)
             return None
 
+    @_redis_circuit_breaker_guard
+    async def async_refresh_ttl(self, key: str, ttl: int | None = None) -> bool:
+        """EXPIRE an existing key without touching its value. False when the key is absent."""
+        _used_ttl: Final = self.get_ttl(ttl=ttl)
+        if _used_ttl is None:
+            return False
+        return await self._async_commands().expire(self.check_and_fix_namespace(key=key), _used_ttl)
+
     @_redis_circuit_breaker_guard
     async def async_rpush(
         self,

litellm/completion_extras/litellm_responses_transformation/handler.py

@@ -2,7 +2,7 @@
 Handler for transforming /chat/completions api requests to litellm.responses requests
 """
 
-from collections.abc import Coroutine
+from collections.abc import AsyncIterable, Coroutine, Iterable
 from typing import TYPE_CHECKING, Any, Final, Union
 
 from typing_extensions import TypedDict
@@ -74,7 +74,7 @@ class ResponsesToCompletionBridgeHandler:
                     existing.setdefault(key, value)
         return response
 
-    def _collect_response_from_stream(self, stream_iter: Any) -> "ResponsesAPIResponse":
+    def _collect_response_from_stream(self, stream_iter: Iterable[object]) -> "ResponsesAPIResponse":
         for _ in stream_iter:
             pass
 
@@ -89,7 +89,7 @@ class ResponsesToCompletionBridgeHandler:
             raise ValueError("Stream completed response is invalid")
         return response
 
-    async def _collect_response_from_stream_async(self, stream_iter: Any) -> "ResponsesAPIResponse":
+    async def _collect_response_from_stream_async(self, stream_iter: AsyncIterable[object]) -> "ResponsesAPIResponse":
         async for _ in stream_iter:
             pass
 

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -6,7 +6,7 @@ import json
 import os
 from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast, get_args
+from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, Union, cast, get_args
 
 from openai.types.chat import ChatCompletion
 from openai.types.responses import Response
@@ -52,7 +52,7 @@ from litellm.types.llms.openai import (
 from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
 
 if TYPE_CHECKING:
-    from openai.types.responses import ResponseInputImageParam
+    from openai.types.responses import ResponseInputImageParam, ResponseOutputItem
     from openai.types.responses.response_text_config_param import (
         ResponseTextConfigParam as ResponseText,
     )
@@ -197,6 +197,9 @@ def _as_chat_reasoning_items(
     return cast(list[ChatCompletionReasoningItem], list(reasoning_items))
 
 
+_ToolChoiceT = TypeVar("_ToolChoiceT")
+
+
 def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Literal["length", "content_filter"]:
     if incomplete_reason == "content_filter":
         return "content_filter"
@@ -291,7 +294,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
     def __init__(self):
         pass
 
-    def _normalize_tool_choice_for_responses_api(self, tool_choice: Any) -> Any:
+    def _normalize_tool_choice_for_responses_api(
+        self, tool_choice: _ToolChoiceT
+    ) -> _ToolChoiceT | ToolChoiceFunctionParam | ToolChoiceCustomParam | Literal["auto", "none", "required"]:
         """Chat tool_choice nests the name under function/custom; Responses API expects top-level name."""
         if not isinstance(tool_choice, dict):
             return tool_choice
@@ -497,7 +502,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                 responses_api_request["max_output_tokens"] = value
             elif key == "tools" and value is not None:
                 responses_api_request["tools"] = self._convert_tools_to_responses_format(
-                    cast(list[dict[str, Any]], value)
+                    cast(list[dict[str, object]], value)
                 )
             elif key == "response_format":
                 text_format = self._transform_response_format_to_text_format(value)
@@ -828,7 +833,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         response_output: Final = response_payload.get("output")
         if not isinstance(response_output, list) or len(response_output) == 0:
             return None
-        return cast(list[dict[str, Any]], response_output)
+        return cast(list[dict[str, object]], response_output)
 
     @classmethod
     def _recover_output_items_from_raw_sse(cls, raw_sse: str | None) -> list[dict[str, object]]:
@@ -911,10 +916,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
         output_items = raw_response.output
         if len(output_items) == 0:
-            recovered_output_items: Final = self._recover_output_items_from_logging(logging_obj)
+            recovered_output_items: Final[list[ResponseOutputItem | dict[str, object]]] = [
+                *self._recover_output_items_from_logging(logging_obj)
+            ]
             if recovered_output_items:
-                output_items = cast(Any, recovered_output_items)
-                raw_response.output = cast(Any, recovered_output_items)
+                output_items = recovered_output_items
+                raw_response.output = recovered_output_items
                 verbose_logger.warning(
                     "Recovered empty Responses API output from raw SSE for model=%s",
                     model,
@@ -1110,7 +1117,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             verbose_logger.debug("Chat provider: Other content type -> %s", result)
             return result
 
-    def _convert_tools_to_responses_format(self, tools: list[dict[str, Any]]) -> list["ALL_RESPONSES_API_TOOL_PARAMS"]:
+    def _convert_tools_to_responses_format(
+        self, tools: list[dict[str, object]]
+    ) -> list["ALL_RESPONSES_API_TOOL_PARAMS"]:
         """Convert chat completion tools to responses API tools format"""
         responses_tools: Final[list[ALL_RESPONSES_API_TOOL_PARAMS]] = []
         for tool in tools:
@@ -1126,12 +1135,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         description=function_tool.get("description"),
                     )
                 )
-            elif tool.get("type") == "custom" and isinstance(tool.get("custom"), dict):
+            elif tool.get("type") == "custom" and isinstance(custom_payload := tool.get("custom"), dict):
                 from litellm.litellm_core_utils.prompt_templates.common_utils import (
                     convert_custom_tool_format_to_responses_shape,
                 )
 
-                custom_payload = tool["custom"]
                 flat_custom = CustomToolParam(type="custom", name=custom_payload.get("name", ""))
                 if custom_payload.get("description") is not None:
                     flat_custom["description"] = custom_payload["description"]

litellm/constants.py

@@ -750,6 +750,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
     "inception",
     "vercel_ai_gateway",
     "wandb",
+    "edenai",
     "ovhcloud",
     "lemonade",
     "docker_model_runner",
@@ -925,6 +926,7 @@ openai_compatible_endpoints: Final[list] = [
     "https://api.hyperbolic.xyz/v1",
     "https://ai-gateway.helicone.ai/",
     "https://ai-gateway.vercel.sh/v1",
+    "https://api.edenai.run/v3",
     "https://api.inference.wandb.ai/v1",
     "https://api.clarifai.com/v2/ext/openai/v1",
     "https://api.libertai.io/v1",
@@ -994,6 +996,7 @@ openai_compatible_providers: Final[list] = [
     "hyperbolic",
     "vercel_ai_gateway",
     "aiml",
+    "edenai",
     "wandb",
     "cometapi",
     "clarifai",
@@ -2118,3 +2121,6 @@ BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY: Final = "batch_enqueued_token_limit"
 # Shared read-only empty mapping, for defaulting optional Mapping parameters without
 # constructing a fresh mutable dict at each call site.
 EMPTY_MAPPING: Final = MappingProxyType({})
+
+# API endpoint for breached password k-anonymity search
+HIBP_RANGE_API_BASE: Final = "https://api.pwnedpasswords.com/range"

litellm/cost_calculator.py

@@ -353,7 +353,7 @@ def cost_per_token(
     data_residency: str | None = None,  # for OpenAI regional-processing uplift (e.g. "eu", "us")
     ### VERTEX LOCATION ###
     vertex_location: str | None = None,  # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
-    response: Any | None = None,
+    response: object | None = None,
     ### REQUEST MODEL ###
     request_model: str | None = None,  # original request model for router detection
     custom_model_info: OCRPricing | None = None,
@@ -609,7 +609,7 @@ def cost_per_token(
             model=model,
             custom_llm_provider=custom_llm_provider,
             number_of_queries=number_of_queries or 1,
-            optional_params=(response._hidden_params if response and hasattr(response, "_hidden_params") else None),
+            optional_params=(getattr(response, "_hidden_params", None) if response else None),
         )
     elif custom_llm_provider == "vertex_ai":
         cost_router: Final = google_cost_router(
@@ -999,7 +999,7 @@ def _is_known_usage_objects(usage_obj):
     )
 
 
-def _infer_call_type(call_type: CallTypesLiteral | None, completion_response: Any) -> CallTypesLiteral | None:
+def _infer_call_type(call_type: CallTypesLiteral | None, completion_response: object) -> CallTypesLiteral | None:
     if call_type is not None:
         return call_type
 

litellm/images/main.py

@@ -388,6 +388,7 @@ def image_generation(
             litellm.LlmProviders.DASHSCOPE,
             litellm.LlmProviders.QWENCLOUD,
             litellm.LlmProviders.QWEN_AI_PLATFORM,
+            litellm.LlmProviders.EDENAI,
         ):
             if image_generation_config is None:
                 raise ValueError(f"image generation config is not supported for {custom_llm_provider}")

litellm/integrations/SlackAlerting/batching_handler.py

@@ -1,14 +1,18 @@
 """
 Handles Batching + sending Httpx Post requests to slack
 
-Slack alerts are sent every 10s or when events are greater than X events
+Slack alerts are sent every DEFAULT_FLUSH_INTERVAL_SECONDS or when events are greater than X events
 
 see custom_batch_logger.py for more details / defaults
 """
 
+from collections import Counter
+from collections.abc import Sequence
+from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Final
 
 from litellm._logging import verbose_proxy_logger
+from litellm.types.integrations.slack_alerting import AlertQueueItem, AlertType
 
 from .ms_teams import MS_TEAMS_ALERTING_DESTINATION, build_ms_teams_payload
 
@@ -20,26 +24,20 @@ else:
     SlackAlertingType = Any
 
 
-def squash_payloads(queue):
-    squashed: Final = {}
-    if len(queue) == 0:
-        return squashed
-    if len(queue) == 1:
-        return {"key": {"item": queue[0], "count": 1}}
+@dataclass(frozen=True, slots=True)
+class SquashedAlert:
+    item: AlertQueueItem
+    count: int
 
-    for item in queue:
-        url = item["url"]
-        alert_type = item["alert_type"]
-        _key = (url, alert_type)
 
-        if _key in squashed:
-            squashed[_key]["count"] += 1
-            # Merge the payloads
+def _squash_key(item: AlertQueueItem) -> tuple[str, AlertType | str, str]:
+    return (item["url"], item["alert_type"], item["payload"]["text"])
 
-        else:
-            squashed[_key] = {"item": item, "count": 1}
 
-    return squashed
+def squash_payloads(queue: Sequence[AlertQueueItem]) -> tuple[SquashedAlert, ...]:
+    counts: Final = Counter(_squash_key(item) for item in queue)
+    first_item_by_key: Final = {_squash_key(item): item for item in reversed(queue)}
+    return tuple(SquashedAlert(item=first_item_by_key[key], count=count) for key, count in counts.items())
 
 
 def _print_alerting_payload_warning(payload: dict, slackAlertingInstance: SlackAlertingType):
@@ -53,17 +51,15 @@ def _print_alerting_payload_warning(payload: dict, slackAlertingInstance: SlackA
         verbose_proxy_logger.warning(payload)
 
 
-async def send_to_webhook(slackAlertingInstance: SlackAlertingType, item, count):
+async def send_to_webhook(slackAlertingInstance: SlackAlertingType, item: AlertQueueItem, count: int) -> None:
     """
     Send a single slack alert to the webhook
     """
     import json
 
-    payload: Final = item.get("payload", {})
+    text: Final = item["payload"]["text"]
+    payload: Final = {"text": text if count == 1 else f"[Num Alerts: {count}]\n\n{text}"}
     try:
-        if count > 1:
-            payload["text"] = f"[Num Alerts: {count}]\n\n{payload['text']}"
-
         request_body: Final = (
             build_ms_teams_payload(payload["text"]) if item.get("format") == MS_TEAMS_ALERTING_DESTINATION else payload
         )

litellm/integrations/SlackAlerting/slack_alerting.py

@@ -33,6 +33,7 @@ from litellm.litellm_core_utils.exception_mapping_utils import (
     _add_key_name_and_team_to_alert,
 )
 from litellm.llms.custom_httpx.http_handler import (
+    AsyncHTTPHandler,
     get_async_httpx_client,
     httpxSpecialProvider,
 )
@@ -99,6 +100,7 @@ class SlackAlerting(CustomBatchLogger):
         alerting_args={},
         default_webhook_url: str | None = None,
         alert_type_config: dict[str, dict] | None = None,
+        async_http_handler: AsyncHTTPHandler | None = None,
         **kwargs,
     ):
         if alerting_threshold is None:
@@ -107,7 +109,9 @@ class SlackAlerting(CustomBatchLogger):
         self.alerting = alerting
         self.alert_types = alert_types
         self.internal_usage_cache = internal_usage_cache or DualCache()
-        self.async_http_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
+        self.async_http_handler = async_http_handler or get_async_httpx_client(
+            llm_provider=httpxSpecialProvider.LoggingCallback
+        )
         self.alert_to_webhook_url = process_slack_alerting_variables(alert_to_webhook_url=alert_to_webhook_url)
         self.is_running = False
         self.alerting_args = SlackAlertingArgs(**alerting_args)
@@ -1583,12 +1587,12 @@ Model Info:
         if not self.log_queue:
             return
 
-        squashed_queue: Final = squash_payloads(self.log_queue)
-        tasks: Final = [
-            send_to_webhook(slackAlertingInstance=self, item=item["item"], count=item["count"])
-            for item in squashed_queue.values()
-        ]
-        await asyncio.gather(*tasks)
+        await asyncio.gather(
+            *(
+                send_to_webhook(slackAlertingInstance=self, item=squashed.item, count=squashed.count)
+                for squashed in squash_payloads(self.log_queue)
+            )
+        )
         self.log_queue.clear()
 
     async def _flush_digest_buckets(self):

litellm/integrations/argilla.py

@@ -7,7 +7,8 @@ import json
 import os
 import random
 import types
-from typing import Any, Final
+from collections.abc import Mapping
+from typing import Final
 
 import httpx
 from pydantic import BaseModel
@@ -69,7 +70,7 @@ class ArgillaLogger(CustomBatchLogger):
         self.flush_lock = asyncio.Lock()
         super().__init__(**kwargs, flush_lock=self.flush_lock)
 
-    def validate_argilla_transformation_object(self, argilla_transformation_object: dict[str, Any]):
+    def validate_argilla_transformation_object(self, argilla_transformation_object: Mapping[str, object]):
         if not isinstance(argilla_transformation_object, dict):
             raise Exception("'argilla_transformation_object' must be a dictionary, to log your payload to Argilla.")
 
@@ -115,7 +116,7 @@ class ArgillaLogger(CustomBatchLogger):
             ARGILLA_DATASET_NAME=_credentials_dataset_name,
         )
 
-    def get_chat_messages(self, payload: StandardLoggingPayload) -> list[dict[str, Any]]:
+    def get_chat_messages(self, payload: StandardLoggingPayload) -> list[dict[str, object]]:
         payload_messages: Final = payload.get("messages", None)
 
         if payload_messages is None:

litellm/integrations/braintrust_logging.py

@@ -139,13 +139,13 @@ class BraintrustLogger(CustomLogger):
             ):
                 output = None
             elif response_obj is not None and isinstance(response_obj, litellm.ModelResponse):
-                output = response_obj["choices"][0]["message"].json()
+                output = response_obj.choices[0].message.json()
                 choices = response_obj["choices"]
             elif response_obj is not None and isinstance(response_obj, litellm.TextCompletionResponse):
                 output = response_obj.choices[0].text
                 choices = response_obj.choices
             elif response_obj is not None and isinstance(response_obj, litellm.ImageResponse):
-                output = response_obj["data"]
+                output = response_obj.data
 
             litellm_params: Final = kwargs.get("litellm_params", {}) or {}
             dynamic_metadata: Final = litellm_params.get("metadata", {}) or {}
@@ -264,13 +264,13 @@ class BraintrustLogger(CustomLogger):
             ):
                 output = None
             elif response_obj is not None and isinstance(response_obj, litellm.ModelResponse):
-                output = response_obj["choices"][0]["message"].json()
+                output = response_obj.choices[0].message.json()
                 choices = response_obj["choices"]
             elif response_obj is not None and isinstance(response_obj, litellm.TextCompletionResponse):
                 output = response_obj.choices[0].text
                 choices = response_obj.choices
             elif response_obj is not None and isinstance(response_obj, litellm.ImageResponse):
-                output = response_obj["data"]
+                output = response_obj.data
 
             litellm_params: Final = kwargs.get("litellm_params", {})
             dynamic_metadata: Final = litellm_params.get("metadata", {}) or {}

litellm/integrations/custom_guardrail.py

@@ -150,7 +150,7 @@ class CustomGuardrail(CustomLogger):
 
     def __init_subclass__(cls, **kwargs: object) -> None:  # kwargs-ok: forwarded to cooperative __init_subclass__ hooks
         super().__init_subclass__(**kwargs)
-        own_apply_guardrail: Final = cls.__dict__.get("apply_guardrail")
+        own_apply_guardrail: Final[object] = cls.__dict__.get("apply_guardrail")
         if own_apply_guardrail is None or LOGS_GUARDRAIL_INFORMATION_MARKER in vars(own_apply_guardrail):
             return
         cls.apply_guardrail = log_guardrail_information(own_apply_guardrail)

litellm/integrations/datadog/datadog_llm_obs.py

@@ -54,7 +54,7 @@ from litellm.types.utils import (
     StandardLoggingPayloadErrorInformation,
 )
 
-_EMPTY_MAPPING: Final[Mapping[str, Any]] = MappingProxyType({})
+_EMPTY_MAPPING: Final[Mapping[str, object]] = MappingProxyType({})
 _EMPTY_MESSAGE: Final[Message] = {"role": "", "content": ""}
 _MAX_PARSED_TOOL_ARGUMENT_CHARS: Final = 256 * 1024
 _SAFE_REDACTED_MESSAGE_ROLES: Final = frozenset(
@@ -154,7 +154,7 @@ def _guardrail_information_without_prompt_carriers(
     return tuple(_guardrail_entry_without_prompt_carriers(entry) for entry in _guardrail_entries(guardrail_information))
 
 
-def _metadata_without_prompt_carriers(standard_logging_metadata: Mapping[str, Any]) -> Mapping[str, Any]:
+def _metadata_without_prompt_carriers(standard_logging_metadata: Mapping[str, object]) -> Mapping[str, object]:
     """The metadata minus the records that quote prompts, tool arguments, tool results, or retrieved text."""
     return MappingProxyType(
         {
@@ -237,7 +237,7 @@ def _declared_cost_tags(span_tags: Sequence[str]) -> tuple[str, ...]:
     return tuple(dimension for dimension in _COST_DIMENSIONS if dimension in present)
 
 
-def _reasoning_output_tokens(usage_object: Mapping[str, Any] | None) -> float:
+def _reasoning_output_tokens(usage_object: Mapping[str, object] | None) -> float:
     """The provider's reasoning-token count, from either the chat or the responses spelling."""
     if usage_object is None:
         return 0.0
@@ -254,20 +254,24 @@ def _reasoning_output_tokens(usage_object: Mapping[str, Any] | None) -> float:
     )
 
 
-def _mapping_field(source: Mapping[str, Any], key: str) -> Mapping[str, Any]:
+def _mapping_field(source: Mapping[str, object], key: str) -> Mapping[str, object]:
     """The value at `key` when it is a mapping, else an empty one."""
     value: Final = source.get(key)
     return value if isinstance(value, dict) else _EMPTY_MAPPING
 
 
-def _content_blocks(message: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]:
+def _text_field(source: Mapping[str, object], key: str, default: str = "") -> str:
+    return _safe_identifier(source.get(key, default))
+
+
+def _content_blocks(message: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
     content: Final = message.get("content")
     if not isinstance(content, list):
         return ()
     return tuple(block for block in content if isinstance(block, dict))
 
 
-def _to_dd_arguments(raw_arguments: object) -> dict[str, Any] | str:
+def _to_dd_arguments(raw_arguments: object) -> dict[str, object] | str:
     """
     Arguments as the object LLM Obs types them as, or the raw string when they are not one.
 
@@ -282,7 +286,7 @@ def _to_dd_arguments(raw_arguments: object) -> dict[str, Any] | str:
     return parsed if isinstance(parsed, dict) else raw_arguments
 
 
-def _to_dd_tool_calls(message: Mapping[str, Any]) -> tuple[ToolCall, ...]:
+def _to_dd_tool_calls(message: Mapping[str, object]) -> tuple[ToolCall, ...]:
     """
     The tool calls a message carries, in LLM Obs' ToolCall schema, from either dialect.
 
@@ -293,10 +297,10 @@ def _to_dd_tool_calls(message: Mapping[str, Any]) -> tuple[ToolCall, ...]:
     raw_tool_calls: Final = message.get("tool_calls")
     openai_calls: Final = tuple(
         ToolCall(
-            name=function.get("name", ""),
+            name=_text_field(function, "name"),
             arguments=_to_dd_arguments(function.get("arguments", "")),
-            tool_id=tool_call.get("id", ""),
-            type=tool_call.get("type", "function"),
+            tool_id=_text_field(tool_call, "id"),
+            type=_text_field(tool_call, "type", "function"),
         )
         for tool_call in (raw_tool_calls if isinstance(raw_tool_calls, list) else ())
         if isinstance(tool_call, dict)
@@ -304,9 +308,9 @@ def _to_dd_tool_calls(message: Mapping[str, Any]) -> tuple[ToolCall, ...]:
     )
     anthropic_calls: Final = tuple(
         ToolCall(
-            name=block.get("name", ""),
+            name=_text_field(block, "name"),
             arguments=_to_dd_arguments(block.get("input") or {}),
-            tool_id=block.get("id", ""),
+            tool_id=_text_field(block, "id"),
             type="tool_use",
         )
         for block in _content_blocks(message)
@@ -315,7 +319,7 @@ def _to_dd_tool_calls(message: Mapping[str, Any]) -> tuple[ToolCall, ...]:
     return openai_calls + anthropic_calls
 
 
-def _to_dd_tool_results(message: Mapping[str, Any], tool_call_names: Mapping[str, str]) -> tuple[ToolResult, ...]:
+def _to_dd_tool_results(message: Mapping[str, object], tool_call_names: Mapping[str, str]) -> tuple[ToolResult, ...]:
     """
     The tool results a message carries, linked back to the call each answers.
 
@@ -400,14 +404,14 @@ def _to_dd_messages(messages: object) -> tuple[Message, ...]:
     return tuple(_to_dd_message(message, tool_call_names) for message in messages)
 
 
-def _to_dd_tool_definition(entry: Mapping[str, Any]) -> ToolDefinition | None:
+def _to_dd_tool_definition(entry: Mapping[str, object]) -> ToolDefinition | None:
     function: Final = entry.get("function")
-    declared: Final[Mapping[str, Any]] = function if isinstance(function, dict) else entry
-    name: Final = declared.get("name")
+    declared: Final[Mapping[str, object]] = function if isinstance(function, dict) else entry
+    name: Final = _text_field(declared, "name")
     if not name:
         return None
     schema: Final = declared.get("parameters") or declared.get("input_schema")
-    description: Final = declared.get("description", "")
+    description: Final = _text_field(declared, "description")
     if not isinstance(schema, dict):
         return ToolDefinition(name=name, description=description)
     return ToolDefinition(name=name, description=description, schema=schema)
@@ -683,7 +687,7 @@ class DataDogLLMObsLogger(CustomBatchLogger):
             if callable(current_span_fn):
                 current_span: Final = current_span_fn()
                 if current_span is not None:
-                    trace_id: Final = getattr(current_span, "trace_id", None)
+                    trace_id: Final[object] = getattr(current_span, "trace_id", None)
                     if trace_id is not None:
                         return str(trace_id)
         except Exception:
@@ -716,7 +720,7 @@ class DataDogLLMObsLogger(CustomBatchLogger):
     def redacts_messages_itself(self) -> bool:
         return True
 
-    def _payload_logging_is_off(self, kwargs: Mapping[str, Any]) -> bool:
+    def _payload_logging_is_off(self, kwargs: Mapping[str, object]) -> bool:
         return (
             bool(self.turn_off_message_logging)
             or self.message_logging is not True

litellm/integrations/dynamodb.py

@@ -3,12 +3,21 @@
 
 import os
 import traceback
-from typing import Any, Final
+from collections.abc import Mapping
+from typing import Final, Protocol
 
 import litellm
 from litellm._uuid import uuid
 
 
+class _DynamoTable(Protocol):
+    def put_item(self, *, Item: Mapping[str, object]) -> object: ...
+
+
+class _DynamoResource(Protocol):
+    def Table(self, name: str) -> _DynamoTable: ...
+
+
 class DyanmoDBLogger:
     # Class variables or attributes
 
@@ -16,7 +25,7 @@ class DyanmoDBLogger:
         # Instance variables
         import boto3
 
-        self.dynamodb: Any = boto3.resource("dynamodb", region_name=os.environ["AWS_REGION_NAME"])
+        self.dynamodb: Final[_DynamoResource] = boto3.resource("dynamodb", region_name=os.environ["AWS_REGION_NAME"])
         if litellm.dynamodb_table_name is None:
             raise ValueError(
                 "LiteLLM Error, trying to use DynamoDB but not table name passed. Create a table and set `litellm.dynamodb_table_name=<your-table>`"
@@ -41,7 +50,7 @@ class DyanmoDBLogger:
             id: Final = response_obj.get("id", str(uuid.uuid4()))
 
             # Build the initial payload
-            payload: Final = {
+            payload: Final[dict[str, object]] = {
                 "id": id,
                 "call_type": call_type,
                 "startTime": start_time,

litellm/integrations/focus/database.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 from datetime import datetime
-from typing import Any, Final
+from typing import Final
 
 import polars as pl
 
@@ -32,7 +32,7 @@ class FocusLiteLLMDatabase:
         client: Final = self._ensure_prisma_client()
 
         where_clauses: Final[list[str]] = []
-        query_params: Final[list[Any]] = []
+        query_params: Final[list[datetime | int]] = []
         placeholder_index = 1
         if start_time_utc:
             where_clauses.append(f"dus.updated_at >= ${placeholder_index}::timestamptz")
@@ -112,7 +112,7 @@ class FocusLiteLLMDatabase:
         except Exception as exc:
             raise RuntimeError(f"Error retrieving usage data: {exc}") from exc
 
-    async def get_table_info(self) -> dict[str, Any]:
+    async def get_table_info(self) -> dict[str, object]:
         """Return metadata about the spend table for diagnostics."""
         client: Final = self._ensure_prisma_client()