Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_python_version_ci

# Conflicts:
#	basedpyright-code-budget.json
#	tests/sdk_function_trace/profiler.py
#	tests/sdk_function_trace/test_profiler.py

.github/actions/setup-uv-with-retries/action.yml

@@ -1,11 +1,7 @@
 name: "Set up uv with retries"
 description: >-
-  Install uv via astral-sh/setup-uv, retrying on transient failures. Even with
-  an exact pinned version, the action resolves the artifact URL by fetching
-  https://raw.githubusercontent.com/astral-sh/versions/main/v1/uv.ndjson in a
-  single request with no retry, timeout, or fallback, so one connection-level
-  network error ("fetch failed") fails the whole job before any test runs.
-  Retrying the full step covers the manifest fetch and the binary download.
+  Install uv via astral-sh/setup-uv, retrying the full setup step so manifest
+  resolution and binary downloads get fresh attempts after transient failures.
 
 inputs:
   version:
@@ -18,7 +14,7 @@ runs:
     - name: Set up uv (attempt 1)
       id: attempt-1
       continue-on-error: true
-      uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
       with:
         version: ${{ inputs.version }}
 
@@ -31,7 +27,7 @@ runs:
       id: attempt-2
       if: steps.attempt-1.outcome == 'failure'
       continue-on-error: true
-      uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
       with:
         version: ${{ inputs.version }}
 
@@ -42,6 +38,6 @@ runs:
 
     - name: Set up uv (attempt 3)
       if: steps.attempt-2.outcome == 'failure'
-      uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
       with:
         version: ${{ inputs.version }}

.github/ci-coverage-allowlist.yml

@@ -4,6 +4,11 @@ description: >-
   by a job nor listed here, so every entry below is a decision on the record.
 
 test_paths:
+  - reason: >-
+      The Rust/Python parity harness is run manually through its local CLI. Recorded replay,
+      fixture generation, and harness checks are intentionally outside pull request CI
+    paths:
+      - tests/rust-python-harness
   - reason: >-
       What is left of the caching suite in tests/local_testing that runs nowhere. Every job that
       globs that directory either deselects it (local_testing_part1 and part2 carry `-k "... and
@@ -74,6 +79,11 @@ test_paths:
       - tests/load_tests/test_otel_load_test.py
       - tests/load_tests/test_vertex_embeddings_load_test.py
       - tests/load_tests/test_vertex_load_tests.py
+  - reason: >-
+      Env-gated saturation benchmark requires a live proxy and provider credentials, so it is run
+      locally rather than in pull-request jobs
+    paths:
+      - tests/load_tests/test_granian_admission_saturation.py
   - reason: >-
       A local-only agent rig: test_a2a_completion_bridge.py needs a LangGraph server on
       localhost:2024 and test_a2a.py drives a live A2A endpoint, so neither can run in a

.github/workflows/test-litellm-ui-build.yml

@@ -19,9 +19,6 @@ jobs:
   build-ui:
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    defaults:
-      run:
-        working-directory: ui/litellm-dashboard
 
     steps:
       - name: Checkout repository
@@ -35,18 +32,11 @@ jobs:
         with:
           category: ui
 
-      - name: Setup Node.js
+      # Built through the image stage rather than the checkout, because the
+      # stage copies ui/litellm-dashboard/ alone: an import reaching above the
+      # dashboard root resolves in a checkout and fails in every image we ship.
+      # Dockerfile, docker/Dockerfile.non_root and ui/Dockerfile share this
+      # stage verbatim, so building one covers all three.
+      - name: Build the dashboard as the shipped images build it
         if: steps.changes.outputs.decision != 'skip'
-        uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
-        with:
-          node-version-file: ui/litellm-dashboard/.nvmrc
-          cache: "npm"
-          cache-dependency-path: ui/litellm-dashboard/package-lock.json
-
-      - name: Install dependencies
-        if: steps.changes.outputs.decision != 'skip'
-        run: npm ci
-
-      - name: Build
-        if: steps.changes.outputs.decision != 'skip'
-        run: npm run build
+        run: docker build --target ui-builder -f Dockerfile .

.github/workflows/test-rust.yml

@@ -74,12 +74,19 @@ jobs:
       - name: Run Clippy with Bedrock auth
         run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings
 
+      - name: Run Clippy with all gateway features
+        run: cargo clippy -p litellm-ai-gateway --all-targets --all-features --locked -- -D warnings
+
       - name: Run Rust tests
         run: cargo test --workspace --locked
 
       - name: Run core tests with Bedrock auth
         run: cargo test -p litellm-core --features bedrock-auth --locked
 
+      # Not --all-features: python-config links libpython, which this job does not install.
+      - name: Run gateway tests with the server feature
+        run: cargo test -p litellm-ai-gateway --features server --locked
+
   release-wheel:
     name: release wheel
     runs-on: ubuntu-latest

.github/workflows/test-unit.yml

@@ -96,7 +96,6 @@ jobs:
           - shard: misc
             artifact-name: misc
             test-path: >-
-              tests/sdk_function_trace
               tests/test_litellm/batches
               tests/test_litellm/secret_managers
               tests/test_litellm/a2a_protocol
@@ -151,6 +150,7 @@ jobs:
               tests/test_litellm/proxy/google_endpoints
               tests/test_litellm/proxy/openai_files_endpoint
               tests/test_litellm/proxy/batches_endpoints
+              tests/test_litellm/proxy/container_endpoints
               tests/test_litellm/proxy/fine_tuning_endpoints
               tests/test_litellm/proxy/vector_store_files_endpoints
               tests/test_litellm/proxy/video_endpoints

basedpyright-code-budget.json

@@ -3,7 +3,7 @@
     "limit": 14074
   },
   "reportArgumentType": {
-    "limit": 2215
+    "limit": 2206
   },
   "reportAssignmentType": {
     "limit": 319
@@ -42,7 +42,7 @@
     "limit": 12
   },
   "reportIndexIssue": {
-    "limit": 25
+    "limit": 24
   },
   "reportInvalidTypeForm": {
     "limit": 32
@@ -57,7 +57,7 @@
     "limit": 5601
   },
   "reportMissingTypeArgument": {
-    "limit": 15288
+    "limit": 15285
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -99,19 +99,19 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44362
+    "limit": 44360
   },
   "reportUnknownLambdaType": {
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 38324
+    "limit": 38311
   },
   "reportUnknownParameterType": {
-    "limit": 19623
+    "limit": 19624
   },
   "reportUnknownVariableType": {
-    "limit": 29860
+    "limit": 29847
   },
   "reportUnnecessaryCast": {
     "limit": 111

docker/build_from_pip/Dockerfile.build_from_pip

@@ -55,7 +55,10 @@ RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/
 
 ENV PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
+COPY docker/prod_entrypoint.sh /app/docker/prod_entrypoint.sh
+RUN sed -i 's/\r$//' /app/docker/prod_entrypoint.sh && chmod +x /app/docker/prod_entrypoint.sh
+
 EXPOSE 4000/tcp
 
-ENTRYPOINT ["litellm"]
+ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]
 CMD ["--port", "4000"]

docker/component_entrypoint.sh

@@ -1,8 +1,10 @@
 #!/bin/sh
 
-if [ "$USE_DDTRACE" = "true" ]; then
-    export DD_TRACE_OPENAI_ENABLED="False"
-    exec ddtrace-run "$@"
-fi
+case "$USE_DDTRACE" in
+    [Tt][Rr][Uu][Ee])
+        export DD_TRACE_OPENAI_ENABLED="False"
+        exec ddtrace-run "$@"
+        ;;
+esac
 
 exec "$@"

docker/prod_entrypoint.sh

@@ -1,8 +1,10 @@
 #!/bin/sh
 
-if [ "$USE_DDTRACE" = "true" ]; then
-    export DD_TRACE_OPENAI_ENABLED="False"
-    exec ddtrace-run litellm "$@"
-else
-    exec litellm "$@"
-fi
+case "$USE_DDTRACE" in
+    [Tt][Rr][Uu][Ee])
+        export DD_TRACE_OPENAI_ENABLED="False"
+        exec ddtrace-run litellm "$@"
+        ;;
+esac
+
+exec litellm "$@"

enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py

@@ -11,17 +11,35 @@ import sys
 sys.path.insert(
     0, os.path.abspath("../..")
 )  # Adds the parent directory to the system path
+import functools
 import tempfile
-from typing import Optional
+from contextvars import ContextVar
+from typing import TYPE_CHECKING, ClassVar, Literal, Optional
 
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.caching import DualCache
-from litellm.integrations.custom_guardrail import CustomGuardrail
+from litellm.integrations.custom_guardrail import (
+    CustomGuardrail,
+    log_guardrail_information,
+)
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.guardrails._content_utils import walk_user_text
+from litellm.types.guardrails import GuardrailEventHooks
+from litellm.types.utils import GenericGuardrailAPIInputs
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 
 GUARDRAIL_NAME = "hide_secrets"
 
+GUARDRAIL_PROVIDER = "hide-secrets"
+
+# Per-invocation tally of redacted secrets by detect-secrets plugin type; None
+# means the guardrail did not run, so _process_response records nothing.
+_masked_entity_count: ContextVar[Optional[dict]] = ContextVar(
+    "hide_secrets_masked_entity_count", default=None
+)
+
 _custom_plugins_path = "file://" + os.path.join(
     os.path.dirname(os.path.abspath(__file__)), "secrets_plugins"
 )
@@ -422,6 +440,10 @@ _default_detect_secrets_config = {
 
 
 class _ENTERPRISE_SecretDetection(CustomGuardrail):
+    # Keeps proxied traffic on async_pre_call_hook (the unified apply_guardrail
+    # path skips should_run_check and never sees data["prompt"]).
+    use_native_lifecycle_hooks: ClassVar[bool] = True
+
     def __init__(self, detect_secrets_config: Optional[dict] = None, **kwargs):
         self.user_defined_detect_secrets_config = detect_secrets_config
         super().__init__(**kwargs)
@@ -455,6 +477,26 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
 
         return detected_secrets
 
+    def redact_text(self, text: str, source: str = "message") -> str:
+        """Replace every detected secret in ``text`` with ``[REDACTED]`` and
+        tally the detected types into the per-invocation masked-entity count."""
+        detected_secrets = self.scan_message_for_secrets(text)
+        if not detected_secrets:
+            return text
+        counts = _masked_entity_count.get()
+        if counts is not None:
+            for secret in detected_secrets:
+                counts[secret["type"]] = counts.get(secret["type"], 0) + 1
+        secret_types = [secret["type"] for secret in detected_secrets]
+        verbose_proxy_logger.warning(
+            f"Detected and redacted secrets in {source}: {secret_types}"
+        )
+        return functools.reduce(
+            lambda redacted, secret: redacted.replace(secret["value"], "[REDACTED]"),
+            detected_secrets,
+            text,
+        )
+
     async def should_run_check(self, user_api_key_dict: UserAPIKeyAuth) -> bool:
         if user_api_key_dict.permissions is not None:
             if GUARDRAIL_NAME in user_api_key_dict.permissions:
@@ -463,7 +505,45 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
 
         return True
 
+    @log_guardrail_information
+    async def apply_guardrail(
+        self,
+        inputs: GenericGuardrailAPIInputs,
+        request_data: dict,
+        input_type: Literal["request", "response"],
+        logging_obj: Optional["LiteLLMLoggingObj"] = None,
+    ) -> GenericGuardrailAPIInputs:
+        """Unified-interface entrypoint, used by /guardrails/apply_guardrail
+        (the UI test playground). Proxied traffic keeps using
+        ``async_pre_call_hook``, see ``use_native_lifecycle_hooks``."""
+        texts = inputs.get("texts")
+        if not texts or not any(texts):
+            return inputs
+        _masked_entity_count.set({})
+        return {**inputs, "texts": [self.redact_text(text) for text in texts]}
+
+    def _redact_prompt(self, data: dict) -> int:
+        """Redact ``data["prompt"]`` (the text-completion shape, which
+        ``walk_user_text`` does not cover) and return how many non-empty
+        strings were inspected."""
+        prompt = data.get("prompt")
+        if isinstance(prompt, str):
+            if not prompt:
+                return 0
+            data["prompt"] = self.redact_text(prompt, source="prompt")
+            return 1
+        if isinstance(prompt, list):
+            data["prompt"] = [  # mutable-ok: data["prompt"] is a list on the wire
+                self.redact_text(item, source="prompt")
+                if isinstance(item, str) and item
+                else item
+                for item in prompt
+            ]
+            return sum(1 for item in prompt if isinstance(item, str) and item)
+        return 0
+
     #### CALL HOOKS - proxy only ####
+    @log_guardrail_information
     async def async_pre_call_hook(
         self,
         user_api_key_dict: UserAPIKeyAuth,
@@ -471,53 +551,84 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
         data: dict,
         call_type: str,  # "completion", "embeddings", "image_generation", "moderation"
     ):
+        _masked_entity_count.set(None)
         if await self.should_run_check(user_api_key_dict) is False:
             return
 
+        _masked_entity_count.set({})
+
         # Covers multimodal list content + Responses-API input.
-        def _redact_message_text(text: str) -> str:
-            detected_secrets = self.scan_message_for_secrets(text)
-            for secret in detected_secrets:
-                text = text.replace(secret["value"], "[REDACTED]")
-            if detected_secrets:
-                secret_types = [secret["type"] for secret in detected_secrets]
-                verbose_proxy_logger.warning(
-                    f"Detected and redacted secrets in message: {secret_types}"
-                )
-            return text
+        inspected = walk_user_text(data, self.redact_text) + self._redact_prompt(data)
 
-        walk_user_text(data, _redact_message_text)
+        if inspected == 0:
+            # Image-only, empty-text, and unsupported payloads inspected
+            # nothing, so recording "allow" would count a run that never
+            # looked at any content.
+            _masked_entity_count.set(None)
 
-        if "prompt" in data:
-            if isinstance(data["prompt"], str):
-                detected_secrets = self.scan_message_for_secrets(data["prompt"])
-                for secret in detected_secrets:
-                    data["prompt"] = data["prompt"].replace(
-                        secret["value"], "[REDACTED]"
-                    )
-                if len(detected_secrets) > 0:
-                    secret_types = [secret["type"] for secret in detected_secrets]
-                    verbose_proxy_logger.warning(
-                        f"Detected and redacted secrets in prompt: {secret_types}"
-                    )
-            elif isinstance(data["prompt"], list):
-                # Index back into the list — assigning to ``item`` would only
-                # rebind the loop variable and leave ``data["prompt"]``
-                # carrying the unredacted secret.
-                for idx, item in enumerate(data["prompt"]):
-                    if isinstance(item, str):
-                        detected_secrets = self.scan_message_for_secrets(item)
-                        for secret in detected_secrets:
-                            item = item.replace(secret["value"], "[REDACTED]")
-                        data["prompt"][idx] = item
-                        if len(detected_secrets) > 0:
-                            secret_types = [
-                                secret["type"] for secret in detected_secrets
-                            ]
-                            verbose_proxy_logger.warning(
-                                f"Detected and redacted secrets in prompt: {secret_types}"
-                            )
-
-        # ``data["input"]`` (Responses API and embeddings/moderation) is
-        # already covered by ``walk_user_text`` above.
         return
+
+    def _process_response(
+        self,
+        response: Optional[dict],
+        request_data: dict,
+        start_time: Optional[float] = None,
+        end_time: Optional[float] = None,
+        duration: Optional[float] = None,
+        event_type: Optional[GuardrailEventHooks] = None,
+        original_inputs: Optional[dict] = None,
+    ):
+        """Record allow/mask plus the masked-entity tally for a completed run.
+
+        Records nothing when the guardrail inspected nothing (opted-out key,
+        empty inputs) or when the instance has no guardrail_name (legacy
+        ``litellm_settings.callbacks`` deployments, which predate guardrail
+        telemetry and stay without it).
+        """
+        counts = _masked_entity_count.get()
+        _masked_entity_count.set(None)
+        if counts is None or self.guardrail_name is None:
+            return response
+        self.add_standard_logging_guardrail_information_to_request_data(
+            guardrail_json_response="mask" if counts else "allow",
+            request_data=request_data,
+            guardrail_status="success",
+            duration=duration,
+            start_time=start_time,
+            end_time=end_time,
+            event_type=event_type,
+            guardrail_provider=GUARDRAIL_PROVIDER,
+            masked_entity_count=counts,
+        )
+        return response
+
+    def _process_error(
+        self,
+        e: Exception,
+        request_data: dict,
+        start_time: Optional[float] = None,
+        end_time: Optional[float] = None,
+        duration: Optional[float] = None,
+        event_type: Optional[GuardrailEventHooks] = None,
+    ):
+        """Label the failed run with this guardrail's provider so error rows
+        group with the successful ones in the monitor. Nameless legacy
+        instances record nothing, matching ``_process_response``."""
+        _masked_entity_count.set(None)
+        if self.guardrail_name is None:
+            raise e
+        self.add_standard_logging_guardrail_information_to_request_data(
+            guardrail_json_response=e,
+            request_data=request_data,
+            guardrail_status=(
+                "guardrail_intervened"
+                if self._is_guardrail_intervention(e)
+                else "guardrail_failed_to_respond"
+            ),
+            duration=duration,
+            start_time=start_time,
+            end_time=end_time,
+            event_type=event_type,
+            guardrail_provider=GUARDRAIL_PROVIDER,
+        )
+        raise e

enterprise/litellm_enterprise/proxy/audit_logging_endpoints.py

@@ -48,6 +48,18 @@ def _build_json_field_or_condition(json_key: str, value: str) -> dict[str, objec
     }
 
 
+def _build_search_condition(search: str) -> dict[str, object]:
+    """Match a row whose id, changed_by, object_id, or changed_by_api_key equals the search value."""
+    return {
+        "OR": (
+            {"id": search},
+            {"changed_by": search},
+            {"object_id": search},
+            {"changed_by_api_key": search},
+        )
+    }
+
+
 @router.get(
     "/audit",
     tags=["Audit Logging"],
@@ -83,6 +95,10 @@ async def get_audit_logs(
         None,
         description="Filter by token (key hash) present in before_value or updated_values JSON (PostgreSQL only)",
     ),
+    search: str | None = Query(
+        None,
+        description="Match a row whose id, object_id, changed_by, or changed_by_api_key equals this value",
+    ),
     # Sorting parameters
     sort_by: str | None = Query(
         None,
@@ -118,6 +134,11 @@ async def get_audit_logs(
         *([_build_json_field_or_condition("token", object_key_hash)] if object_key_hash else []),
     ]
 
+    and_conditions: Final[tuple[dict[str, object], ...]] = (
+        *json_field_conditions,
+        *((_build_search_condition(search),) if search else ()),
+    )
+
     # Build filter conditions
     where_conditions: Final[dict[str, object]] = {
         **({"changed_by": changed_by} if changed_by else {}),
@@ -126,14 +147,14 @@ async def get_audit_logs(
         **({"table_name": table_name} if table_name else {}),
         **({"object_id": object_id} if object_id else {}),
         **({"updated_at": date_filter} if start_date or end_date else {}),
-        **({"AND": json_field_conditions} if json_field_conditions else {}),
+        **({"AND": and_conditions} if and_conditions else {}),
     }
 
     order_by: Final[dict[str, str]] = (
         {sort_by: sort_order} if sort_by and isinstance(sort_by, str) else {"updated_at": sort_order}
     )
 
-    audit_log_table: Final[TableActions["prisma_models.LiteLLM_AuditLog"]] = AuditLogRepository(prisma_client).table
+    audit_log_table: Final[TableActions[prisma_models.LiteLLM_AuditLog]] = AuditLogRepository(prisma_client).table
 
     # Get paginated results
     audit_logs: Final = await audit_log_table.find_many(
@@ -195,7 +216,7 @@ async def get_audit_log_by_id(
             detail={"message": CommonProxyErrors.db_not_connected_error.value},
         )
 
-    audit_log_table: Final[TableActions["prisma_models.LiteLLM_AuditLog"]] = AuditLogRepository(prisma_client).table
+    audit_log_table: Final[TableActions[prisma_models.LiteLLM_AuditLog]] = AuditLogRepository(prisma_client).table
 
     # Get the audit log by ID
     audit_log: Final = await audit_log_table.find_unique(where={"id": id})

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -50,10 +50,12 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
     MAX_FILE_LIST_LIMIT,
     _is_base64_encoded_unified_file_id,
     apply_unified_file_ids,
+    decode_model_from_file_id,
     ensure_batch_response_managed_file_ids,
     get_batch_id_from_unified_batch_id,
     get_content_type_from_file_object,
     get_model_id_from_unified_batch_id,
+    get_original_file_id,
     map_raw_file_ids_to_unified,
     normalize_mime_type_for_provider,
     resolve_managed_output_file_model_name,
@@ -427,6 +429,103 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             detail=f"Object not found: {unified_object_id}",
         )
 
+    async def enforce_batch_object_access(
+        self, object_id: str, user_api_key_dict: UserAPIKeyAuth
+    ) -> None:
+        """Deny access to a provider-format batch id owned by another caller.
+
+        Ids with no ownership row (batches created before ownership tracking,
+        or directly on the provider account) stay accessible so pass-through
+        reads keep working.
+        """
+        if self.prisma_client is None:
+            return
+        managed_object = (
+            await self.prisma_client.db.litellm_managedobjecttable.find_first(
+                where={"OR": [{"unified_object_id": object_id}, {"model_object_id": object_id}]}
+            )
+        )
+        if managed_object is None:
+            return
+        if not can_access_resource(
+            user_api_key_dict=user_api_key_dict,
+            created_by=managed_object.created_by,
+            resource_team_id=managed_object.team_id,
+        ):
+            raise HTTPException(
+                status_code=403,
+                detail=f"User {user_api_key_dict.user_id} does not have access to the object {object_id}",
+            )
+
+    async def enforce_provider_file_access(
+        self, file_id: str, user_api_key_dict: UserAPIKeyAuth
+    ) -> None:
+        """Deny access to a provider-format file id owned by another caller.
+
+        Ownership rows for provider-format ids are written when a managed
+        batch's output/error files are first synced; ids with no row stay
+        accessible so pass-through reads keep working.
+        """
+        if self.prisma_client is None:
+            return
+        managed_file = (
+            await self.prisma_client.db.litellm_managedfiletable.find_first(
+                where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]}
+            )
+        )
+        if managed_file is None:
+            return
+        if not can_access_resource(
+            user_api_key_dict=user_api_key_dict,
+            created_by=managed_file.created_by,
+            resource_team_id=managed_file.team_id,
+        ):
+            raise HTTPException(
+                status_code=403,
+                detail=f"User {user_api_key_dict.user_id} does not have access to the file {file_id}",
+            )
+
+    async def store_batch_output_file_ownership(
+        self, response: LiteLLMBatch, litellm_parent_otel_span: Optional[Span]
+    ) -> None:
+        """Record ownership rows for a batch's provider-format output/error
+        file ids, inherited from the owning batch row (never the caller), so
+        file reads can be isolation-checked."""
+        provider_file_ids = tuple(
+            file_id
+            for file_id in (
+                getattr(response, "output_file_id", None),
+                getattr(response, "error_file_id", None),
+            )
+            if file_id and not _is_base64_encoded_unified_file_id(file_id)
+        )
+        if not provider_file_ids:
+            return
+        if self.prisma_client is None:
+            return
+        batch_row = (
+            await self.prisma_client.db.litellm_managedobjecttable.find_first(
+                where={"unified_object_id": response.id}
+            )
+        )
+        if batch_row is None or (
+            batch_row.created_by is None and batch_row.team_id is None
+        ):
+            return
+        owner_identity = UserAPIKeyAuth(
+            user_id=batch_row.created_by, team_id=batch_row.team_id
+        )
+        for file_id in provider_file_ids:
+            model_name = decode_model_from_file_id(file_id)
+            raw_file_id = get_original_file_id(file_id)
+            await self.store_unified_file_id(
+                file_id=file_id,
+                file_object=None,
+                litellm_parent_otel_span=litellm_parent_otel_span,
+                model_mappings={model_name: raw_file_id} if model_name else {},
+                user_api_key_dict=owner_identity,
+            )
+
     async def list_user_batches(
         self,
         user_api_key_dict: UserAPIKeyAuth,
@@ -613,6 +712,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     status_code=403,
                     detail=f"User {user_api_key_dict.user_id} does not have access to the file {retrieve_file_id}",
                 )
+        if retrieve_file_id:
+            await self.enforce_provider_file_access(
+                retrieve_file_id, user_api_key_dict
+            )
         return False
 
     async def check_file_ids_access(self, file_ids: List[str], user_api_key_dict: UserAPIKeyAuth) -> None:
@@ -765,6 +868,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     )
                 data["model"] = potential_model_id
                 data[accessor_key] = get_batch_id_from_unified_batch_id(potential_llm_object_id)
+            elif retrieve_object_id and accessor_key == "batch_id":
+                await self.enforce_batch_object_access(retrieve_object_id, user_api_key_dict)
         elif call_type == CallTypes.acreate_fine_tuning_job.value:
             input_file_id = cast(Optional[str], data.get("training_file"))
             if input_file_id:
@@ -1297,7 +1402,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 user_api_key_dict=user_api_key_dict,
                 request_tags=request_tags_from_metadata(request_metadata if isinstance(request_metadata, dict) else {}),
                 persist_attribution=is_batch_create,
+                create_if_missing=is_batch_create,
             )
+            if not is_batch_create:
+                await self.store_batch_output_file_ownership(
+                    response=response,
+                    litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
+                )
 
             # Only record batch creation metric on actual create (not retrieve/cancel).
             # unified_file_id in _hidden_params is only set by the create_batch endpoint.

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.63"
+version = "0.1.64"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.63"
+version = "0.1.64"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

helm/litellm/templates/_helpers.tpl

@@ -428,3 +428,16 @@ envFrom:
 {{- end }}
 {{- end }}
 {{- end -}}
+
+{{/*
+ingress-nginx's admission webhook rejects a dot in an Exact or Prefix path
+(strict-validate-path-type) and serves ImplementationSpecific as a plain
+prefix location, so a dotted path takes that type there.
+*/}}
+{{- define "litellm.ingress.pathType" -}}
+{{- if and (eq .controller "nginx") (contains "." .path) -}}
+ImplementationSpecific
+{{- else -}}
+{{- .pathType -}}
+{{- end -}}
+{{- end -}}

helm/litellm/templates/ingress.yaml

@@ -5,6 +5,10 @@
 {{- $gatewayPort := .Values.gateway.service.port -}}
 {{- $backendPort := .Values.backend.service.port -}}
 {{- $uiPort := .Values.ui.service.port -}}
+{{- $controller := .Values.ingress.controller | default "alb" -}}
+{{- if not (has $controller (list "alb" "nginx")) }}
+{{- fail (printf "ingress.controller: unknown controller %q, expected one of alb, nginx" $controller) }}
+{{- end }}
 {{/*
   Backends addressable from ingress.extraPaths, keyed by the `service` field.
 */}}
@@ -27,10 +31,11 @@
   /litellm-asset-prefix, so without /*.txt they fall to the backend catch-all
   → 404 → client-side navigation never settles and the login flow spins in an
   infinite redirect loop (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt
-  from the export; the rule only routes the request to it. Needs an ingress
-  controller whose ImplementationSpecific path is a wildcard pattern
-  (AWS ALB: `*` = 0+ chars); this chart targets the AWS Load Balancer
-  Controller.
+  from the export; the rule only routes the request to it. It needs an
+  ingress controller whose ImplementationSpecific path is a wildcard pattern
+  (AWS ALB: `*` = 0+ chars), so it is rendered for ingress.controller=alb
+  only: ingress-nginx serves ImplementationSpecific as a literal prefix
+  location, where /*.txt can never match.
 */}}
 {{- $uiPaths := list
     (dict "path" "/" "pathType" "Exact")
@@ -38,8 +43,10 @@
     (dict "path" "/litellm-asset-prefix" "pathType" "Prefix")
     (dict "path" "/_next" "pathType" "Prefix")
     (dict "path" "/ui" "pathType" "Prefix")
-    (dict "path" "/*.txt" "pathType" "ImplementationSpecific")
 -}}
+{{- if eq $controller "alb" }}
+{{- $uiPaths = append $uiPaths (dict "path" "/*.txt" "pathType" "ImplementationSpecific") }}
+{{- end }}
 {{/*
   Gateway data-plane prefixes — must mirror gateway/routes/allowlist.py.
   Versioned paths are listed explicitly to avoid routing management routes
@@ -83,12 +90,6 @@
   adding to it.
 */}}
 {{- $builtinPathKeys := list "/test|Exact" "/|Prefix" -}}
-{{- range $uiPaths }}
-{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" .path .pathType) }}
-{{- end }}
-{{- range $gatewayPrefixes }}
-{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Prefix" .) }}
-{{- end }}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -115,8 +116,10 @@ spec:
         paths:
           # --- UI (Next.js static export) ---
           {{- range $uiPaths }}
+          {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" .path "pathType" .pathType) }}
+          {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" .path $pathType) }}
           - path: {{ .path }}
-            pathType: {{ .pathType }}
+            pathType: {{ $pathType }}
             backend:
               service:
                 name: {{ $uiName }}
@@ -134,8 +137,10 @@ spec:
                 port:
                   number: {{ $gatewayPort }}
           {{- range $gatewayPrefixes }}
+          {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" . "pathType" "Prefix") }}
+          {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" . $pathType) }}
           - path: {{ . }}
-            pathType: Prefix
+            pathType: {{ $pathType }}
             backend:
               service:
                 name: {{ $gatewayName }}
@@ -147,10 +152,11 @@ spec:
             Rendered after every built-in path so an entry can never take
             precedence over a default, and before the backend catch-all.
             Position only decides the match on controllers that honour manifest
-            order: the AWS Load Balancer Controller this chart targets sorts
-            Exact paths first and Prefix paths longest-first, but keeps
+            order: the AWS Load Balancer Controller (ingress.controller=alb)
+            sorts Exact paths first and Prefix paths longest-first, but keeps
             ImplementationSpecific paths in manifest order, which is what the
-            /*.txt rule above already depends on.
+            /*.txt rule above already depends on. ingress-nginx ignores order
+            and serves the longest matching location.
           */}}
           {{- range $idx, $extra := .Values.ingress.extraPaths }}
           {{- if not (kindIs "map" $extra) }}
@@ -164,10 +170,11 @@ spec:
           {{- if not $target }}
           {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown service %q, expected one of backend, gateway, ui" $idx $extra.path $service) }}
           {{- end }}
-          {{- $pathType := $extra.pathType | default "Prefix" }}
-          {{- if not (has $pathType (list "Prefix" "Exact" "ImplementationSpecific")) }}
-          {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown pathType %q, expected one of Exact, ImplementationSpecific, Prefix" $idx $extra.path $pathType) }}
+          {{- $requestedPathType := $extra.pathType | default "Prefix" }}
+          {{- if not (has $requestedPathType (list "Prefix" "Exact" "ImplementationSpecific")) }}
+          {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown pathType %q, expected one of Exact, ImplementationSpecific, Prefix" $idx $extra.path $requestedPathType) }}
           {{- end }}
+          {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" $extra.path "pathType" $requestedPathType) }}
           {{- if eq $extra.path "/" }}
           {{- fail (printf "ingress.extraPaths[%d]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture" $idx) }}
           {{- end }}

helm/litellm/tests/ingress_controller_tests.yaml

@@ -0,0 +1,205 @@
+suite: test ingress.controller
+templates:
+  - ingress.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: keeps the AWS Load Balancer Controller path types by default
+    set:
+      ingress.enabled: true
+    asserts:
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /favicon.ico
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /eu.assemblyai
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /*.txt
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+
+  - it: renders no dotted Exact or Prefix path for ingress-nginx, whose admission webhook rejects them
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+    asserts:
+      - notMatchRegexRaw:
+          pattern: 'path: /\S*\.\S*\n\s+pathType: (Exact|Prefix)\n'
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /favicon.ico
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /eu.assemblyai
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+
+  - it: drops the /*.txt wildcard for ingress-nginx and keeps every other route as is
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+    asserts:
+      - notContains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /*.txt
+          any: true
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /ui
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /test
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - equal:
+          path: spec.rules[0].http.paths[-1]
+          value:
+            path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-backend
+                port:
+                  number: 4001
+
+  - it: rejects an extraPaths entry that repeats a built-in path at the pathType ingress-nginx renders it with
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+      ingress.extraPaths:
+        - path: /favicon.ico
+          service: ui
+          pathType: ImplementationSpecific
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /favicon.ico with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: rejects a controller it has no path types for
+    set:
+      ingress.enabled: true
+      ingress.controller: traefik
+    asserts:
+      - failedTemplate:
+          errorMessage: 'ingress.controller: unknown controller "traefik", expected one of alb, nginx'
+
+  - it: rejects an extraPaths entry that repeats a built-in path once ingress-nginx normalizes its pathType
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+      ingress.extraPaths:
+        - path: /favicon.ico
+          service: ui
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /favicon.ico with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: renders a dotted extraPaths entry as ImplementationSpecific for ingress-nginx
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+      ingress.extraPaths:
+        - path: /eu.assemblyai.custom
+          service: gateway
+        - path: /robots.txt
+          service: ui
+          pathType: Exact
+    asserts:
+      - notMatchRegexRaw:
+          pattern: 'path: "?/\S*\.\S*"?\n\s+pathType: (Exact|Prefix)\n'
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /eu.assemblyai.custom
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /robots.txt
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+
+  - it: keeps the requested pathType of a dotted extraPaths entry for the AWS Load Balancer Controller
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /eu.assemblyai.custom
+          service: gateway
+        - path: /robots.txt
+          service: ui
+          pathType: Exact
+    asserts:
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /eu.assemblyai.custom
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /robots.txt
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000

helm/litellm/values.yaml

@@ -10,6 +10,18 @@ imagePullSecrets: []
 ingress:
   enabled: false
   className: ""
+  # Which ingress controller serves this Ingress. Controllers disagree on the
+  # pathTypes they accept, so this picks the pathType of the dotted paths, the
+  # built-in ones and any dotted extraPaths entry alike:
+  #   alb    AWS Load Balancer Controller (default): Exact and Prefix paths plus
+  #          the /*.txt wildcard that routes the UI's RSC payloads.
+  #   nginx  ingress-nginx: its admission webhook rejects a dot in an Exact or
+  #          Prefix path (strict-validate-path-type, on by default from v1.12.0
+  #          until v1.12.6 / v1.13.2 allowed dots again), so /favicon.ico and
+  #          /eu.assemblyai render as ImplementationSpecific, which nginx serves
+  #          as a plain prefix location. /*.txt is dropped: nginx has no
+  #          wildcard pathType, so that rule could never match there.
+  controller: alb
   annotations: {}
   host: ""        # optional; if set, becomes the rule's host
   tls: []
@@ -26,7 +38,8 @@ ingress:
   #
   #   path      required; the HTTP path to route
   #   service   which component serves it: gateway (default), backend, or ui
-  #   pathType  Prefix (default), Exact, or ImplementationSpecific
+  #   pathType  Prefix (default), Exact, or ImplementationSpecific; a dotted
+  #             path renders as ImplementationSpecific when controller is nginx
   #
   # The target component only answers paths its own route allowlist keeps, so
   # a path here still has to be one that component serves.

litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py

@@ -18,10 +18,15 @@ recoverable one.
 constant: it grows with the number of pending migrations, so a fresh database
 that has to replay every migration this package ships overruns a per-command
 budget sized for the short bookkeeping commands, on a laptop as much as on a
-slow CI runner. The Python ``prisma`` wrapper spawns Node and the schema engine
-as separate children, so killing the wrapper on timeout leaves them running:
-the retry then contends with that orphan for Prisma's advisory lock and cannot
-finish any sooner. Migrate deploy therefore runs under its own budget.
+slow CI runner. Migrate deploy therefore runs under its own budget.
+
+The Python ``prisma`` wrapper spawns Node, which spawns the Rust schema
+engine, so killing only the wrapper on timeout leaves the engine running with
+no parent: it keeps mutating the database after the proxy has given up, holds
+Prisma's advisory lock so every retry and every later boot queues behind it,
+and dies mid-migration once its pipes close, leaving a half-applied ledger row.
+Every Prisma command therefore runs in a process group of its own, and a
+timeout kills the whole group.
 
 All three budgets are overridable so an operator can widen them without a
 release: ``LITELLM_PRISMA_BOOTSTRAP_TIMEOUT`` for the toolchain install,
@@ -35,10 +40,12 @@ the deploy override says otherwise.
 import math
 import os
 import shutil
+import signal
 import subprocess
+from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Optional
+from typing import IO, Optional, Union
 
 from litellm_proxy_extras._logging import logger
 
@@ -167,6 +174,49 @@ def heal_incomplete_nodeenv_cache() -> bool:
     return True
 
 
+def _kill_process_group(process: "subprocess.Popen[str]") -> None:
+    if os.name == "nt":
+        process.kill()
+        return
+    try:
+        os.killpg(process.pid, signal.SIGKILL)
+    except ProcessLookupError:
+        return
+
+
+def run_prisma(
+    argv: Sequence[str],
+    *,
+    timeout: float,
+    env: Mapping[str, str],
+    stdout: Union[IO[str], int, None] = subprocess.PIPE,
+    stderr: Optional[int] = subprocess.PIPE,
+) -> "subprocess.CompletedProcess[str]":
+    """Run one Prisma CLI command in its own process group, bounded by ``timeout``.
+
+    Raises ``subprocess.TimeoutExpired`` once the budget is spent, after killing
+    the command together with every process it spawned, and
+    ``subprocess.CalledProcessError`` on a non-zero exit. Output is captured as
+    text unless ``stdout``/``stderr`` say otherwise.
+    """
+    with subprocess.Popen(
+        argv,
+        env=env,
+        stdout=stdout,
+        stderr=stderr,
+        text=True,
+        start_new_session=True,
+    ) as process:
+        try:
+            out, err = process.communicate(timeout=timeout)
+        except BaseException:
+            _kill_process_group(process)
+            raise
+    if process.returncode:
+        raise subprocess.CalledProcessError(process.returncode, process.args, out, err)
+    return subprocess.CompletedProcess(process.args, process.returncode, out, err)
+
+
 def ensure_prisma_toolchain(
     prisma_command: str, prisma_env: dict[str, str]
 ) -> ToolchainBootstrap:
@@ -179,14 +229,7 @@ def ensure_prisma_toolchain(
     timeout = prisma_bootstrap_timeout()
     logger.info("Preparing the Prisma CLI toolchain (timeout %ss)", timeout)
     try:
-        subprocess.run(
-            [prisma_command, BOOTSTRAP_ARG],
-            timeout=timeout,
-            check=True,
-            capture_output=True,
-            text=True,
-            env=prisma_env,
-        )
+        run_prisma([prisma_command, BOOTSTRAP_ARG], timeout=timeout, env=prisma_env)
     except subprocess.TimeoutExpired:
         logger.warning(
             "Preparing the Prisma CLI toolchain timed out after %ss. Raise %s "

litellm-proxy-extras/litellm_proxy_extras/replica_identity.py

@@ -16,7 +16,7 @@ import tempfile
 from pathlib import Path
 
 from litellm_proxy_extras._logging import logger
-from litellm_proxy_extras.prisma_toolchain import prisma_command_timeout
+from litellm_proxy_extras.prisma_toolchain import prisma_command_timeout, run_prisma
 
 REPLICA_IDENTITY_FULL_ENV_VAR = "LITELLM_SET_REPLICA_IDENTITY_FULL"
 
@@ -66,7 +66,7 @@ def apply_replica_identity_full(
         with tempfile.TemporaryDirectory(prefix="litellm_replica_identity_") as tmp_dir:
             sql_path = Path(tmp_dir) / "replica_identity_full.sql"
             sql_path.write_text(REPLICA_IDENTITY_FULL_SQL)
-            subprocess.run(
+            run_prisma(
                 [
                     prisma_command,
                     "db",
@@ -77,9 +77,6 @@ def apply_replica_identity_full(
                     schema_path,
                 ],
                 timeout=prisma_command_timeout(),
-                check=True,
-                capture_output=True,
-                text=True,
                 env=prisma_env,
             )
     except subprocess.CalledProcessError as e:

litellm-proxy-extras/litellm_proxy_extras/utils.py

@@ -6,9 +6,11 @@ import shutil
 import subprocess
 import tempfile
 import time
+from dataclasses import dataclass, replace
 from pathlib import Path
 from typing import Optional
 
+from litellm_proxy_extras import prisma_toolchain
 from litellm_proxy_extras._logging import logger
 from litellm_proxy_extras.replica_identity import (
     REPLICA_IDENTITY_FULL_ENV_VAR,
@@ -45,6 +47,38 @@ _MIGRATION_TS_RE = re.compile(r"^(\d{14})_")
 
 _MIGRATION_DEADLOCK_MARKER = "deadlock detected"
 
+MAX_MIGRATE_DEPLOY_ATTEMPTS = 4
+
+
+@dataclass(frozen=True)
+class _MigrateAttemptBudget:
+    """Retries left, and the recoveries already run.
+
+    A recovery that lands something new costs nothing, so a database full of
+    objects `prisma db push` created works through them one per pass. Anything
+    that made no progress spends an attempt, so a stuck run still gives up.
+    """
+
+    attempts_left: int
+    recoveries: frozenset[str] = frozenset()
+
+    @property
+    def exhausted(self) -> bool:
+        return self.attempts_left <= 0
+
+    @property
+    def attempt_number(self) -> int:
+        return MAX_MIGRATE_DEPLOY_ATTEMPTS - self.attempts_left + 1
+
+    def spend(self) -> "_MigrateAttemptBudget":
+        return replace(self, attempts_left=self.attempts_left - 1)
+
+    def after_recovery(self, recovery: str) -> "_MigrateAttemptBudget":
+        if recovery in self.recoveries:
+            return self.spend()
+        return replace(self, recoveries=self.recoveries | {recovery})
+
+
 _SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE)
 _SPEND_LOGS_ARTIFACT_DROP_RE = re.compile(
     r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE
@@ -198,7 +232,7 @@ class ProxyExtrasDBManager:
             # 1. Generate migration SQL file by comparing empty state to current db state
             logger.info("Generating baseline migration...")
             migration_file = init_dir / "migration.sql"
-            subprocess.run(
+            prisma_toolchain.run_prisma(
                 [
                     _get_prisma_command(),
                     "migrate",
@@ -209,14 +243,13 @@ class ProxyExtrasDBManager:
                     "--script",
                 ],
                 stdout=open(migration_file, "w"),
-                check=True,
                 timeout=prisma_command_timeout(),
                 env=prisma_env,
             )
 
             # 3. Mark the migration as applied since it represents current state
             logger.info("Marking baseline migration as applied...")
-            subprocess.run(
+            prisma_toolchain.run_prisma(
                 [
                     _get_prisma_command(),
                     "migrate",
@@ -224,7 +257,6 @@ class ProxyExtrasDBManager:
                     "--applied",
                     "0_init",
                 ],
-                check=True,
                 timeout=prisma_command_timeout(),
                 env=prisma_env,
             )
@@ -253,7 +285,7 @@ class ProxyExtrasDBManager:
         """Mark a specific migration as rolled back"""
         # Set up environment for offline mode if configured
         prisma_env = _get_prisma_env()
-        subprocess.run(
+        prisma_toolchain.run_prisma(
             [
                 _get_prisma_command(),
                 "migrate",
@@ -262,8 +294,6 @@ class ProxyExtrasDBManager:
                 migration_name,
             ],
             timeout=prisma_command_timeout(),
-            check=True,
-            capture_output=True,
             env=prisma_env,
         )
 
@@ -315,11 +345,9 @@ class ProxyExtrasDBManager:
     def _resolve_specific_migration(migration_name: str):
         """Mark a specific migration as applied"""
         prisma_env = _get_prisma_env()
-        subprocess.run(
+        prisma_toolchain.run_prisma(
             [_get_prisma_command(), "migrate", "resolve", "--applied", migration_name],
             timeout=prisma_command_timeout(),
-            check=True,
-            capture_output=True,
             env=prisma_env,
         )
 
@@ -403,7 +431,7 @@ class ProxyExtrasDBManager:
         try:
             logger.info("Generating migration diff between DB and schema.prisma...")
             with open(diff_sql_path, "w") as f:
-                subprocess.run(
+                prisma_toolchain.run_prisma(
                     [
                         _get_prisma_command(),
                         "migrate",
@@ -414,7 +442,6 @@ class ProxyExtrasDBManager:
                         schema_path,
                         "--script",
                     ],
-                    check=True,
                     timeout=prisma_command_timeout(),
                     stdout=f,
                     env=_get_prisma_env(),
@@ -437,7 +464,7 @@ class ProxyExtrasDBManager:
             migration_files = sorted(Path(migrations_dir).glob("*/migration.sql"))
             for mig_file in migration_files:
                 try:
-                    subprocess.run(
+                    prisma_toolchain.run_prisma(
                         [
                             _get_prisma_command(),
                             "db",
@@ -448,9 +475,6 @@ class ProxyExtrasDBManager:
                             schema_path,
                         ],
                         timeout=prisma_command_timeout(),
-                        check=True,
-                        capture_output=True,
-                        text=True,
                         env=_get_prisma_env(),
                     )
                     logger.info(f"Applied migration: {mig_file.parent.name}")
@@ -483,7 +507,7 @@ class ProxyExtrasDBManager:
         applied_ok = False
         try:
             logger.info("Running prisma db execute to apply the migration diff...")
-            result = subprocess.run(
+            result = prisma_toolchain.run_prisma(
                 [
                     _get_prisma_command(),
                     "db",
@@ -494,9 +518,6 @@ class ProxyExtrasDBManager:
                     schema_path,
                 ],
                 timeout=prisma_command_timeout(),
-                check=True,
-                capture_output=True,
-                text=True,
                 env=_get_prisma_env(),
             )
             logger.info(f"prisma db execute stdout: {result.stdout}")
@@ -525,7 +546,7 @@ class ProxyExtrasDBManager:
         for migration_name in migration_names:
             try:
                 logger.info(f"Resolving migration: {migration_name}")
-                subprocess.run(
+                prisma_toolchain.run_prisma(
                     [
                         _get_prisma_command(),
                         "migrate",
@@ -534,9 +555,6 @@ class ProxyExtrasDBManager:
                         migration_name,
                     ],
                     timeout=prisma_command_timeout(),
-                    check=True,
-                    capture_output=True,
-                    text=True,
                     env=_get_prisma_env(),
                 )
                 logger.debug(f"Resolved migration: {migration_name}")
@@ -716,6 +734,9 @@ class ProxyExtrasDBManager:
         Ahead-of-HEAD state (DB has migrations newer than this build ships)
         is logged as a warning, not a fatal error — users whose DBs got into
         weird shapes from the old thrashing should still be able to start.
+
+        The retry budget only counts attempts that made no progress: see
+        _MigrateAttemptBudget.
         """
         schema_path = ProxyExtrasDBManager._get_prisma_dir() + "/schema.prisma"
         migrations_dir = ProxyExtrasDBManager._get_prisma_dir()
@@ -726,11 +747,12 @@ class ProxyExtrasDBManager:
             original_dir = os.getcwd()
             os.chdir(migrations_dir)
             try:
-                subprocess.run(
+                prisma_toolchain.run_prisma(
                     [_get_prisma_command(), "db", "push", "--accept-data-loss"],
                     timeout=prisma_command_timeout(),
-                    check=True,
                     env=_get_prisma_env(),
+                    stdout=None,
+                    stderr=None,
                 )
                 return True
             except (
@@ -749,15 +771,13 @@ class ProxyExtrasDBManager:
         original_dir = os.getcwd()
         os.chdir(migrations_dir)
         deploy_timeout = prisma_migrate_deploy_timeout()
+        budget = _MigrateAttemptBudget(attempts_left=MAX_MIGRATE_DEPLOY_ATTEMPTS)
         try:
-            for attempt in range(4):
+            while not budget.exhausted:
                 try:
-                    result = subprocess.run(
+                    result = prisma_toolchain.run_prisma(
                         [_get_prisma_command(), "migrate", "deploy"],
                         timeout=deploy_timeout,
-                        check=True,
-                        capture_output=True,
-                        text=True,
                         env=_get_prisma_env(),
                     )
                     logger.info(f"prisma migrate deploy stdout: {result.stdout}")
@@ -767,168 +787,155 @@ class ProxyExtrasDBManager:
                     logger.warning(
                         "prisma migrate deploy attempt %s timed out after %ss, retrying. "
                         "Raise %s if this database needs longer to apply its pending migrations.",
-                        attempt + 1,
+                        budget.attempt_number,
                         deploy_timeout,
                         PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR,
                     )
-                    time.sleep(random.randrange(5, 15))
-                    continue
+                    next_budget = budget.spend()
 
                 except subprocess.CalledProcessError as e:
-                    stderr = e.stderr or ""
+                    next_budget = ProxyExtrasDBManager._budget_after_deploy_failure(
+                        e, budget, schema_path
+                    )
 
-                    if "P3005" in stderr and "database schema is not empty" in stderr:
-                        logger.info(
-                            "Schema exists but no migrations ledger — creating baseline"
-                        )
-                        ProxyExtrasDBManager._create_baseline_migration(schema_path)
-                        continue
-
-                    if "P3009" in stderr:
-                        migration_match = re.search(r"`(\d+_\S+?)`", stderr)
-                        if (
-                            migration_match
-                            and ProxyExtrasDBManager._is_idempotent_error(stderr)
-                        ):
-                            name = migration_match.group(1)
-                            logger.info(
-                                f"Migration {name} failed idempotently — marking applied and retrying"
-                            )
-                            try:
-                                ProxyExtrasDBManager._roll_back_migration(name)
-                            except (
-                                subprocess.CalledProcessError,
-                                subprocess.TimeoutExpired,
-                            ):
-                                pass  # may already be rolled-back
-                            try:
-                                ProxyExtrasDBManager._resolve_specific_migration(name)
-                            except (
-                                subprocess.CalledProcessError,
-                                subprocess.TimeoutExpired,
-                            ) as resolve_err:
-                                # We're already inside the outer
-                                # `except CalledProcessError` handler —
-                                # re-raising CalledProcessError from here
-                                # would escape as itself, bypassing
-                                # proxy_cli.py's `except RuntimeError`.
-                                raise RuntimeError(
-                                    f"Failed to mark migration {name} as applied "
-                                    f"after idempotent recovery. Manual "
-                                    f"intervention may be required.\n\n"
-                                    f"Detail: {resolve_err}"
-                                ) from resolve_err
-                            continue
-                        if migration_match:
-                            migration_name = migration_match.group(1)
-                            ledger_logs = ProxyExtrasDBManager._failed_migration_logs(migration_name)
-                            if ledger_logs is not None and (
-                                ledger_logs == "" or _MIGRATION_DEADLOCK_MARKER in ledger_logs
-                            ):
-                                logger.info(
-                                    "Migration %s failed in a concurrent migrate deploy "
-                                    "deadlock race, rolling its ledger row back and retrying",
-                                    migration_name,
-                                )
-                                ProxyExtrasDBManager._roll_back_migration_best_effort(migration_name)
-                                time.sleep(random.randrange(5, 15))
-                                continue
-                        raise RuntimeError(
-                            "Database migration failed and cannot be auto-recovered. "
-                            f"Manual intervention required.\n\nPrisma error:\n{stderr}"
-                        ) from e
-
-                    if "P3018" in stderr:
-                        if ProxyExtrasDBManager._is_permission_error(stderr):
-                            raise RuntimeError(
-                                "Database migration failed due to insufficient "
-                                "permissions. Please grant the required privileges "
-                                f"and retry.\n\nPrisma error:\n{stderr}"
-                            ) from e
-
-                        migration_match = re.search(
-                            r"Migration name: (\d+_\S+)", stderr
-                        )
-                        if (
-                            migration_match
-                            and ProxyExtrasDBManager._is_idempotent_error(stderr)
-                        ):
-                            name = migration_match.group(1)
-                            logger.info(
-                                f"Migration {name} SQL hit idempotent error — marking applied and retrying"
-                            )
-                            try:
-                                ProxyExtrasDBManager._roll_back_migration(name)
-                            except (
-                                subprocess.CalledProcessError,
-                                subprocess.TimeoutExpired,
-                            ):
-                                pass  # may already be rolled-back
-                            try:
-                                ProxyExtrasDBManager._resolve_specific_migration(name)
-                            except (
-                                subprocess.CalledProcessError,
-                                subprocess.TimeoutExpired,
-                            ) as resolve_err:
-                                raise RuntimeError(
-                                    f"Failed to mark migration {name} as applied "
-                                    f"after idempotent recovery. Manual "
-                                    f"intervention may be required.\n\n"
-                                    f"Detail: {resolve_err}"
-                                ) from resolve_err
-                            continue
-
-                        if migration_match and _MIGRATION_DEADLOCK_MARKER in stderr:
-                            logger.info(
-                                "Migration %s deadlocked against a concurrent "
-                                "migrate deploy, rolling its ledger row back "
-                                "and retrying",
-                                migration_match.group(1),
-                            )
-                            ProxyExtrasDBManager._roll_back_migration_best_effort(
-                                migration_match.group(1)
-                            )
-                            time.sleep(random.randrange(5, 15))
-                            continue
-
-                        raise RuntimeError(
-                            "Database migration failed and cannot be auto-recovered. "
-                            f"Manual intervention required.\n\nPrisma error:\n{stderr}"
-                        ) from e
-
-                    if _MIGRATION_DEADLOCK_MARKER in stderr:
-                        logger.info(
-                            "prisma migrate deploy attempt %s deadlocked against "
-                            "a concurrent migrate deploy, retrying",
-                            attempt + 1,
-                        )
-                        time.sleep(random.randrange(5, 15))
-                        continue
-
-                    if "P1002" in stderr and "advisory lock" in stderr:
-                        logger.info(
-                            "prisma migrate deploy attempt %s timed out waiting for "
-                            "the advisory lock a concurrent migrate deploy holds, retrying",
-                            attempt + 1,
-                        )
-                        time.sleep(random.randrange(5, 15))
-                        continue
-
-                    raise RuntimeError(
-                        "Database migration failed and cannot be auto-recovered. "
-                        f"Manual intervention required.\n\nPrisma error:\n{stderr}"
-                    ) from e
+                if next_budget.attempts_left < budget.attempts_left:
+                    time.sleep(random.randrange(5, 15))
+                budget = next_budget  # rebind-ok: the loop carries the budget from one migrate deploy pass to the next
 
             raise RuntimeError(
-                "Database migration failed after 4 attempts (retry loop "
-                "exhausted by timeouts, deadlock retries, or repeated "
-                "idempotent-recovery continues). Check database connectivity, "
+                f"Database migration failed after {MAX_MIGRATE_DEPLOY_ATTEMPTS} "
+                "attempts that made no progress (timeouts, deadlock retries, or a "
+                "recovery that had already run once). Check database connectivity, "
                 "load, and _prisma_migrations ledger state, and raise "
                 f"{PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR} if the attempts timed out."
             )
         finally:
             os.chdir(original_dir)
 
+    @staticmethod
+    def _budget_after_deploy_failure(
+        error: subprocess.CalledProcessError,
+        budget: "_MigrateAttemptBudget",
+        schema_path: str,
+    ) -> "_MigrateAttemptBudget":
+        """Recover from one failed `prisma migrate deploy`, and price the pass.
+
+        Returns the budget the next pass runs under, or raises when the failure
+        is not one this resolver knows how to recover from.
+        """
+        stderr = error.stderr or ""
+
+        if "P3005" in stderr and "database schema is not empty" in stderr:
+            logger.info("Schema exists but no migrations ledger — creating baseline")
+            if ProxyExtrasDBManager._create_baseline_migration(schema_path):
+                return budget.after_recovery("baseline")
+            return budget.spend()
+
+        if "P3009" in stderr:
+            migration_match = re.search(r"`(\d+_\S+?)`", stderr)
+            if migration_match and ProxyExtrasDBManager._is_idempotent_error(stderr):
+                name = migration_match.group(1)
+                logger.info(
+                    f"Migration {name} failed idempotently — marking applied and retrying"
+                )
+                ProxyExtrasDBManager._mark_migration_applied(name)
+                return budget.after_recovery(f"resolved:{name}")
+            if migration_match:
+                migration_name = migration_match.group(1)
+                ledger_logs = ProxyExtrasDBManager._failed_migration_logs(migration_name)
+                if ledger_logs is not None and (
+                    ledger_logs == "" or _MIGRATION_DEADLOCK_MARKER in ledger_logs
+                ):
+                    logger.info(
+                        "Migration %s failed in a concurrent migrate deploy "
+                        "deadlock race, rolling its ledger row back and retrying",
+                        migration_name,
+                    )
+                    ProxyExtrasDBManager._roll_back_migration_best_effort(migration_name)
+                    return budget.spend()
+            raise RuntimeError(
+                "Database migration failed and cannot be auto-recovered. "
+                f"Manual intervention required.\n\nPrisma error:\n{stderr}"
+            ) from error
+
+        if "P3018" in stderr:
+            if ProxyExtrasDBManager._is_permission_error(stderr):
+                raise RuntimeError(
+                    "Database migration failed due to insufficient "
+                    "permissions. Please grant the required privileges "
+                    f"and retry.\n\nPrisma error:\n{stderr}"
+                ) from error
+
+            migration_match = re.search(r"Migration name: (\d+_\S+)", stderr)
+            if migration_match and ProxyExtrasDBManager._is_idempotent_error(stderr):
+                name = migration_match.group(1)
+                logger.info(
+                    f"Migration {name} SQL hit idempotent error — marking applied and retrying"
+                )
+                ProxyExtrasDBManager._mark_migration_applied(name)
+                return budget.after_recovery(f"resolved:{name}")
+
+            if migration_match and _MIGRATION_DEADLOCK_MARKER in stderr:
+                logger.info(
+                    "Migration %s deadlocked against a concurrent "
+                    "migrate deploy, rolling its ledger row back "
+                    "and retrying",
+                    migration_match.group(1),
+                )
+                ProxyExtrasDBManager._roll_back_migration_best_effort(
+                    migration_match.group(1)
+                )
+                return budget.spend()
+
+            raise RuntimeError(
+                "Database migration failed and cannot be auto-recovered. "
+                f"Manual intervention required.\n\nPrisma error:\n{stderr}"
+            ) from error
+
+        if _MIGRATION_DEADLOCK_MARKER in stderr:
+            logger.info(
+                "prisma migrate deploy attempt %s deadlocked against "
+                "a concurrent migrate deploy, retrying",
+                budget.attempt_number,
+            )
+            return budget.spend()
+
+        if "P1002" in stderr and "advisory lock" in stderr:
+            logger.info(
+                "prisma migrate deploy attempt %s timed out waiting for "
+                "the advisory lock a concurrent migrate deploy holds, retrying",
+                budget.attempt_number,
+            )
+            return budget.spend()
+
+        raise RuntimeError(
+            "Database migration failed and cannot be auto-recovered. "
+            f"Manual intervention required.\n\nPrisma error:\n{stderr}"
+        ) from error
+
+    @staticmethod
+    def _mark_migration_applied(name: str) -> None:
+        """Roll a failed ledger row back if it is still there, then mark it applied."""
+        try:
+            ProxyExtrasDBManager._roll_back_migration(name)
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
+            pass  # may already be rolled-back
+        try:
+            ProxyExtrasDBManager._resolve_specific_migration(name)
+        except (
+            subprocess.CalledProcessError,
+            subprocess.TimeoutExpired,
+        ) as resolve_err:
+            # We're called from inside an `except CalledProcessError` handler —
+            # re-raising CalledProcessError from here would escape as itself,
+            # bypassing proxy_cli.py's `except RuntimeError`.
+            raise RuntimeError(
+                f"Failed to mark migration {name} as applied "
+                f"after idempotent recovery. Manual "
+                f"intervention may be required.\n\n"
+                f"Detail: {resolve_err}"
+            ) from resolve_err
+
     @staticmethod
     def apply_replica_identity_full_if_requested() -> bool:
         """
@@ -1007,12 +1014,9 @@ class ProxyExtrasDBManager:
                     logger.info("Running prisma migrate deploy")
                     try:
                         # Set migrations directory for Prisma
-                        result = subprocess.run(
+                        result = prisma_toolchain.run_prisma(
                             [_get_prisma_command(), "migrate", "deploy"],
                             timeout=prisma_migrate_deploy_timeout(),
-                            check=True,
-                            capture_output=True,
-                            text=True,
                             env=_get_prisma_env(),
                         )
                         logger.info(f"prisma migrate deploy stdout: {result.stdout}")
@@ -1084,7 +1088,7 @@ class ProxyExtrasDBManager:
                                         f"Found failed migration: {failed_migration}, marking as rolled back"
                                     )
                                     # Mark the failed migration as rolled back
-                                    subprocess.run(
+                                    prisma_toolchain.run_prisma(
                                         [
                                             _get_prisma_command(),
                                             "migrate",
@@ -1093,9 +1097,6 @@ class ProxyExtrasDBManager:
                                             failed_migration,
                                         ],
                                         timeout=prisma_command_timeout(),
-                                        check=True,
-                                        capture_output=True,
-                                        text=True,
                                         env=_get_prisma_env(),
                                     )
                                     logger.info(
@@ -1220,10 +1221,12 @@ class ProxyExtrasDBManager:
                     if ProxyExtrasDBManager.spend_logs_is_partitioned():
                         raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
                     # Use prisma db push with increased timeout
-                    subprocess.run(
+                    prisma_toolchain.run_prisma(
                         [_get_prisma_command(), "db", "push", "--accept-data-loss"],
                         timeout=prisma_command_timeout(),
-                        check=True,
+                        stdout=None,
+                        stderr=None,
+                        env=_get_prisma_env(),
                     )
                     return True
             except subprocess.TimeoutExpired:

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.92"
+version = "0.4.93"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.92"
+version = "0.4.93"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-proxy-extras/tests/test_setup_database_fail_fast.py

@@ -42,7 +42,7 @@ def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path):
         "Error: P3018\nMigration name: 20250326162113_baseline\n"
         "Database error code: 42501\npermission denied for schema public"
     )
-    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+    with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
         with pytest.raises(RuntimeError, match="permission"):
             ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
 
@@ -60,7 +60,7 @@ def test_v2_non_idempotent_p3009_raises_runtime_error(monkeypatch, tmp_path):
         "Error: P3009\nMigration `20260101000000_genuinely_broken` failed\n"
         'Reason: syntax error at or near "BRKN" LINE 42'
     )
-    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+    with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
         with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
             ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
 
@@ -124,7 +124,7 @@ def test_v1_default_still_calls_resolve_all_migrations(monkeypatch, tmp_path):
     def fake_resolve(*args, **kwargs):
         resolve_called["n"] += 1
 
-    monkeypatch.setattr("subprocess.run", fake_run)
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", fake_run)
     monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", fake_resolve)
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True)  # v2 flag NOT set
@@ -139,7 +139,7 @@ def test_v2_db_push_wraps_subprocess_error_as_runtime_error(monkeypatch, tmp_pat
     (tmp_path / "schema.prisma").write_text("// stub")
 
     stderr = "db push error"
-    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+    with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
         with pytest.raises(RuntimeError, match="prisma db push failed"):
             ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True)
 
@@ -209,7 +209,7 @@ def test_v2_resolve_specific_migration_failure_raises_runtime_error(
         "Error: P3009\nMigration `20260101000000_some_migration` failed\n"
         "relation already exists"
     )
-    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+    with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
         with pytest.raises(
             RuntimeError, match="Failed to mark migration .* as applied"
         ):
@@ -228,7 +228,7 @@ def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path):
         stdout = "Applied migration.\n"
         stderr = ""
 
-    monkeypatch.setattr("subprocess.run", lambda *a, **kw: FakeResult())
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", lambda *a, **kw: FakeResult())
 
     resolve_called = {"n": 0}
     monkeypatch.setattr(
@@ -296,7 +296,7 @@ def test_v2_p3018_deadlock_rolls_back_and_retries(monkeypatch, tmp_path):
         "_resolve_specific_migration",
         lambda name: pytest.fail("a deadlocked migration must never be marked applied"),
     )
-    monkeypatch.setattr("subprocess.run", _succeed_after(1, _DEADLOCK_P3018_STDERR))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, _DEADLOCK_P3018_STDERR))
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
     assert ok is True
@@ -309,7 +309,7 @@ def test_v2_p3018_persistent_deadlock_exhausts_attempts(monkeypatch, tmp_path):
     monkeypatch.setattr(ProxyExtrasDBManager, "_roll_back_migration", lambda name: None)
 
     with patch(
-        "subprocess.run",
+        "litellm_proxy_extras.prisma_toolchain.run_prisma",
         side_effect=_fake_migrate_deploy_failure(1, _DEADLOCK_P3018_STDERR),
     ):
         with pytest.raises(RuntimeError, match="after 4 attempts"):
@@ -343,7 +343,7 @@ def test_v2_p3009_deadlocked_ledger_row_rolls_back_and_retries(monkeypatch, tmp_
         "_resolve_specific_migration",
         lambda name: pytest.fail("a deadlocked migration must never be marked applied"),
     )
-    monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
     assert ok is True
@@ -372,7 +372,7 @@ def test_v2_p3009_empty_ledger_logs_rolls_back_and_retries(monkeypatch, tmp_path
         "_resolve_specific_migration",
         lambda name: pytest.fail("a deadlocked migration must never be marked applied"),
     )
-    monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
     assert ok is True
@@ -395,7 +395,7 @@ def test_v2_p3009_unreadable_ledger_still_raises(monkeypatch, tmp_path):
         "_roll_back_migration",
         lambda name: pytest.fail("an unreadable ledger must not trigger a retry"),
     )
-    monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
 
     with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
         ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
@@ -417,7 +417,7 @@ def test_v2_p3009_non_deadlock_ledger_row_still_raises(monkeypatch, tmp_path):
         lambda name: 'ERROR: syntax error at or near "BRKN"',
     )
 
-    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+    with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)):
         with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
             ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
 
@@ -427,7 +427,7 @@ def test_v2_bare_deadlock_stderr_retries(monkeypatch, tmp_path):
     waiter as victim) is retried, not fatal."""
     _stub_v2_env(monkeypatch, tmp_path)
     monkeypatch.setattr(
-        "subprocess.run", _succeed_after(1, "Database error: deadlock detected")
+        "litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, "Database error: deadlock detected")
     )
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
@@ -446,7 +446,10 @@ def test_v2_advisory_lock_timeout_retries(monkeypatch, tmp_path):
     """v2: the advisory-lock waiter that times out while a peer's retry holds
     the lock retries instead of dying."""
     _stub_v2_env(monkeypatch, tmp_path)
-    monkeypatch.setattr("subprocess.run", _succeed_after(2, _P1002_ADVISORY_LOCK_STDERR))
+    monkeypatch.setattr(
+        "litellm_proxy_extras.prisma_toolchain.run_prisma",
+        _succeed_after(2, _P1002_ADVISORY_LOCK_STDERR),
+    )
 
     ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
     assert ok is True
@@ -456,7 +459,7 @@ def test_v2_p1002_without_advisory_lock_context_still_raises(monkeypatch, tmp_pa
     """v2: a plain P1002 (database unreachable) stays fatal."""
     _stub_v2_env(monkeypatch, tmp_path)
     stderr = "Error: P1002\n\nThe database server at `db`:`5432` was reached but timed out."
-    monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr))
+    monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr))
 
     with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
         ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)

litellm-rust/ADDING_A_PROVIDER.md

@@ -26,4 +26,4 @@ variants of it. The test for a good abstraction is that adding the next provider
 is a few declarative lines, not a new file of duplicated flow. Only diverge from
 the base when behavior is genuinely different, and say so explicitly in the PR.
 
-**Calling:** hosts invoke the core entrypoint — the Python bridge and the `ai-gateway` route service both call `litellm_core::messages::messages`. Never add a provider handler to `ai-gateway`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`.
+**Calling:** hosts invoke the core entrypoint — the Python bridge and the `ai-gateway` route service both call `litellm_core::messages::messages`. Never add a provider handler to `ai-gateway`. Register new modules in `lib.rs` / `mod.rs`, then run the commands under "Checks" in [CLAUDE.md](CLAUDE.md).

litellm-rust/AGENTS.md

@@ -1,17 +1,18 @@
 # AGENTS.md
 
-litellm-rust has four crates. A crate is a layer or shared foundation, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are modules inside the layers.
+litellm-rust has five crates. A crate is a layer or shared foundation, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are modules inside the layers.
 
 ## Crates
 
 | Crate | Role |
 |-------|------|
 | litellm-core | The LiteLLM SDK in Rust. One public entrypoint per top-level call (`messages::messages()`), owning types, transforms, provider resolution, auth, and the provider HTTP call. Call it, get a typed response. |
+| litellm-config | Config-loading boundary. Returns resolved core deployment data and optionally delegates loading to Python. |
 | litellm-ai-gateway | The axum server (behind the `server` feature) plus the WebSocket hosts. Translates HTTP/WS to core entrypoints; owns no provider logic and no handlers. |
 | litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. |
 | litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. |
 
-Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate.
+Dependency direction is acyclic: `litellm-config` depends on `litellm-core`, the gateway depends on both, and `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`. The interop foundation depends on no LiteLLM domain crate.
 
 ## Where a route lives
 

litellm-rust/CLAUDE.md

@@ -24,6 +24,7 @@ the base when behavior is genuinely different, and say so explicitly in the PR.
 ## Crates (see AGENTS.md)
 
 `litellm-core` **is** the LiteLLM SDK in Rust: it makes the LLM call.
+`litellm-config` is the config-loading boundary and returns resolved core types.
 `litellm-ai-gateway` is an HTTP/WebSocket server in front of it, and
 `litellm-python-bridge` exposes it to the Python SDK. `litellm-python-interop`
 holds domain-neutral PyO3 primitives shared by Python-facing Rust code. A crate
@@ -174,10 +175,14 @@ for changes under `litellm-rust/`.
 ```bash
 cd litellm-rust
 cargo fmt --check
+cargo clippy --workspace --all-targets -- -D warnings
+cargo clippy -p litellm-core --all-targets --features bedrock-auth -- -D warnings
 # the ai-gateway binary + server code is behind the `server` feature
-cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings
-cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings
+cargo clippy -p litellm-ai-gateway --all-targets --all-features -- -D warnings
 cargo test --workspace
+cargo test -p litellm-core --features bedrock-auth
+# the `auth`, `routes`, `state` and `realtime` tests only exist under `server`
+cargo test -p litellm-ai-gateway --features server
 ```
 
 When a Rust path is exposed through Python, add Python parity tests that compare

litellm-rust/Cargo.lock

@@ -1412,8 +1412,8 @@ dependencies = [
  "base64",
  "futures-channel",
  "futures-util",
+ "litellm-config",
  "litellm-core",
- "pyo3",
  "reqwest",
  "serde",
  "serde_json",
@@ -1425,6 +1425,16 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "litellm-config"
+version = "0.1.0"
+dependencies = [
+ "litellm-core",
+ "pyo3",
+ "serde_json",
+ "thiserror 2.0.19",
+]
+
 [[package]]
 name = "litellm-core"
 version = "0.1.0"
@@ -1435,14 +1445,17 @@ dependencies = [
  "aws-sigv4",
  "aws-smithy-runtime-api",
  "aws-types",
+ "base64",
  "rand 0.8.7",
  "reqwest",
+ "rstest",
  "serde",
  "serde_json",
  "sha2 0.10.9",
  "thiserror 2.0.19",
  "tokio",
  "tracing",
+ "tracing-subscriber",
 ]
 
 [[package]]
@@ -1461,7 +1474,6 @@ dependencies = [
  "tokio",
  "tokio-tungstenite",
  "tracing",
- "tracing-subscriber",
 ]
 
 [[package]]
@@ -1511,6 +1523,16 @@ version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
+[[package]]
+name = "mime_guess"
+version = "2.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
+dependencies = [
+ "mime",
+ "unicase",
+]
+
 [[package]]
 name = "mio"
 version = "1.2.2"
@@ -1969,6 +1991,7 @@ dependencies = [
  "hyper-util",
  "js-sys",
  "log",
+ "mime_guess",
  "percent-encoding",
  "pin-project-lite",
  "quinn",
@@ -2736,6 +2759,12 @@ version = "1.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
 
+[[package]]
+name = "unicase"
+version = "2.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.24"

litellm-rust/Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 members = [
     "crates/core",
+    "crates/config",
     "crates/ai-gateway",
     "crates/python-interop",
     "crates/python-bridge",
@@ -17,6 +18,7 @@ repository = "https://github.com/BerriAI/litellm"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] }
 litellm-core = { path = "crates/core" }
+litellm-config = { path = "crates/config" }
 litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false }
 litellm-python-interop = { path = "crates/python-interop" }
 axum = "0.7"
@@ -24,10 +26,10 @@ pyo3 = "0.29.2"
 pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] }
 pythonize = "0.29.0"
 rand = "0.8"
-reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "http2", "stream"] }
+reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "multipart", "rustls-tls", "http2", "stream"] }
 rstest = "0.26.1"
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
+serde_json = { version = "1.0", features = ["float_roundtrip"] }
 sha2 = "0.10"
 subtle = "2"
 thiserror = "2.0"

litellm-rust/README.md

@@ -25,11 +25,12 @@ coverage and production evidence.
 | Crate | Role |
 |-------|------|
 | litellm-core | The SDK. Per-route entrypoints (`messages::messages()`), types, provider transforms (modules under `providers/`), provider resolution, auth, the provider HTTP call, and the router. |
+| litellm-config | Config-loading boundary. Returns resolved deployments and optionally delegates loading to Python. |
 | litellm-ai-gateway | The axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. |
 | litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. |
 | litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. |
 
-Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate.
+Dependency direction is acyclic: config depends on core, the gateway depends on config and core, and the Python bridge depends on the domain layers and Python interop.
 
 ## Layout
 
@@ -38,6 +39,7 @@ crates/
   core/           The SDK: route modules + provider transforms.
     src/messages/   mod.rs (entrypoint), types, transformation, prepare, handler, client
     src/providers/anthropic/messages/transformation.rs
+  config/         Config loading and resolved deployments.
   ai-gateway/     Axum server + WebSocket hosts; calls core entrypoints.
   python-interop/ Domain-neutral PyO3 conversion and GIL primitives.
   python-bridge/  PyO3 API adapter for Python LiteLLM.
@@ -49,11 +51,6 @@ function per top-level route, mirroring the core entrypoints.
 
 ## Checks
 
-Run these before pushing Rust changes. GitHub Actions runs the same checks for
-changes under `litellm-rust/`.
-
-```bash
-cargo fmt --check
-cargo clippy --workspace --all-targets -- -D warnings
-cargo test --workspace
-```
+Run the commands under "Checks" in [CLAUDE.md](CLAUDE.md) before pushing Rust
+changes. That list is the single source of truth and matches what GitHub Actions
+runs for changes under `litellm-rust/`.

litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md

@@ -45,15 +45,9 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. `messages`
 
 22. A Python -> Rust bridge keeps the Python side minimal: the Python interface only marshals inputs and calls the Rust interface, with no transform, handler, or business logic. Aim for well under 100 lines of interface code per route; if the Python grows past that, the logic belongs in Rust.
 23. Do not bloat `litellm/main.py`. A route's provider dispatch lives in a thin dispatch class under `litellm/llms/<provider>/<route>/` that calls the Rust bridge; `main.py` only instantiates it and calls its sync/async method.
-24. Do not add new feature flags unless explicitly requested. Reuse the existing litellm rust rollout mechanism (`use_litellm_rust`); never introduce a per-route env flag such as `LITELLM_USE_RUST_<ROUTE>`.
+24. Do not add new feature flags unless explicitly requested. Reuse the existing LiteLLM Rust rollout mechanism (`litellm.rust`); never introduce a per-route env flag such as `LITELLM_USE_RUST_<ROUTE>`.
 
 ## Checks before push
 
-25. Run, and keep green:
-    ```bash
-    cd litellm-rust
-    cargo fmt --check
-    cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings
-    cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings
-    cargo test --workspace
-    ```
+25. Run, and keep green, the commands under "Checks" in `litellm-rust/CLAUDE.md`.
+    That list is the single source of truth and matches what GitHub Actions runs.

litellm-rust/crates/ai-gateway/AGENTS.md

@@ -9,19 +9,15 @@ such as `litellm_core::messages::messages`. No provider handler lives here.
 src/
   main.rs            # entrypoint: build AppState (router + master key), bind, serve
   state.rs           # AppState — shared Arc<Router> + master_key
-  gil.rs             # GIL-activity tracker (records Python acquisitions)
   auth/              # authentication as an axum extractor — added to handler args
     mod.rs           #   RequireMasterKey: FromRequestParts, single master key (LITELLM_MASTER_KEY)
   routes/            # one module per route, all matching the same template
     AGENTS.md        #   ← the route template (read this before adding a route)
     mod.rs           #   app(): merges every module's router()
     health.rs        #   simple route (one file): router() + liveness/readiness
-    gil.rs           #   simple route (one file): router() + GET /health/gil
     realtime/        #   route with logic → axum surface + a no-axum service:
       mod.rs         #     router() + handler + WS<->events adapter (the axum surface)
       service.rs     #     business logic (select deployment, call provider) — no axum, testable
-  python/            # Python interop (feature: python-config) — load-time only
-    mod.rs, config.rs, AGENTS.md
 ```
 
 ## Rules
@@ -53,5 +49,6 @@ proxy in a later phase. Health routes don't add the extractor (unauthenticated).
 
 ## Python interop
 
-Anything that calls into Python lives in `python/` and is **load-time only** — see
-`python/AGENTS.md`. The realtime data path never takes the GIL.
+Python-backed loading lives in `litellm-config` and is **load-time only**. The
+gateway's `python-config` feature forwards to that crate. The realtime data path
+never takes the GIL.

litellm-rust/crates/ai-gateway/ARCHITECTURE.md

@@ -9,4 +9,6 @@ flowchart LR
   C[client] <--> G[Rust ai-gateway<br/>LLM inference]
   G <--> O[OpenAI realtime]
   G -. spend tracking callback .-> P[litellm proxy]
+  F[litellm-config<br/>load-time only] --> G
+  F -. Python backend .-> P
 ```

litellm-rust/crates/ai-gateway/Cargo.toml

@@ -16,6 +16,7 @@ required-features = ["server"]
 [dependencies]
 tracing.workspace = true
 litellm-core = { workspace = true, features = ["bedrock-auth"] }
+litellm-config.workspace = true
 # reqwest (rustls + json) is used by io/ocr and ships realtime logs to the
 # Python proxy callbacks API.
 reqwest.workspace = true
@@ -31,14 +32,15 @@ subtle = { workspace = true, optional = true }
 # sha2 hashes the master key into user_api_key_hash (matches the proxy's
 # SHA-256 hash_token) so the plaintext credential never enters a log payload.
 sha2 = { workspace = true, optional = true }
-pyo3 = { workspace = true, features = ["auto-initialize"], optional = true }
+tower = { version = "0.5.3", features = ["util"], optional = true }
 
 [features]
 default = []
 server = ["dep:axum", "dep:subtle", "dep:sha2"]
 # Build the gateway's config from the proxy YAML via an embedded Python
 # interpreter (links libpython; requires `litellm` importable at runtime).
-python-config = ["dep:pyo3"]
+python-config = ["litellm-config/python"]
+trace-parity = ["server", "dep:tower", "litellm-core/observability"]
 
 [dev-dependencies]
 futures-channel = "0.3"

litellm-rust/crates/ai-gateway/Dockerfile

@@ -36,12 +36,12 @@ FROM chef AS builder
 # whenever only gateway source changes.
 COPY --from=planner /build/litellm-rust/recipe.json recipe.json
 RUN cargo chef cook --locked --release \
-        -p litellm-ai-gateway --features python-config \
+        -p litellm-ai-gateway --features server,python-config \
         --recipe-path recipe.json
 # Now copy the real sources and build the gateway binary. Deps are already cooked
 # above, so this step only recompiles the gateway crate.
 COPY litellm-rust/ .
-RUN cargo build --locked --release -p litellm-ai-gateway --features python-config
+RUN cargo build --locked --release -p litellm-ai-gateway --bin litellm-ai-gateway --features server,python-config
 
 # ---- Runtime ----------------------------------------------------------------
 # python:3.11-slim-bookworm ships libpython3.11, matching the builder's PyO3

litellm-rust/crates/ai-gateway/README.md

@@ -6,25 +6,30 @@ dials OpenAI upstream, and splices the two sockets frame-by-frame.
 
 ## Crates
 
-`litellm-rust` has four crates. A crate is a layer or shared foundation, not a route:
+`litellm-rust` has five crates. A crate is a layer or shared foundation, not a route:
 
 | Crate | Role |
 |-------|------|
 | litellm-core | The LiteLLM SDK in Rust — per-route entrypoints (`messages::messages()`) that resolve the provider, transform, and make the call; plus types, provider transforms, and the router. |
+| litellm-config | Config-loading boundary. Returns resolved deployments and optionally delegates loading to Python. |
 | litellm-ai-gateway | The Axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. |
 | litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. |
 | litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. |
 
-Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate.
+Dependency direction is acyclic: config depends on core, the gateway depends on config and core, and the Python bridge depends on the domain layers and Python interop.
 
 - **Client endpoint:** `wss://<host>/v1/realtime?model=<model>` (WebSocket)
 - **Auth:** `Authorization: Bearer $LITELLM_MASTER_KEY` (fails closed if unset)
-- **Health:** `GET /health/readiness`, `GET /health/liveness`, `GET /health/gil`
+- **Health:** `GET /health/readiness`, `GET /health/liveness`
 - **Request logs:** POSTed to a LiteLLM proxy at `/v1/rust_control_plane/logs` (see [Request logging](#request-logging))
 
 > **Realtime serving is pure Rust.** Python is used at **load time only** — to
 > read the config once at boot. The realtime hot path never touches Python.
 
+The former `/health/gil` route and its acquisition counter were removed. They
+only observed the single startup config load and did not prove that every GIL
+acquisition was instrumented
+
 ## Configuration (config.yaml)
 
 The gateway loads its `model_list` from a **config.yaml**, the same as the
@@ -43,9 +48,10 @@ model_list:
 LITELLM_CONFIG_PATH=./config.yaml ./litellm-ai-gateway
 ```
 
-At boot the gateway calls into `litellm.proxy.read_model_list`, which reuses the
-**real proxy config reader** (`ProxyConfig.get_config`). That means everything
-the proxy supports in config.yaml works here too:
+At boot `litellm-config` calls into `litellm.proxy.read_model_list` and returns
+resolved deployments to the gateway, which constructs the router. The Python
+backend still reuses the **real proxy config reader** (`ProxyConfig.get_config`),
+so everything the proxy supports in config.yaml works here too:
 
 - `include:` to merge in other config files,
 - `os.environ/VAR` secret references (resolved via the secret manager, never
@@ -82,8 +88,8 @@ stand-in built from the environment:
 |---|---|---|
 | `OPENAI_REALTIME_MODEL` | `gpt-realtime` | The single deployment's model name (also the `?model=` clients pass). |
 
-This mode links no libpython and needs no config file, but it only supports one
-hard-coded OpenAI deployment. **config.yaml is the recommended path** — use the
+The default workspace build links no libpython and needs no config file. This
+fallback mode only supports one hard-coded OpenAI deployment. **config.yaml is the recommended path** — use the
 stand-in only for the leanest possible build.
 
 ## Request logging
@@ -100,7 +106,7 @@ Worker tuning, rarely needed: `LITELLM_LOG_CHANNEL_CAPACITY` (4096),
 
 ## Build & run with Docker
 
-The image is built `--features python-config` and installs litellm **from this
+The image is built `--features server,python-config` and installs litellm **from this
 repo's source** (the config reader is newer than any PyPI release), so the build
 **context is the repo root**:
 
@@ -135,10 +141,10 @@ docker run --rm -p 4001:4001 \
 ```bash
 # config.yaml mode — needs litellm importable in the active python env
 LITELLM_CONFIG_PATH=./crates/ai-gateway/config.yaml \
-  cargo run --release -p litellm-ai-gateway --features python-config
+  cargo run --release -p litellm-ai-gateway --features server,python-config
 
 # env stand-in mode — no python, no config
-cargo run --release -p litellm-ai-gateway
+cargo run --release -p litellm-ai-gateway --features server
 ```
 
 ## Deploy on Render

litellm-rust/crates/ai-gateway/config.yaml

@@ -1,8 +1,8 @@
 # Sample realtime config for the LiteLLM Rust AI Gateway.
 #
-# The gateway loads this model_list at boot via the embedded python config
-# reader (litellm.proxy.read_model_list), which reuses the proxy's own reader —
-# so include:, os.environ/ secrets, and DB-stored models all work here too.
+# litellm-config resolves this model_list at boot through the Python config
+# reader (litellm.proxy.read_model_list), then the gateway builds its router.
+# Includes, environment secrets, and database-stored models still work.
 #
 # Secrets are referenced (never inlined) via os.environ/. A real deploy can
 # override this file (e.g. mount a Render secret file at LITELLM_CONFIG_PATH).

litellm-rust/crates/ai-gateway/src/gil.rs

@@ -1,58 +0,0 @@
-//! GIL-activity tracking.
-//!
-//! Every acquisition of the Python GIL is recorded here so the `/health/gil`
-//! endpoint can report whether Python was touched recently. The design goal is
-//! that the GIL is acquired **only at load time** (config read) and never on the
-//! realtime hot path — polling this endpoint during traffic should show the
-//! count holding steady and `acquired_last_30s` falling to `false`.
-
-use std::sync::atomic::{AtomicU64, Ordering};
-use std::time::{SystemTime, UNIX_EPOCH};
-
-/// Window (seconds) for the "recently acquired" signal.
-pub const RECENT_WINDOW_SECS: u64 = 30;
-
-static GIL_ACQUISITIONS: AtomicU64 = AtomicU64::new(0);
-/// Unix seconds of the last acquisition; `0` means "never".
-static LAST_GIL_UNIX_SECS: AtomicU64 = AtomicU64::new(0);
-
-fn now_unix_secs() -> u64 {
-    SystemTime::now()
-        .duration_since(UNIX_EPOCH)
-        .map(|d| d.as_secs())
-        .unwrap_or(0)
-}
-
-/// Record that the GIL was just acquired. Call immediately before taking the GIL.
-///
-/// Only invoked under the `python-config` feature; without it the gateway never
-/// touches Python, so the recorder is unused (and the endpoint reports zero).
-#[cfg_attr(not(feature = "python-config"), allow(dead_code))]
-pub fn record_acquisition() {
-    GIL_ACQUISITIONS.fetch_add(1, Ordering::Relaxed);
-    LAST_GIL_UNIX_SECS.store(now_unix_secs(), Ordering::Relaxed);
-}
-
-/// Point-in-time view of GIL activity.
-pub struct GilSnapshot {
-    pub total_acquisitions: u64,
-    pub seconds_since_last: Option<u64>,
-    pub acquired_last_30s: bool,
-}
-
-/// Read the current GIL-activity snapshot.
-pub fn snapshot() -> GilSnapshot {
-    let total = GIL_ACQUISITIONS.load(Ordering::Relaxed);
-    let last = LAST_GIL_UNIX_SECS.load(Ordering::Relaxed);
-    let seconds_since_last = if last == 0 {
-        None
-    } else {
-        Some(now_unix_secs().saturating_sub(last))
-    };
-    let acquired_last_30s = seconds_since_last.is_some_and(|secs| secs <= RECENT_WINDOW_SECS);
-    GilSnapshot {
-        total_acquisitions: total,
-        seconds_since_last,
-        acquired_last_30s,
-    }
-}

litellm-rust/crates/ai-gateway/src/lib.rs

@@ -10,29 +10,23 @@
 //! - [`io`]: compatibility exports and realtime WebSocket splice helpers.
 //! - The server modules ([`auth`], [`routes`], [`state`]) and anything pulling
 //!   `axum` are gated behind the `server` feature, which the `litellm-ai-gateway`
-//!   binary turns on. The `python-config` feature additionally pulls in [`python`]
-//!   for the load-time config reader.
+//!   binary turns on.
 
 pub mod audio_transcription;
 mod client;
 pub mod io;
 pub mod ocr;
 
-/// GIL-activity tracking. Pure (atomics only); shared by the `server` routes and
-/// the `python-config` reader, so it is available without either feature.
-pub mod gil;
-
 #[cfg(feature = "server")]
 pub mod auth;
 #[cfg(feature = "server")]
 pub mod routes;
 #[cfg(feature = "server")]
 pub mod state;
+#[cfg(feature = "trace-parity")]
+pub mod trace_parity;
 
 mod constants;
 pub mod integrations;
 #[cfg(feature = "server")]
 mod realtime;
-
-#[cfg(feature = "python-config")]
-pub mod python;

litellm-rust/crates/ai-gateway/src/main.rs

@@ -14,12 +14,12 @@ use std::sync::Arc;
 use litellm_ai_gateway::io::realtime_pool::{PoolConfig, RealtimePool, upstream_key};
 use litellm_ai_gateway::routes;
 use litellm_ai_gateway::state::AppState;
+#[cfg(feature = "python-config")]
+use litellm_config::load_model_list;
 use litellm_core::router::{Deployment, LiteLLMParams, Router};
 
 use litellm_ai_gateway::integrations::custom_logger::CustomLogger;
 use litellm_ai_gateway::integrations::litellm_python_proxy_api::LiteLLMPythonProxyAPILogger;
-#[cfg(feature = "python-config")]
-use litellm_ai_gateway::python;
 
 /// Bind to localhost by default so the gateway is not a public, unauthenticated
 /// provider proxy out of the box. Override with `HOST` (e.g. `0.0.0.0`).
@@ -124,10 +124,10 @@ fn resolve_port() -> u16 {
 fn build_router() -> Router {
     #[cfg(feature = "python-config")]
     if let Ok(config_path) = std::env::var("LITELLM_CONFIG_PATH") {
-        match python::config::load_router_from_config(&config_path) {
-            Ok(router) => {
+        match load_model_list(std::path::Path::new(&config_path)) {
+            Ok(deployments) => {
                 eprintln!("loaded model_list from {config_path} via python config reader");
-                return router;
+                return Router::new(deployments);
             }
             Err(err) => {
                 eprintln!("config load failed ({err}); falling back to env deployment");

litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs

@@ -12,6 +12,7 @@ use litellm_core::providers::azure_ai::ocr::transformation::{
     AZURE_AI_OCR_CONFIG, AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG,
 };
 use litellm_core::providers::mistral::ocr::transformation::MISTRAL_OCR_CONFIG;
+use litellm_core::providers::reducto::ocr::transformation as reducto;
 use litellm_core::providers::vertex_ai::ocr::transformation as vertex_ai;
 use litellm_core::providers::vertex_ai::ocr::transformation::{
     VERTEX_AI_DEEPSEEK_OCR_CONFIG, VERTEX_AI_OCR_CONFIG,
@@ -39,6 +40,7 @@ pub(super) fn ocr_provider_config(
 ) -> Option<&'static dyn OcrProviderConfig> {
     match provider {
         "mistral" => Some(&MISTRAL_OCR_CONFIG),
+        "reducto" => reducto::config_for_model(model),
         "azure_ai" if is_azure_document_intelligence_model(model) => {
             Some(&AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG)
         }
@@ -334,6 +336,7 @@ fn operation_status(response_json: &Value) -> Result<&str, Error> {
     }
 }
 
+#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
 pub(super) async fn poll_document_intelligence(
     operation_url: &str,
     original_url: &str,
@@ -392,9 +395,11 @@ pub(super) async fn poll_document_intelligence(
 
 #[cfg(test)]
 mod tests {
-    use super::*;
+    use litellm_core::ocr::transformation::OcrResponseHandling;
     use serde_json::json;
 
+    use super::*;
+
     #[test]
     fn blocks_private_and_metadata_ips() {
         assert!(is_blocked_ip("127.0.0.1".parse().unwrap()));
@@ -438,4 +443,87 @@ mod tests {
 
         assert_eq!(transformed, document);
     }
+
+    #[test]
+    fn truncate_error_body_passes_short_strings_through() {
+        let body = "Unauthorized";
+        assert_eq!(truncate_error_body(body), "Unauthorized");
+    }
+
+    #[test]
+    fn truncate_error_body_caps_long_payloads() {
+        let body = "x".repeat(306);
+        let truncated = truncate_error_body(&body);
+
+        assert!(truncated.ends_with("... (truncated)"));
+        let prefix_chars = truncated
+            .strip_suffix("... (truncated)")
+            .expect("truncated marker present")
+            .chars()
+            .count();
+        assert_eq!(prefix_chars, 256);
+    }
+
+    #[test]
+    fn truncate_error_body_does_not_split_multibyte_chars() {
+        let body = "é".repeat(266);
+        let truncated = truncate_error_body(&body);
+        assert!(truncated.is_char_boundary(truncated.len()));
+    }
+
+    #[test]
+    fn ocr_dispatch_supports_migrated_providers() {
+        assert!(ocr_provider_config("mistral", "mistral-ocr-latest").is_some());
+        assert!(
+            ocr_provider_config("azure_ai", "pixtral-12b-2409")
+                .expect("azure ai config resolves")
+                .requires_data_uri_document()
+        );
+        assert_eq!(
+            ocr_provider_config("azure_ai", "doc-intelligence/prebuilt-read")
+                .expect("document intelligence config resolves")
+                .response_handling(),
+            OcrResponseHandling::AzureDocumentIntelligencePoll
+        );
+        assert!(
+            ocr_provider_config("vertex_ai", "deepseek-ocr-maas")
+                .expect("vertex deepseek config resolves")
+                .supported_ocr_params()
+                .contains(&"temperature")
+        );
+        assert!(ocr_provider_config("openai", "gpt-4o").is_none());
+    }
+
+    #[test]
+    fn string_headers_accepts_string_values() {
+        let headers = json!({
+            "x-trace-id": "trace-1"
+        })
+        .as_object()
+        .unwrap()
+        .clone();
+
+        assert_eq!(
+            string_headers(Some(headers)).expect("string headers accepted"),
+            vec![("x-trace-id".to_string(), "trace-1".to_string())]
+        );
+    }
+
+    #[test]
+    fn string_headers_rejects_non_string_values() {
+        let headers = json!({
+            "x-retry-count": 3
+        })
+        .as_object()
+        .unwrap()
+        .clone();
+
+        let err = string_headers(Some(headers)).expect_err("non-string header rejected");
+        assert_eq!(
+            err,
+            Error::InvalidRequest(
+                "OCR extra_headers.x-retry-count must be a string, got number".to_string()
+            )
+        );
+    }
 }

litellm-rust/crates/ai-gateway/src/ocr/handler.rs

@@ -50,7 +50,11 @@ pub(crate) async fn execute_ocr_provider_call(
         .await?;
         return Ok(request
             .config
-            .transform_ocr_response(&request.model, response_json)?
+            .transform_ocr_response_with_params(
+                &request.model,
+                response_json,
+                &request.optional_params,
+            )?
             .into_json());
     }
 
@@ -71,6 +75,10 @@ pub(crate) async fn execute_ocr_provider_call(
 
     Ok(request
         .config
-        .transform_ocr_response(&request.model, response_json)?
+        .transform_ocr_response_with_params(
+            &request.model,
+            response_json,
+            &request.optional_params,
+        )?
         .into_json())
 }

litellm-rust/crates/ai-gateway/src/ocr/hooks.rs

@@ -1,11 +1,15 @@
 use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming};
 use litellm_core::error::Error;
+use litellm_core::providers::reducto::ocr::transformation::{
+    build_upload_request, extract_document_source, extract_upload_file_id,
+};
 use serde_json::{Map, Value, json};
 use std::future::Future;
 use std::pin::Pin;
 
-use super::common_utils::{convert_document_url_to_data_uri, string_headers};
+use super::common_utils::{convert_document_url_to_data_uri, string_headers, truncate_error_body};
 use super::types::{PreparedOcrRequest, ProviderOcrRequest};
+use crate::client::http_client;
 use crate::integrations::custom_guardrail::{
     CustomGuardrailRunner, GuardrailContext, GuardrailError, GuardrailRequest,
 };
@@ -89,22 +93,39 @@ impl OcrLifecycleHooks {
         )?;
         let model = request.model.clone();
         let custom_llm_provider = request.custom_llm_provider.clone();
-        let document = if config.requires_data_uri_document() {
+        let is_reducto = custom_llm_provider == "reducto";
+        let document = if is_reducto {
+            let guarded_document = self
+                .run_during_call_guardrails(&model, &custom_llm_provider, &url, request.document)
+                .await?;
+            upload_reducto_document(
+                &guarded_document,
+                request.api_base.as_deref(),
+                request.timeout,
+                &upstream_headers,
+            )
+            .await?
+        } else if config.requires_data_uri_document() {
             convert_document_url_to_data_uri(request.document).await?
         } else {
             request.document
         };
+        let optional_params = request.optional_params;
         let body = config
-            .transform_ocr_request(&request.model, document, request.optional_params)?
+            .transform_ocr_request(&request.model, document, optional_params.clone())?
             .data;
-        let body = self
-            .run_during_call_guardrails(&model, &custom_llm_provider, &url, body)
-            .await?;
+        let body = if is_reducto {
+            body
+        } else {
+            self.run_during_call_guardrails(&model, &custom_llm_provider, &url, body)
+                .await?
+        };
         Ok(ProviderOcrRequest {
             model,
             config,
             url,
             body,
+            optional_params,
             upstream_headers,
             timeout: request.timeout,
         })
@@ -165,6 +186,63 @@ impl OcrLifecycleHooks {
     }
 }
 
+async fn upload_reducto_document(
+    document: &Value,
+    api_base: Option<&str>,
+    timeout: Option<std::time::Duration>,
+    upstream_headers: &[(String, String)],
+) -> Result<Value, Error> {
+    let source = extract_document_source(document)?;
+    let Some(authorization) = upstream_headers
+        .iter()
+        .find(|(name, _)| name.eq_ignore_ascii_case("authorization"))
+        .map(|(_, value)| value.as_str())
+    else {
+        return Err(Error::Auth(
+            "Reducto upload requires an Authorization header".to_string(),
+        ));
+    };
+    let Some(upload) = build_upload_request(source, authorization, api_base) else {
+        return Ok(document.clone());
+    };
+    let part = reqwest::multipart::Part::bytes(upload.bytes)
+        .file_name(upload.file_name)
+        .mime_str(&upload.mime_type)
+        .map_err(|error| Error::InvalidRequest(error.to_string()))?;
+    let form = reqwest::multipart::Form::new().part("file", part);
+    let mut request_builder = http_client().post(upload.url).multipart(form);
+    for (name, value) in upstream_headers {
+        if !name.eq_ignore_ascii_case("content-type")
+            && !name.eq_ignore_ascii_case("content-length")
+        {
+            request_builder = request_builder.header(name, value);
+        }
+    }
+    if let Some(timeout) = timeout {
+        request_builder = request_builder.timeout(timeout);
+    }
+    let response = request_builder
+        .send()
+        .await
+        .map_err(|error| Error::Network(error.to_string()))?;
+    let status = response.status();
+    let body = response
+        .text()
+        .await
+        .map_err(|error| Error::Network(error.to_string()))?;
+    if !status.is_success() {
+        return Err(Error::Http {
+            status: status.as_u16(),
+            body: truncate_error_body(&body),
+        });
+    }
+    let response_json: Value = serde_json::from_str(&body).map_err(|error| {
+        Error::InvalidResponse(format!("invalid Reducto upload response JSON: {error}"))
+    })?;
+    let file_id = extract_upload_file_id(&response_json)?;
+    Ok(json!({"type": "document_url", "document_url": file_id}))
+}
+
 impl CallLifecycleHooks<PreparedOcrRequest, PreparedOcrRequest, Value> for OcrLifecycleHooks {
     type PreCallFuture<'a> = OcrFuture<'a, PreparedOcrRequest>;
     type DuringCallFuture<'a> = OcrFuture<'a, PreparedOcrRequest>;

litellm-rust/crates/ai-gateway/src/ocr/mod.rs

@@ -24,4 +24,151 @@ pub async fn ocr(request: OcrRequest<'_>) -> Result<Value, Error> {
 }
 
 #[cfg(test)]
-mod tests;
+mod tests {
+    use serde_json::{Map, json};
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::{TcpListener, TcpStream};
+
+    use super::{OcrRequest, ocr};
+    use crate::integrations::types::RequestMetadata;
+
+    async fn read_http_request(socket: &mut TcpStream) -> String {
+        let mut request = Vec::new();
+        let mut buffer = [0_u8; 1024];
+        let header_end = loop {
+            let n = socket.read(&mut buffer).await.expect("reads request");
+            if n == 0 {
+                break request.len();
+            }
+            request.extend_from_slice(&buffer[..n]);
+            if let Some(position) = request.windows(4).position(|window| window == b"\r\n\r\n") {
+                break position + 4;
+            }
+        };
+        let headers = String::from_utf8_lossy(&request[..header_end]);
+        let content_length = headers
+            .lines()
+            .find_map(|line| {
+                let (name, value) = line.split_once(':')?;
+                name.eq_ignore_ascii_case("content-length")
+                    .then(|| value.trim().parse::<usize>().ok())
+                    .flatten()
+            })
+            .unwrap_or(0);
+        while request.len().saturating_sub(header_end) < content_length {
+            let n = socket.read(&mut buffer).await.expect("reads body");
+            if n == 0 {
+                break;
+            }
+            request.extend_from_slice(&buffer[..n]);
+        }
+        String::from_utf8(request).expect("request is utf8")
+    }
+
+    fn base_ocr_request(model: &str) -> OcrRequest<'_> {
+        OcrRequest {
+            model,
+            document: json!({
+                "type": "document_url",
+                "document_url": "https://example.com/doc.pdf"
+            }),
+            api_key: Some("sk-test"),
+            api_base: None,
+            custom_llm_provider: None,
+            extra_headers: None,
+            optional_params: Map::new(),
+            timeout: None,
+            callbacks: Vec::new(),
+            guardrails: Vec::new(),
+            request_metadata: RequestMetadata::default(),
+            litellm_call_id: None,
+        }
+    }
+
+    #[tokio::test]
+    async fn reducto_file_upload_then_parse_maps_response() {
+        let listener = TcpListener::bind("127.0.0.1:0")
+            .await
+            .expect("test listener binds");
+        let address = listener.local_addr().expect("listener has local address");
+        let server = tokio::spawn(async move {
+            let (mut upload_socket, _) = listener.accept().await.expect("accepts upload request");
+            let upload_request = read_http_request(&mut upload_socket).await;
+            let upload_body = r#"{"file_id":"reducto://uploaded.pdf"}"#;
+            let upload_response = format!(
+                "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
+                upload_body.len(),
+                upload_body
+            );
+            upload_socket
+                .write_all(upload_response.as_bytes())
+                .await
+                .expect("writes upload response");
+
+            let (mut parse_socket, _) = listener.accept().await.expect("accepts parse request");
+            let parse_request = read_http_request(&mut parse_socket).await;
+            let parse_body = r#"{"job_id":"job_123","usage":{"num_pages":3,"credits":3},"result":{"chunks":[{"content":"Page 1 block A","blocks":[{"content":"Page 1 block A","bbox":{"page":1},"kind":"text"}]},{"content":"Page 2 block A","blocks":[{"content":"Page 2 block A","bbox":{"page":2},"kind":"table"}]},{"content":"Page 1 block B","blocks":[{"content":"Page 1 block B","bbox":{"page":1},"kind":"text"}]},{"content":"Page 3 block A","blocks":[{"content":"Page 3 block A","bbox":{"page":3},"kind":"figure"}]}]}}"#;
+            let parse_response = format!(
+                "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
+                parse_body.len(),
+                parse_body
+            );
+            parse_socket
+                .write_all(parse_response.as_bytes())
+                .await
+                .expect("writes parse response");
+            (upload_request, parse_request)
+        });
+        let api_base = format!("http://{address}");
+        let mut request = base_ocr_request("reducto/parse-v3");
+        request.api_base = Some(&api_base);
+        request.api_key = None;
+        request.extra_headers = Some(Map::from_iter([
+            ("Authorization".to_string(), json!("Bearer test-key")),
+            ("x-trace-id".to_string(), json!("trace-1")),
+        ]));
+        request.document = json!({
+            "type": "document_url",
+            "document_url": "data:application/pdf;base64,JVBERi0xLjQ="
+        });
+        request.optional_params = Map::from_iter([
+            (
+                "formatting".to_string(),
+                json!({"table_output_format": "html"}),
+            ),
+            ("retrieval".to_string(), json!({"chunk_mode": "section"})),
+            ("settings".to_string(), json!({"ocr_system": "standard"})),
+        ]);
+
+        let response = ocr(request).await.expect("Reducto OCR succeeds");
+
+        assert_eq!(response["pages"].as_array().map(Vec::len), Some(3));
+        assert_eq!(
+            response["pages"][0]["markdown"],
+            "Page 1 block A\n\nPage 1 block B"
+        );
+        assert_eq!(response["pages"][1]["markdown"], "Page 2 block A");
+        assert_eq!(response["pages"][2]["markdown"], "Page 3 block A");
+        assert_eq!(response["usage_info"]["pages_processed"], 3);
+        assert_eq!(response["usage_info"]["credits"], 3);
+        assert_eq!(response["provider_native_response"]["job_id"], "job_123");
+        let (upload_request, parse_request) = server.await.expect("server task completes");
+        assert!(
+            upload_request
+                .to_ascii_lowercase()
+                .contains("authorization: bearer test-key")
+        );
+        assert!(upload_request.contains("application/pdf"));
+        assert!(upload_request.contains("%PDF-1.4"));
+        assert!(upload_request.contains("x-trace-id: trace-1"));
+        assert!(
+            parse_request
+                .to_ascii_lowercase()
+                .contains("authorization: bearer test-key")
+        );
+        assert!(parse_request.contains(r#""input":"reducto://uploaded.pdf""#));
+        assert!(parse_request.contains(r#""table_output_format":"html""#));
+        assert!(parse_request.contains(r#""chunk_mode":"section""#));
+        assert!(parse_request.contains(r#""ocr_system":"standard""#));
+    }
+}

litellm-rust/crates/ai-gateway/src/ocr/prepare.rs

@@ -2,6 +2,7 @@ use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::{SystemTime, UNIX_EPOCH};
 
 use litellm_core::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider};
+use serde_json::{Map, Value};
 
 use super::common_utils::ocr_provider_config;
 use super::hooks::OcrLifecycleHooks;
@@ -28,17 +29,33 @@ pub(crate) fn prepare_ocr_call(request: OcrRequest<'_>) -> PreparedOcrCall {
     let model = provider_info.model.to_string();
     let custom_llm_provider = provider_info.custom_llm_provider.to_string();
     let config = ocr_provider_config(&custom_llm_provider, &model)
-        .ok_or_else(|| litellm_core::Error::InvalidProvider(custom_llm_provider.clone()));
+        .ok_or_else(|| litellm_core::Error::InvalidProvider(custom_llm_provider.clone()))
+        .and_then(|config| {
+            validate_request_format(config, &request.optional_params, &custom_llm_provider)?;
+            Ok(config)
+        });
     let optional_params = match &config {
         Ok(config) => {
             let supported = config.supported_ocr_params();
-            config.map_ocr_params(
+            let mut mapped = config.map_ocr_params(
                 &request
                     .optional_params
-                    .into_iter()
+                    .iter()
                     .filter(|(name, _)| supported.contains(&name.as_str()))
+                    .map(|(name, value)| (name.clone(), value.clone()))
                     .collect(),
-            )
+            );
+            for name in [
+                "vertex_project",
+                "vertex_ai_project",
+                "vertex_location",
+                "vertex_ai_location",
+            ] {
+                if let Some(value) = request.optional_params.get(name) {
+                    mapped.insert(name.to_string(), value.clone());
+                }
+            }
+            mapped
         }
         Err(_) => request.optional_params,
     };
@@ -64,6 +81,26 @@ pub(crate) fn prepare_ocr_call(request: OcrRequest<'_>) -> PreparedOcrCall {
     }
 }
 
+fn validate_request_format(
+    config: &'static dyn litellm_core::ocr::transformation::OcrProviderConfig,
+    optional_params: &Map<String, Value>,
+    provider: &str,
+) -> Result<(), litellm_core::Error> {
+    let Some(format) = optional_params.get("req_format") else {
+        return Ok(());
+    };
+    match format.as_str() {
+        Some("litellm") => Ok(()),
+        Some("native") if config.supported_ocr_params().contains(&"req_format") => Ok(()),
+        Some("native") => Err(litellm_core::Error::InvalidRequest(format!(
+            "`req_format=native` is not supported for provider {provider}"
+        ))),
+        _ => Err(litellm_core::Error::InvalidRequest(format!(
+            "Invalid `req_format`: {format}. Expected `litellm` or `native`"
+        ))),
+    }
+}
+
 fn new_ocr_call_id() -> String {
     static COUNTER: AtomicU64 = AtomicU64::new(1);
     let sequence = COUNTER.fetch_add(1, Ordering::Relaxed);
@@ -73,3 +110,54 @@ fn new_ocr_call_id() -> String {
         .unwrap_or(0);
     format!("ocr-{timestamp}-{sequence}")
 }
+
+#[cfg(test)]
+mod tests {
+    use litellm_core::error::Error;
+    use serde_json::{Map, json};
+
+    use super::{OcrRequest, prepare_ocr_call};
+    use crate::integrations::types::RequestMetadata;
+
+    fn base_ocr_request(model: &str) -> OcrRequest<'_> {
+        OcrRequest {
+            model,
+            document: json!({
+                "type": "document_url",
+                "document_url": "https://example.com/doc.pdf"
+            }),
+            api_key: Some("sk-test"),
+            api_base: None,
+            custom_llm_provider: None,
+            extra_headers: None,
+            optional_params: Map::new(),
+            timeout: None,
+            callbacks: Vec::new(),
+            guardrails: Vec::new(),
+            request_metadata: RequestMetadata::default(),
+            litellm_call_id: None,
+        }
+    }
+
+    fn request_with_format(format: &str) -> OcrRequest<'_> {
+        let mut request = base_ocr_request("mistral/mistral-ocr-latest");
+        request.optional_params = Map::from_iter([("req_format".to_string(), json!(format))]);
+        request
+    }
+
+    #[test]
+    fn native_format_rejected_for_provider_without_support_as_bad_request() {
+        let prepared = prepare_ocr_call(request_with_format("native"));
+        assert!(
+            matches!(prepared.request.config, Err(Error::InvalidRequest(message)) if message.contains("not supported for provider"))
+        );
+    }
+
+    #[test]
+    fn unknown_format_rejected_for_provider_without_support_as_bad_request() {
+        let prepared = prepare_ocr_call(request_with_format("raw"));
+        assert!(
+            matches!(prepared.request.config, Err(Error::InvalidRequest(message)) if message.contains("Invalid `req_format`"))
+        );
+    }
+}

litellm-rust/crates/ai-gateway/src/ocr/types.rs

@@ -53,6 +53,7 @@ pub(crate) struct ProviderOcrRequest {
     pub(crate) config: &'static dyn OcrProviderConfig,
     pub(crate) url: String,
     pub(crate) body: Value,
+    pub(crate) optional_params: Map<String, Value>,
     pub(crate) upstream_headers: Vec<(String, String)>,
     pub(crate) timeout: Option<Duration>,
 }

litellm-rust/crates/ai-gateway/src/python/AGENTS.md

@@ -1,27 +0,0 @@
-# ai-gateway/src/python — Python interop (load-time only)
-
-Functions here embed the Python interpreter (pyo3) and take the GIL to call into
-`litellm` (e.g. read the proxy `model_list`). Compiled only under the
-`python-config` feature.
-
-## Hard rule: non-hot-path functions only
-
-Everything in this folder MUST run **at most once per process lifetime — at
-startup / load time** (config read, warm-up). NEVER call into Python on the
-request path:
-
-- No GIL acquisition per request, per connection, or per realtime event.
-- No Python call inside a route handler, the router's hot path, or any loop that
-  scales with traffic.
-
-**Why:** the GIL serializes execution and would cap throughput; the realtime data
-path must stay pure Rust. Every acquisition is recorded by `crate::gil` — poll
-`GET /health/gil`, and `total_acquisitions` MUST stay flat under load.
-
-## How to add one
-
-Resolve whatever Python-derived data you need **once at boot** and hand the rest
-of the gateway an owned, plain-Rust value (e.g. build a `Router` from the
-resolved `model_list`). Record the acquisition via `crate::gil::record_acquisition()`
-immediately before taking the GIL. If a function would need to run per request,
-it does not belong here — move the work to Rust, or pre-resolve it at startup.

litellm-rust/crates/ai-gateway/src/python/config.rs

@@ -1,37 +0,0 @@
-//! Build the router by calling the Python proxy config reader (load time only).
-//!
-//! Embeds the interpreter via pyo3 and calls
-//! `litellm.proxy.read_model_list.read_model_list`, which reuses the proxy's
-//! `os.environ/` + secret-manager resolution. The GIL is taken **once at boot**
-//! (and recorded in [`crate::gil`]); the realtime hot path never touches Python.
-//!
-//! Compiled only under the `python-config` feature.
-use litellm_core::error::Error;
-use litellm_core::router::{Deployment, Router};
-use pyo3::prelude::*;
-
-use crate::gil;
-
-/// Load the router's `model_list` from `config_path` via the Python reader.
-pub fn load_router_from_config(config_path: &str) -> Result<Router, Error> {
-    gil::record_acquisition();
-    Python::attach(|py| {
-        let model_list = py
-            .import("litellm.proxy.read_model_list")
-            .and_then(|module| module.getattr("read_model_list"))
-            .and_then(|reader| reader.call1((config_path,)))
-            .map_err(|err| Error::Routing(format!("read_model_list failed: {err}")))?;
-
-        let model_list_json: String = py
-            .import("json")
-            .and_then(|json| json.getattr("dumps"))
-            .and_then(|dumps| dumps.call1((model_list,)))
-            .and_then(|encoded| encoded.extract())
-            .map_err(|err| Error::Routing(format!("serializing model_list failed: {err}")))?;
-
-        let deployments: Vec<Deployment> = serde_json::from_str(&model_list_json)
-            .map_err(|err| Error::Routing(format!("parsing model_list failed: {err}")))?;
-
-        Ok(Router::new(deployments))
-    })
-}

litellm-rust/crates/ai-gateway/src/python/mod.rs

@@ -1,4 +0,0 @@
-//! Python interop for the gateway. See `AGENTS.md`: **load-time / non-hot-path
-//! only.** Compiled only under the `python-config` feature.
-
-pub mod config;

litellm-rust/crates/ai-gateway/src/realtime/streaming.rs

@@ -106,16 +106,16 @@ impl RealTimeStreaming {
     /// `litellm_call_id`, replacing the gateway-generated fallback.
     fn on_session(&mut self, event: &RealtimeEvent) {
         let session = event.data.get("session").and_then(Value::as_object);
-        if let Some(id) = session.and_then(|s| s.get("id")).and_then(Value::as_str) {
-            if !id.is_empty() {
-                self.id = id.to_string();
-                self.litellm_call_id = id.to_string();
-            }
+        if let Some(id) = session.and_then(|s| s.get("id")).and_then(Value::as_str)
+            && !id.is_empty()
+        {
+            self.id = id.to_string();
+            self.litellm_call_id = id.to_string();
         }
-        if let Some(model) = session.and_then(|s| s.get("model")).and_then(Value::as_str) {
-            if !model.is_empty() {
-                self.model = model.to_string();
-            }
+        if let Some(model) = session.and_then(|s| s.get("model")).and_then(Value::as_str)
+            && !model.is_empty()
+        {
+            self.model = model.to_string();
         }
     }
 
@@ -323,6 +323,32 @@ mod tests {
         assert_eq!(streaming.dropped(), 0);
     }
 
+    #[test]
+    fn blank_session_id_and_model_keep_the_gateway_fallbacks() {
+        let mut streaming = RealTimeStreaming::new(
+            Vec::new(),
+            "call_fallback".to_string(),
+            "gpt-realtime".to_string(),
+            RequestMetadata::default(),
+        );
+
+        streaming.observe(&event(
+            r#"{"type":"session.created","session":{"id":"","model":""}}"#,
+        ));
+        let payload = streaming.build_payload();
+        assert_eq!(payload.id, "call_fallback");
+        assert_eq!(payload.litellm_call_id, "call_fallback");
+        assert_eq!(payload.model, "gpt-realtime");
+
+        streaming.observe(&event(
+            r#"{"type":"session.updated","session":{"id":"sess_002","model":""}}"#,
+        ));
+        let payload = streaming.build_payload();
+        assert_eq!(payload.id, "sess_002");
+        assert_eq!(payload.litellm_call_id, "sess_002");
+        assert_eq!(payload.model, "gpt-realtime");
+    }
+
     #[test]
     fn payload_serializes_with_camelcase_times_and_realtime_call_type() {
         let mut streaming = RealTimeStreaming::new(

litellm-rust/crates/ai-gateway/src/routes/AGENTS.md

@@ -13,7 +13,7 @@ private). This is the norm — don't split until it hurts.
 pub fn router() -> Router<AppState> { Router::new().route(PATH, get(handle)) }
 async fn handle(...) -> impl IntoResponse { ... }
 ```
-`health.rs` and `gil.rs` are examples.
+`health.rs` is the example.
 
 ## Split out `service` when there's real logic
 When a route has business logic worth testing without axum, put it in a sibling

litellm-rust/crates/ai-gateway/src/routes/gil.rs

@@ -1,30 +0,0 @@
-//! `GET /health/gil` — poll to confirm Python is only touched at load time.
-//! Simple-route template: a `router()` plus its handler, in one file.
-
-use axum::routing::get;
-use axum::{Json, Router};
-use serde::Serialize;
-
-use crate::gil;
-use crate::state::AppState;
-
-/// This route's contribution to the app router.
-pub fn router() -> Router<AppState> {
-    Router::new().route("/health/gil", get(status))
-}
-
-#[derive(Debug, Serialize)]
-struct GilStatusResponse {
-    gil_acquired_last_30s: bool,
-    total_acquisitions: u64,
-    seconds_since_last: Option<u64>,
-}
-
-async fn status() -> Json<GilStatusResponse> {
-    let snapshot = gil::snapshot();
-    Json(GilStatusResponse {
-        gil_acquired_last_30s: snapshot.acquired_last_30s,
-        total_acquisitions: snapshot.total_acquisitions,
-        seconds_since_last: snapshot.seconds_since_last,
-    })
-}

litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs

@@ -21,6 +21,12 @@ pub fn router() -> Router<AppState> {
     Router::new().route(MESSAGES_ROUTE_PATH, post(handle))
 }
 
+#[tracing::instrument(
+    name = "messages_gateway_route",
+    target = "litellm::function_trace",
+    level = "trace",
+    skip_all
+)]
 async fn handle(
     _auth: RequireMasterKey,
     State(state): State<AppState>,

litellm-rust/crates/ai-gateway/src/routes/messages/service.rs

@@ -12,6 +12,12 @@ pub(crate) enum MessagesResponse {
     Stream(reqwest::Response),
 }
 
+#[tracing::instrument(
+    name = "messages_gateway_service",
+    target = "litellm::function_trace",
+    level = "trace",
+    skip_all
+)]
 pub async fn run(
     router: &Arc<Router>,
     body: Value,

litellm-rust/crates/ai-gateway/src/routes/mod.rs

@@ -2,10 +2,9 @@
 //!
 //! **Template:** every route module exposes `pub fn router() -> Router<AppState>`
 //! that mounts its own paths; [`app`] merges them. A trivial route is a single
-//! file (`health.rs`, `gil.rs`); a non-trivial one is a folder (`realtime/`) with
+//! file (`health.rs`); a non-trivial one is a folder (`realtime/`) with
 //! `handler` (entry) + `service` (logic) + `transport` (adapters). See AGENTS.md.
 
-pub mod gil;
 pub mod health;
 pub mod messages;
 pub mod realtime;
@@ -19,7 +18,6 @@ use crate::state::AppState;
 pub fn app(state: AppState) -> Router {
     Router::new()
         .merge(health::router())
-        .merge(gil::router())
         .merge(messages::router())
         .merge(realtime::router())
         .merge(responses::router())

litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs

@@ -50,18 +50,17 @@ where
         provider_model,
         params.api_key.as_deref(),
         params.api_base.as_deref(),
-    ) {
-        if let Some(handoff) = pool.take(&key) {
-            return crate::io::realtime::realtime_warm(
-                provider_model,
-                handoff,
-                idle_timeout,
-                observe,
-                client_in,
-                client_out,
-            )
-            .await;
-        }
+    ) && let Some(handoff) = pool.take(&key)
+    {
+        return crate::io::realtime::realtime_warm(
+            provider_model,
+            handoff,
+            idle_timeout,
+            observe,
+            client_in,
+            client_out,
+        )
+        .await;
     }
 
     // Cold path: fresh dial (the original behavior).

litellm-rust/crates/ai-gateway/src/trace_parity.rs

@@ -0,0 +1,65 @@
+//! Harness-only in-process adapters. Never mounted as production routes.
+
+use std::sync::Arc;
+
+use axum::body::{Body, to_bytes};
+use axum::http::header::{AUTHORIZATION, CONTENT_TYPE};
+use axum::http::{Request, StatusCode};
+use litellm_core::Error;
+use litellm_core::router::{Deployment, LiteLLMParams, Router as ModelRouter};
+use serde::Serialize;
+use serde_json::Value;
+use tower::ServiceExt;
+
+use crate::io::realtime_pool::RealtimePool;
+use crate::routes;
+use crate::state::AppState;
+
+#[derive(Debug, Serialize)]
+pub struct GatewayResponse {
+    pub status: u16,
+    pub body: Value,
+}
+
+pub async fn messages_request(
+    model_alias: String,
+    provider_model: String,
+    api_base: String,
+    body: Value,
+) -> Result<GatewayResponse, Error> {
+    let state = AppState {
+        router: Arc::new(ModelRouter::new(vec![Deployment {
+            model_name: model_alias,
+            litellm_params: LiteLLMParams {
+                model: provider_model,
+                api_key: Some("trace-provider-key".to_string()),
+                api_base: Some(api_base),
+            },
+        }])),
+        master_key: Some(Arc::from("trace-master-key")),
+        loggers: Arc::new(Vec::new()),
+        realtime_pool: RealtimePool::disabled(),
+    };
+    let request = Request::builder()
+        .method("POST")
+        .uri("/v1/messages")
+        .header(AUTHORIZATION, "Bearer trace-master-key")
+        .header(CONTENT_TYPE, "application/json")
+        .body(Body::from(body.to_string()))
+        .map_err(|error| Error::InvalidRequest(error.to_string()))?;
+    let response = routes::app(state)
+        .oneshot(request)
+        .await
+        .map_err(|error| match error {})?;
+    let status: StatusCode = response.status();
+    let bytes = to_bytes(response.into_body(), usize::MAX)
+        .await
+        .map_err(|error| Error::InvalidResponse(error.to_string()))?;
+    let body = serde_json::from_slice(&bytes).map_err(|error| {
+        Error::InvalidResponse(format!("gateway returned invalid JSON: {error}"))
+    })?;
+    Ok(GatewayResponse {
+        status: status.as_u16(),
+        body,
+    })
+}

litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs

@@ -1,23 +1,19 @@
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
-use litellm_core::error::Error;
-use litellm_core::http_utils::has_header;
-use litellm_core::ocr::transformation::OcrResponseHandling;
-use serde_json::{Map, Value, json};
-use tokio::io::{AsyncReadExt, AsyncWriteExt};
-use tokio::net::{TcpListener, TcpStream};
-
-use super::common_utils::{ocr_provider_config, string_headers, truncate_error_body};
-use super::{OcrRequest, ocr};
-use crate::integrations::custom_guardrail::{
+use litellm_ai_gateway::integrations::custom_guardrail::{
     CustomGuardrail, GuardrailContext, GuardrailDecision, GuardrailError, GuardrailEventHook,
     GuardrailFuture, GuardrailRequest,
 };
-use crate::integrations::custom_logger::{
+use litellm_ai_gateway::integrations::custom_logger::{
     CallbackTiming, CallbackValue, CustomLogger, LogFuture, ModelCallDetails,
 };
-use crate::integrations::types::RequestMetadata;
+use litellm_ai_gateway::integrations::types::RequestMetadata;
+use litellm_ai_gateway::ocr::{OcrRequest, ocr};
+use litellm_core::error::Error;
+use serde_json::{Map, Value, json};
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+use tokio::net::{TcpListener, TcpStream};
 
 async fn read_http_headers(socket: &mut TcpStream) -> String {
     let mut request = Vec::new();
@@ -136,6 +132,7 @@ struct RecordingOcrGuardrail {
     hooks: Vec<GuardrailEventHook>,
     events: Mutex<Vec<&'static str>>,
     block_pre_call: bool,
+    block_during_call: bool,
 }
 
 impl RecordingOcrGuardrail {
@@ -144,6 +141,7 @@ impl RecordingOcrGuardrail {
             hooks,
             events: Mutex::new(Vec::new()),
             block_pre_call: false,
+            block_during_call: false,
         }
     }
 
@@ -152,6 +150,16 @@ impl RecordingOcrGuardrail {
             hooks: vec![GuardrailEventHook::PreCall],
             events: Mutex::new(Vec::new()),
             block_pre_call: true,
+            block_during_call: false,
+        }
+    }
+
+    fn blocking_during_call() -> Self {
+        Self {
+            hooks: vec![GuardrailEventHook::DuringCall],
+            events: Mutex::new(Vec::new()),
+            block_pre_call: false,
+            block_during_call: true,
         }
     }
 
@@ -193,91 +201,95 @@ impl CustomGuardrail for RecordingOcrGuardrail {
     ) -> GuardrailFuture<'a> {
         Box::pin(async move {
             self.events.lock().unwrap().push("async_moderation_hook");
+            if self.block_during_call {
+                return Ok(GuardrailDecision::Block(GuardrailError::blocked(
+                    "blocked before provider",
+                )));
+            }
             request.data["body"]["guarded_during"] = json!(true);
             Ok(GuardrailDecision::Mask(request))
         })
     }
 }
 
-#[test]
-fn truncate_error_body_passes_short_strings_through() {
-    let body = "Unauthorized";
-    assert_eq!(truncate_error_body(body), "Unauthorized");
+fn base_ocr_request(model: &str) -> OcrRequest<'_> {
+    OcrRequest {
+        model,
+        document: json!({
+            "type": "document_url",
+            "document_url": "https://example.com/doc.pdf"
+        }),
+        api_key: Some("sk-test"),
+        api_base: None,
+        custom_llm_provider: None,
+        extra_headers: None,
+        optional_params: Map::new(),
+        timeout: None,
+        callbacks: Vec::new(),
+        guardrails: Vec::new(),
+        request_metadata: RequestMetadata::default(),
+        litellm_call_id: None,
+    }
 }
 
-#[test]
-fn truncate_error_body_caps_long_payloads() {
-    let body = "x".repeat(306);
-    let truncated = truncate_error_body(&body);
+#[tokio::test]
+async fn reducto_during_call_guardrail_blocks_before_upload() {
+    let listener = TcpListener::bind("127.0.0.1:0")
+        .await
+        .expect("test listener binds");
+    let address = listener.local_addr().expect("listener has local address");
+    let api_base = format!("http://{address}");
+    let guardrail = Arc::new(RecordingOcrGuardrail::blocking_during_call());
+    let mut request = base_ocr_request("reducto/parse-v3");
+    request.api_base = Some(&api_base);
+    request.document = json!({
+        "type": "document_url",
+        "document_url": "data:application/pdf;base64,JVBERi0xLjQ="
+    });
+    request.guardrails = vec![guardrail.clone()];
 
-    assert!(truncated.ends_with("... (truncated)"));
-    let prefix_chars = truncated
-        .strip_suffix("... (truncated)")
-        .expect("truncated marker present")
-        .chars()
-        .count();
-    assert_eq!(prefix_chars, 256);
+    let error = ocr(request).await.expect_err("guardrail blocks upload");
+
+    assert!(matches!(error, Error::InvalidRequest(_)));
+    assert_eq!(guardrail.events(), vec!["async_moderation_hook"]);
+    let accepted = tokio::time::timeout(Duration::from_millis(100), listener.accept()).await;
+    assert!(accepted.is_err(), "upload socket should not be touched");
 }
 
-#[test]
-fn truncate_error_body_does_not_split_multibyte_chars() {
-    let body = "é".repeat(266);
-    let truncated = truncate_error_body(&body);
-    assert!(truncated.is_char_boundary(truncated.len()));
-}
+#[tokio::test]
+async fn reducto_upload_error_body_is_truncated() {
+    let listener = TcpListener::bind("127.0.0.1:0")
+        .await
+        .expect("test listener binds");
+    let address = listener.local_addr().expect("listener has local address");
+    let server = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.expect("accepts upload request");
+        let _request = read_http_request(&mut socket).await;
+        let body = "x".repeat(300);
+        let response = format!(
+            "HTTP/1.1 500 Internal Server Error\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
+            body.len(),
+            body
+        );
+        socket
+            .write_all(response.as_bytes())
+            .await
+            .expect("writes upload response");
+    });
+    let api_base = format!("http://{address}");
+    let mut request = base_ocr_request("reducto/parse-v3");
+    request.api_base = Some(&api_base);
+    request.document = json!({
+        "type": "document_url",
+        "document_url": "data:application/pdf;base64,JVBERi0xLjQ="
+    });
+
+    let error = ocr(request).await.expect_err("upload should fail");
 
-#[test]
-fn ocr_dispatch_supports_migrated_providers() {
-    assert!(ocr_provider_config("mistral", "mistral-ocr-latest").is_some());
     assert!(
-        ocr_provider_config("azure_ai", "pixtral-12b-2409")
-            .expect("azure ai config resolves")
-            .requires_data_uri_document()
+        matches!(error, Error::Http { status: 500, body } if body.chars().count() < 300 && body.ends_with("... (truncated)"))
     );
-    assert_eq!(
-        ocr_provider_config("azure_ai", "doc-intelligence/prebuilt-read")
-            .expect("document intelligence config resolves")
-            .response_handling(),
-        OcrResponseHandling::AzureDocumentIntelligencePoll
-    );
-    assert!(
-        ocr_provider_config("vertex_ai", "deepseek-ocr-maas")
-            .expect("vertex deepseek config resolves")
-            .supported_ocr_params()
-            .contains(&"temperature")
-    );
-    assert!(ocr_provider_config("openai", "gpt-4o").is_none());
-}
-
-#[test]
-fn string_headers_accepts_string_values() {
-    let headers = json!({
-        "x-trace-id": "trace-1"
-    })
-    .as_object()
-    .unwrap()
-    .clone();
-
-    assert_eq!(
-        string_headers(Some(headers)).expect("string headers accepted"),
-        vec![("x-trace-id".to_string(), "trace-1".to_string())]
-    );
-}
-
-#[test]
-fn auth_header_detection_is_case_insensitive() {
-    let headers = vec![
-        ("x-trace-id".to_string(), "trace-1".to_string()),
-        ("authorization".to_string(), "Bearer sk-test".to_string()),
-    ];
-
-    assert!(has_header(&headers, "authorization"));
-
-    let headers = vec![("Authorization".to_string(), "Bearer sk-test".to_string())];
-    assert!(has_header(&headers, "authorization"));
-
-    let headers = vec![("x-trace-id".to_string(), "trace-1".to_string())];
-    assert!(!has_header(&headers, "authorization"));
+    server.await.expect("server task completes");
 }
 
 #[tokio::test]
@@ -595,21 +607,3 @@ async fn document_intelligence_poll_uses_resolved_subscription_key() {
         "{poll_request}"
     );
 }
-
-#[test]
-fn string_headers_rejects_non_string_values() {
-    let headers = json!({
-        "x-retry-count": 3
-    })
-    .as_object()
-    .unwrap()
-    .clone();
-
-    let err = string_headers(Some(headers)).expect_err("non-string header rejected");
-    assert_eq!(
-        err,
-        Error::InvalidRequest(
-            "OCR extra_headers.x-retry-count must be a string, got number".to_string()
-        )
-    );
-}

litellm-rust/crates/config/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "litellm-config"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-core.workspace = true
+pyo3 = { workspace = true, features = ["auto-initialize"], optional = true }
+serde_json.workspace = true
+thiserror.workspace = true
+
+[features]
+default = []
+python = ["dep:pyo3"]

litellm-rust/crates/config/src/error.rs

@@ -0,0 +1,11 @@
+use thiserror::Error as ThisError;
+
+#[derive(Debug, ThisError)]
+pub enum Error {
+    #[error("read_model_list failed: {0}")]
+    PythonLoading(String),
+    #[error("serializing model_list failed: {0}")]
+    Serialization(String),
+    #[error("parsing model_list failed: {0}")]
+    ModelListParsing(#[source] serde_json::Error),
+}

litellm-rust/crates/config/src/lib.rs

@@ -0,0 +1,7 @@
+mod error;
+#[cfg(feature = "python")]
+mod python;
+
+pub use error::Error;
+#[cfg(feature = "python")]
+pub use python::load_model_list;

litellm-rust/crates/config/src/python.rs

@@ -0,0 +1,76 @@
+use std::path::Path;
+
+use litellm_core::router::Deployment;
+use pyo3::prelude::*;
+
+use crate::Error;
+
+pub fn load_model_list(config_path: &Path) -> Result<Vec<Deployment>, Error> {
+    Python::attach(|python| {
+        let model_list = python
+            .import("litellm.proxy.read_model_list")
+            .and_then(|module| module.getattr("read_model_list"))
+            .and_then(|reader| reader.call1((config_path.to_string_lossy().as_ref(),)))
+            .map_err(|error| Error::PythonLoading(error.to_string()))?;
+
+        let model_list_json = python
+            .import("json")
+            .and_then(|json| json.getattr("dumps"))
+            .and_then(|dumps| dumps.call1((model_list,)))
+            .and_then(|encoded| encoded.extract::<String>())
+            .map_err(|error| Error::Serialization(error.to_string()))?;
+
+        parse_model_list(&model_list_json)
+    })
+}
+
+fn parse_model_list(model_list_json: &str) -> Result<Vec<Deployment>, Error> {
+    serde_json::from_str(model_list_json).map_err(Error::ModelListParsing)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parses_resolved_model_list() {
+        let deployments = parse_model_list(
+            r#"[
+                {
+                    "model_name": "realtime",
+                    "litellm_params": {
+                        "model": "openai/gpt-realtime",
+                        "api_key": "resolved-secret",
+                        "api_base": "https://api.example.test/v1"
+                    }
+                },
+                {
+                    "model_name": "without-optional-values",
+                    "litellm_params": {"model": "openai/gpt-4.1"}
+                }
+            ]"#,
+        )
+        .expect("resolved model list should parse");
+
+        assert_eq!(deployments.len(), 2);
+        assert_eq!(deployments[0].model_name, "realtime");
+        assert_eq!(
+            deployments[0].litellm_params.api_key.as_deref(),
+            Some("resolved-secret")
+        );
+        assert_eq!(
+            deployments[0].litellm_params.api_base.as_deref(),
+            Some("https://api.example.test/v1")
+        );
+        assert_eq!(deployments[1].litellm_params.api_key, None);
+        assert_eq!(deployments[1].litellm_params.api_base, None);
+    }
+
+    #[test]
+    fn malformed_model_list_returns_parsing_error() {
+        let error = parse_model_list(r#"[{"model_name":"missing-params"}]"#)
+            .expect_err("missing litellm_params should fail");
+
+        assert!(matches!(error, Error::ModelListParsing(_)));
+    }
+}

litellm-rust/crates/core/Cargo.toml

@@ -6,12 +6,14 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+base64.workspace = true
 rand.workspace = true
 reqwest.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 thiserror.workspace = true
 tracing.workspace = true
+tracing-subscriber = { workspace = true, optional = true }
 sha2.workspace = true
 aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true }
 aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"], optional = true }
@@ -30,6 +32,9 @@ bedrock-auth = [
     "dep:aws-types",
     "dep:aws-smithy-runtime-api",
 ]
+observability = ["dep:tracing-subscriber"]
 
 [dev-dependencies]
+rstest.workspace = true
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
+tracing-subscriber.workspace = true

litellm-rust/crates/core/src/constants.rs

@@ -41,3 +41,5 @@ pub const CHAT_COMPLETION_OBJECT: &str = "chat.completion";
 /// `litellm/litellm_core_utils/prompt_templates/factory.py`.
 pub const EMPTY_TEXT_PLACEHOLDER: &str =
     "[System: Empty message content sanitised to satisfy protocol]";
+
+pub const FUNCTION_TRACE_TARGET: &str = "litellm::function_trace";

litellm-rust/crates/core/src/http_utils.rs

@@ -101,6 +101,23 @@ mod tests {
         assert!(!has_header(&headers, "authorization"));
     }
 
+    #[test]
+    fn auth_header_detection_is_case_insensitive() {
+        let headers = vec![
+            ("x-trace-id".to_string(), "trace-1".to_string()),
+            ("authorization".to_string(), "Bearer sk-test".to_string()),
+        ];
+
+        assert!(has_header(&headers, "authorization"));
+
+        let headers = vec![("Authorization".to_string(), "Bearer sk-test".to_string())];
+
+        assert!(has_header(&headers, "authorization"));
+
+        let headers = vec![("x-trace-id".to_string(), "trace-1".to_string())];
+        assert!(!has_header(&headers, "authorization"));
+    }
+
     #[test]
     fn bearer_detection_requires_a_non_empty_token() {
         assert!(has_bearer_auth(&[(

litellm-rust/crates/core/src/lib.rs

@@ -6,6 +6,8 @@ pub mod constants;
 pub mod error;
 pub mod http_utils;
 pub mod messages;
+#[cfg(any(feature = "observability", test))]
+pub mod observability;
 pub mod ocr;
 pub mod providers;
 pub mod realtime;

litellm-rust/crates/core/src/observability/function_trace.rs

@@ -0,0 +1,215 @@
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+
+use serde::Serialize;
+use tracing::span::{Attributes, Id};
+use tracing::{Dispatch, Subscriber};
+use tracing_subscriber::layer::Context;
+use tracing_subscriber::prelude::*;
+use tracing_subscriber::registry::LookupSpan;
+use tracing_subscriber::{Layer, Registry};
+
+use super::function_trace_filter;
+
+#[derive(Clone, Debug, PartialEq, Serialize)]
+pub struct FunctionTraceEvent {
+    pub id: usize,
+    pub parent_id: Option<usize>,
+    pub function: &'static str,
+    pub module_path: Option<&'static str>,
+    pub file: Option<&'static str>,
+    pub line: Option<u32>,
+}
+
+#[derive(Clone, Default)]
+pub struct FunctionTrace {
+    events: Arc<Mutex<Vec<FunctionTraceEvent>>>,
+    span_events: Arc<Mutex<HashMap<Id, usize>>>,
+}
+
+impl FunctionTrace {
+    pub fn dispatcher(&self) -> Dispatch {
+        Dispatch::new(
+            Registry::default().with(
+                FunctionTraceLayer {
+                    trace: self.clone(),
+                }
+                .with_filter(function_trace_filter()),
+            ),
+        )
+    }
+
+    pub fn events(&self) -> Vec<FunctionTraceEvent> {
+        self.events
+            .lock()
+            .unwrap_or_else(|error| error.into_inner())
+            .clone()
+    }
+}
+
+struct FunctionTraceLayer {
+    trace: FunctionTrace,
+}
+
+impl<S> Layer<S> for FunctionTraceLayer
+where
+    S: Subscriber + for<'lookup> LookupSpan<'lookup>,
+{
+    fn on_new_span(&self, attributes: &Attributes<'_>, id: &Id, context: Context<'_, S>) {
+        let parent_id = context.span(id).and_then(|span| {
+            let span_events = self
+                .trace
+                .span_events
+                .lock()
+                .unwrap_or_else(|error| error.into_inner());
+            span.scope()
+                .skip(1)
+                .find_map(|ancestor| span_events.get(&ancestor.id()).copied())
+        });
+        let mut events = self
+            .trace
+            .events
+            .lock()
+            .unwrap_or_else(|error| error.into_inner());
+        let event_id = events.len();
+        events.push(FunctionTraceEvent {
+            id: event_id,
+            parent_id,
+            function: attributes.metadata().name(),
+            module_path: attributes.metadata().module_path(),
+            file: attributes.metadata().file(),
+            line: attributes.metadata().line(),
+        });
+        self.trace
+            .span_events
+            .lock()
+            .unwrap_or_else(|error| error.into_inner())
+            .insert(id.clone(), event_id);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::constants::FUNCTION_TRACE_TARGET;
+
+    use super::*;
+
+    fn event(
+        id: usize,
+        parent_id: Option<usize>,
+        function: &'static str,
+    ) -> (usize, Option<usize>, &'static str) {
+        (id, parent_id, function)
+    }
+
+    fn structural_events(
+        events: &[FunctionTraceEvent],
+    ) -> Vec<(usize, Option<usize>, &'static str)> {
+        events
+            .iter()
+            .map(|event| (event.id, event.parent_id, event.function))
+            .collect()
+    }
+
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
+    async fn outer() {
+        tokio::task::yield_now().await;
+        inner().await;
+    }
+
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
+    async fn inner() {
+        tokio::task::yield_now().await;
+    }
+
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
+    async fn concurrent_parent() {
+        tokio::join!(inner(), inner());
+    }
+
+    #[tokio::test]
+    async fn concurrent_futures_keep_separate_traces_across_yields() {
+        use tracing::instrument::WithSubscriber;
+
+        let first = FunctionTrace::default();
+        let second = FunctionTrace::default();
+        let outside = FunctionTrace::default();
+
+        async {
+            tokio::join!(
+                outer().with_subscriber(first.dispatcher()),
+                inner().with_subscriber(second.dispatcher()),
+            );
+            inner().await;
+        }
+        .with_subscriber(outside.dispatcher())
+        .await;
+
+        assert_eq!(
+            structural_events(&first.events()),
+            vec![event(0, None, "outer"), event(1, Some(0), "inner")],
+        );
+        assert_eq!(
+            structural_events(&second.events()),
+            vec![event(0, None, "inner")],
+        );
+        assert_eq!(
+            structural_events(&outside.events()),
+            vec![event(0, None, "inner")],
+        );
+    }
+
+    #[tokio::test]
+    async fn concurrent_siblings_keep_the_same_parent() {
+        use tracing::instrument::WithSubscriber;
+
+        let trace = FunctionTrace::default();
+        concurrent_parent()
+            .with_subscriber(trace.dispatcher())
+            .await;
+
+        assert_eq!(
+            structural_events(&trace.events()),
+            vec![
+                event(0, None, "concurrent_parent"),
+                event(1, Some(0), "inner"),
+                event(2, Some(0), "inner"),
+            ]
+        );
+    }
+
+    #[test]
+    fn records_matching_spans_in_creation_order() {
+        let trace = FunctionTrace::default();
+        let dispatch = trace.dispatcher();
+
+        tracing::dispatcher::with_default(&dispatch, || {
+            let _ignored = tracing::trace_span!(target: "other", "ignored");
+            let _first = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name");
+            let _wrong_level = tracing::debug_span!(target: FUNCTION_TRACE_TARGET, "wrong_level");
+            let _second = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name");
+        });
+
+        assert_eq!(
+            structural_events(&trace.events()),
+            vec![event(0, None, "same_name"), event(1, None, "same_name")]
+        );
+    }
+
+    #[test]
+    fn records_matching_span_nesting_depth() {
+        let trace = FunctionTrace::default();
+        let dispatch = trace.dispatcher();
+
+        tracing::dispatcher::with_default(&dispatch, || {
+            let outer = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "outer");
+            let _outer_guard = outer.enter();
+            let _inner = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "inner");
+        });
+
+        assert_eq!(
+            structural_events(&trace.events()),
+            vec![event(0, None, "outer"), event(1, Some(0), "inner")]
+        );
+    }
+}

litellm-rust/crates/core/src/observability/mod.rs

@@ -0,0 +1,59 @@
+use tracing::span::Id;
+use tracing::{Level, Metadata, Subscriber};
+use tracing_subscriber::filter::{FilterFn, LevelFilter, filter_fn};
+use tracing_subscriber::layer::Context;
+use tracing_subscriber::registry::LookupSpan;
+
+use crate::constants::FUNCTION_TRACE_TARGET;
+
+pub mod function_trace;
+
+pub use function_trace::{FunctionTrace, FunctionTraceEvent};
+
+pub fn function_trace_filter() -> FilterFn<impl Fn(&Metadata<'_>) -> bool> {
+    filter_fn(|metadata| {
+        metadata.is_span()
+            && metadata.target() == FUNCTION_TRACE_TARGET
+            && *metadata.level() == Level::TRACE
+    })
+    .with_max_level_hint(LevelFilter::TRACE)
+}
+
+pub fn span_depth<S>(context: &Context<'_, S>, id: &Id) -> usize
+where
+    S: Subscriber + for<'lookup> LookupSpan<'lookup>,
+{
+    context
+        .span(id)
+        .map(|span| span.scope().skip(1).count())
+        .unwrap_or_default()
+}
+
+#[cfg(test)]
+mod tests {
+    use tracing::instrument::WithSubscriber;
+
+    use super::*;
+
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
+    async fn instrumented_with_literal_target() {}
+
+    #[tokio::test]
+    async fn literal_instrument_target_matches_filter_constant() {
+        assert_eq!(FUNCTION_TRACE_TARGET, "litellm::function_trace");
+
+        let trace = FunctionTrace::default();
+        instrumented_with_literal_target()
+            .with_subscriber(trace.dispatcher())
+            .await;
+
+        let events = trace.events();
+        assert_eq!(events.len(), 1);
+        assert_eq!(events[0].id, 0);
+        assert_eq!(events[0].parent_id, None);
+        assert_eq!(events[0].function, "instrumented_with_literal_target");
+        assert_eq!(events[0].module_path, Some(module_path!()));
+        assert_eq!(events[0].file, Some(file!()));
+        assert!(events[0].line.is_some());
+    }
+}

litellm-rust/crates/core/src/ocr/transformation.rs

@@ -51,6 +51,15 @@ pub trait OcrProviderConfig: Sync {
         response_json: Value,
     ) -> Result<OcrResponseData, Error>;
 
+    fn transform_ocr_response_with_params(
+        &self,
+        model: &str,
+        response_json: Value,
+        _optional_params: &Map<String, Value>,
+    ) -> Result<OcrResponseData, Error> {
+        self.transform_ocr_response(model, response_json)
+    }
+
     fn complete_url(
         &self,
         api_base: Option<&str>,

litellm-rust/crates/core/src/ocr/types.rs

@@ -1,5 +1,5 @@
 use serde::{Deserialize, Serialize};
-use serde_json::Value;
+use serde_json::{Map, Value};
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct OcrRequestData {
@@ -14,16 +14,25 @@ pub struct OcrResponseData {
     pub document_annotation: Option<Value>,
     pub usage_info: Option<Value>,
     pub object: String,
+    pub extra_fields: Map<String, Value>,
+    pub provider_native_response: Option<Value>,
 }
 
 impl OcrResponseData {
     pub fn into_json(self) -> Value {
-        serde_json::json!({
+        let mut response = serde_json::json!({
             "pages": self.pages,
             "model": self.model,
             "document_annotation": self.document_annotation,
             "usage_info": self.usage_info,
             "object": self.object,
-        })
+        });
+        if let Value::Object(object) = &mut response {
+            object.extend(self.extra_fields);
+            if let Some(native_response) = self.provider_native_response {
+                object.insert("provider_native_response".to_string(), native_response);
+            }
+        }
+        response
     }
 }

litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs

@@ -134,6 +134,8 @@ impl OcrProviderConfig for MistralOcrConfig {
             document_annotation,
             usage_info,
             object: "ocr".to_string(),
+            extra_fields: Map::new(),
+            provider_native_response: None,
         })
     }
 

litellm-rust/crates/core/src/providers/mod.rs

@@ -4,4 +4,5 @@ pub mod azure_ai;
 pub mod bedrock;
 pub mod mistral;
 pub mod openai;
+pub mod reducto;
 pub mod vertex_ai;

litellm-rust/crates/core/src/providers/reducto/mod.rs

@@ -0,0 +1 @@
+pub mod ocr;

litellm-rust/crates/core/src/providers/reducto/ocr/mod.rs

@@ -0,0 +1,4 @@
+pub mod transformation;
+
+#[cfg(test)]
+mod tests;

litellm-rust/crates/core/src/providers/reducto/ocr/tests.rs

@@ -0,0 +1,202 @@
+use rstest::{fixture, rstest};
+use serde_json::{Value, json};
+
+use super::transformation::*;
+use crate::ocr::transformation::OcrProviderConfig;
+
+#[fixture]
+fn parse_response() -> Value {
+    json!({
+        "job_id": "job_123",
+        "usage": {"num_pages": 3, "credits": 3},
+        "result": {
+            "chunks": [
+                {
+                    "content": "Page 1 block A",
+                    "blocks": [{
+                        "content": "Page 1 block A",
+                        "bbox": {"page": 1},
+                        "kind": "text",
+                    }],
+                },
+                {
+                    "content": "Page 2 block A",
+                    "blocks": [{
+                        "content": "Page 2 block A",
+                        "bbox": {"page": 2},
+                        "kind": "table",
+                    }],
+                },
+                {
+                    "content": "Page 1 block B",
+                    "blocks": [{
+                        "content": "Page 1 block B",
+                        "bbox": {"page": 1},
+                        "kind": "text",
+                    }],
+                },
+                {
+                    "content": "Page 3 block A",
+                    "blocks": [{
+                        "content": "Page 3 block A",
+                        "bbox": {"page": 3},
+                        "kind": "figure",
+                    }],
+                },
+            ],
+        },
+    })
+}
+
+#[rstest]
+fn test_parse_v3_file_upload_and_response_mapping(parse_response: Value) {
+    let source = classify_document_source("data:application/pdf;base64,JVBERi0xLjQ=")
+        .expect("PDF data URI should be valid");
+    let upload = build_upload_request(
+        source,
+        "Bearer test-key",
+        Some("https://platform.reducto.ai"),
+    )
+    .expect("data URI should require upload");
+    assert_eq!(upload.url, "https://platform.reducto.ai/upload");
+    assert_eq!(upload.authorization, "Bearer test-key");
+    assert_eq!(upload.file_name, "document");
+    assert_eq!(upload.mime_type, "application/pdf");
+    assert_eq!(upload.bytes, b"%PDF-1.4");
+
+    let optional_params = json!({
+        "formatting": {"table_output_format": "html"},
+        "retrieval": {"chunk_mode": "section"},
+        "settings": {"ocr_system": "standard"},
+    })
+    .as_object()
+    .expect("params should be an object")
+    .clone();
+    let request = build_parse_v3_request("reducto://uploaded.pdf", optional_params);
+    assert_eq!(
+        request.data,
+        json!({
+            "input": "reducto://uploaded.pdf",
+            "formatting": {"table_output_format": "html"},
+            "retrieval": {"chunk_mode": "section"},
+            "settings": {"ocr_system": "standard"},
+        })
+    );
+
+    let transformed = transform_reducto_response("parse-v3", parse_response.clone())
+        .expect("response should transform");
+    assert_eq!(
+        transformed.usage_info,
+        Some(json!({"pages_processed": 3, "credits": 3}))
+    );
+    assert_eq!(transformed.pages.len(), 3);
+    assert_eq!(
+        transformed.pages[0],
+        json!({
+            "index": 0,
+            "markdown": "Page 1 block A\n\nPage 1 block B",
+            "blocks": [
+                {"content": "Page 1 block A", "bbox": {"page": 1}, "kind": "text"},
+                {"content": "Page 1 block B", "bbox": {"page": 1}, "kind": "text"},
+            ],
+        })
+    );
+    assert_eq!(transformed.pages[1]["markdown"], "Page 2 block A");
+    assert_eq!(transformed.pages[2]["markdown"], "Page 3 block A");
+    assert_eq!(transformed.provider_native_response, Some(parse_response));
+}
+
+#[rstest]
+fn test_parse_v3_reducto_id_passthrough_skips_upload(parse_response: Value) {
+    let document = json!({
+        "type": "document_url",
+        "document_url": "reducto://already-uploaded.pdf",
+    });
+    let source = extract_document_source(&document).expect("Reducto ID should be valid");
+    assert!(build_upload_request(source.clone(), "Bearer test-key", None).is_none());
+    assert_eq!(
+        source,
+        ReductoDocumentSource::FileId("reducto://already-uploaded.pdf".to_string())
+    );
+
+    let request = REDUCTO_PARSE_V3_CONFIG
+        .transform_ocr_request(
+            "parse-v3",
+            document,
+            json!({"retrieval": {"chunk_mode": "section"}})
+                .as_object()
+                .expect("params should be object")
+                .clone(),
+        )
+        .expect("direct ID should transform");
+    assert_eq!(request.data["input"], "reducto://already-uploaded.pdf");
+    assert_eq!(request.data["retrieval"]["chunk_mode"], "section");
+
+    let response = REDUCTO_PARSE_V3_CONFIG
+        .transform_ocr_response("parse-v3", parse_response)
+        .expect("response should transform");
+    assert!(
+        response.pages[0]["markdown"]
+            .as_str()
+            .expect("markdown should be string")
+            .starts_with("Page 1 block A")
+    );
+}
+
+#[rstest]
+fn test_parse_legacy_wraps_enhance_under_options() {
+    let request = build_parse_legacy_request(
+        "reducto://legacy.pdf",
+        json!({"enhance": {"agentic": [{"type": "table"}]}})
+            .as_object()
+            .expect("params should be object"),
+    );
+    assert_eq!(
+        request.data,
+        json!({
+            "document_url": "reducto://legacy.pdf",
+            "options": {"enhance": {"agentic": [{"type": "table"}]}},
+        })
+    );
+}
+
+#[rstest]
+fn test_parse_v3_image_data_uri_upload_uses_image_mime() {
+    let source = classify_document_source("data:image/png;base64,iVBORw0KGgo=")
+        .expect("PNG data URI should be valid");
+    let upload = build_upload_request(
+        source,
+        "Bearer programmatic-key",
+        Some("https://custom.reducto.test/"),
+    )
+    .expect("data URI should require upload");
+    assert_eq!(upload.url, "https://custom.reducto.test/upload");
+    assert_eq!(upload.authorization, "Bearer programmatic-key");
+    assert_eq!(upload.mime_type, "image/png");
+    assert_eq!(upload.bytes, b"\x89PNG\r\n\x1a\n");
+}
+
+#[rstest]
+#[case::http("http://example.com/document.pdf")]
+#[case::https("https://example.com/document.pdf")]
+fn test_parse_v3_rejects_plain_http_urls(#[case] source: &str) {
+    let error = classify_document_source(source).expect_err("plain URL should be rejected");
+    assert!(error.to_string().contains("upload the file first"));
+}
+
+#[rstest]
+fn test_parse_v3_uses_programmatic_api_key_over_env() {
+    let key = resolve_api_key(Some("passed-key"), &|_| Some("env-reducto-key".to_string()))
+        .expect("explicit key should resolve");
+    assert_eq!(key, "passed-key");
+
+    let headers = REDUCTO_PARSE_V3_CONFIG
+        .validate_environment(Vec::new(), Some("passed-key"), &|_| {
+            Some("env-reducto-key".to_string())
+        })
+        .expect("headers should validate");
+    assert_eq!(
+        headers,
+        vec![("Authorization".to_string(), "Bearer passed-key".to_string())]
+    );
+}

litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs

@@ -0,0 +1,407 @@
+use std::collections::BTreeMap;
+
+use base64::Engine;
+use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
+use serde_json::{Map, Value, json};
+
+use crate::error::{Error, json_type_name};
+use crate::ocr::transformation::OcrProviderConfig;
+use crate::ocr::types::{OcrRequestData, OcrResponseData};
+
+pub const REDUCTO_API_BASE: &str = "https://platform.reducto.ai";
+pub const REDUCTO_API_KEY_ENV: &str = "REDUCTO_API_KEY";
+pub const REDUCTO_ID_PREFIX: &str = "reducto://";
+
+const PARSE_V3_SUPPORTED_OCR_PARAMS: &[&str] = &["formatting", "retrieval", "settings"];
+const PARSE_LEGACY_SUPPORTED_OCR_PARAMS: &[&str] = &["enhance"];
+const MISSING_KEY_MESSAGE: &str = "Missing REDUCTO_API_KEY - set it in the environment or pass api_key to litellm.ocr()/litellm.aocr()";
+const DATA_URI_UPLOAD_REQUIRED: &str =
+    "Reducto data URI upload must complete before OCR request transformation";
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum ReductoDocumentSource {
+    FileId(String),
+    Upload { bytes: Vec<u8>, mime_type: String },
+}
+
+#[derive(Clone, PartialEq, Eq)]
+pub struct ReductoUploadRequest {
+    pub url: String,
+    pub authorization: String,
+    pub file_name: &'static str,
+    pub bytes: Vec<u8>,
+    pub mime_type: String,
+}
+
+pub struct ReductoParseV3Config;
+pub struct ReductoParseLegacyConfig;
+
+pub const REDUCTO_PARSE_V3_CONFIG: ReductoParseV3Config = ReductoParseV3Config;
+pub const REDUCTO_PARSE_LEGACY_CONFIG: ReductoParseLegacyConfig = ReductoParseLegacyConfig;
+
+pub fn config_for_model(model: &str) -> Option<&'static dyn OcrProviderConfig> {
+    match model {
+        "parse-v3" => Some(&REDUCTO_PARSE_V3_CONFIG),
+        "parse-legacy" => Some(&REDUCTO_PARSE_LEGACY_CONFIG),
+        _ => None,
+    }
+}
+
+pub fn normalize_api_base(api_base: Option<&str>) -> String {
+    api_base
+        .map(str::trim)
+        .filter(|base| !base.is_empty())
+        .unwrap_or(REDUCTO_API_BASE)
+        .trim_end_matches('/')
+        .to_string()
+}
+
+pub fn parse_url(api_base: Option<&str>) -> String {
+    format!("{}/parse", normalize_api_base(api_base))
+}
+
+pub fn upload_url(api_base: Option<&str>) -> String {
+    format!("{}/upload", normalize_api_base(api_base))
+}
+
+pub fn resolve_api_key(
+    api_key: Option<&str>,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> Result<String, Error> {
+    api_key
+        .map(str::trim)
+        .filter(|key| !key.is_empty())
+        .map(str::to_string)
+        .or_else(|| {
+            env_lookup(REDUCTO_API_KEY_ENV)
+                .map(|key| key.trim().to_string())
+                .filter(|key| !key.is_empty())
+        })
+        .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string()))
+}
+
+pub fn extract_document_source(document: &Value) -> Result<ReductoDocumentSource, Error> {
+    let document = document.as_object().ok_or_else(|| Error::InvalidType {
+        expected: "object",
+        actual: json_type_name(document),
+    })?;
+    let source = document
+        .get("document_url")
+        .and_then(Value::as_str)
+        .filter(|source| !source.is_empty())
+        .or_else(|| document.get("image_url").and_then(Value::as_str))
+        .ok_or_else(|| {
+            Error::InvalidRequest(
+                "Reducto expected OCR preprocessing to produce document_url or image_url"
+                    .to_string(),
+            )
+        })?;
+    classify_document_source(source)
+}
+
+pub fn classify_document_source(source: &str) -> Result<ReductoDocumentSource, Error> {
+    if source.starts_with(REDUCTO_ID_PREFIX) {
+        return Ok(ReductoDocumentSource::FileId(source.to_string()));
+    }
+    if source.starts_with("http://") || source.starts_with("https://") {
+        return Err(Error::InvalidRequest(
+            "Reducto requires type='file' (auto-uploaded) or a reducto:// id. Plain http(s) URLs are not supported; upload the file first."
+                .to_string(),
+        ));
+    }
+    if !source.starts_with("data:") {
+        return Err(Error::InvalidRequest(
+            "Reducto requires a reducto:// id or a base64 data URI after OCR preprocessing."
+                .to_string(),
+        ));
+    }
+
+    let (header, encoded) = source
+        .split_once(',')
+        .ok_or_else(|| Error::InvalidRequest("Invalid Reducto data URI provided.".to_string()))?;
+    if !header.split(';').any(|part| part == "base64") {
+        return Err(Error::InvalidRequest(
+            "Reducto only supports base64-encoded data URIs.".to_string(),
+        ));
+    }
+
+    let mime_type = header
+        .strip_prefix("data:")
+        .and_then(|header| header.split(';').next())
+        .filter(|mime| !mime.is_empty())
+        .unwrap_or("application/octet-stream")
+        .to_string();
+    let bytes = BASE64_STANDARD.decode(encoded).map_err(|_| {
+        Error::InvalidRequest("Invalid Reducto base64 payload provided.".to_string())
+    })?;
+
+    Ok(ReductoDocumentSource::Upload { bytes, mime_type })
+}
+
+pub fn build_upload_request(
+    source: ReductoDocumentSource,
+    authorization: &str,
+    api_base: Option<&str>,
+) -> Option<ReductoUploadRequest> {
+    let ReductoDocumentSource::Upload { bytes, mime_type } = source else {
+        return None;
+    };
+
+    Some(ReductoUploadRequest {
+        url: upload_url(api_base),
+        authorization: authorization.to_string(),
+        file_name: "document",
+        bytes,
+        mime_type,
+    })
+}
+
+pub fn extract_upload_file_id(response_json: &Value) -> Result<&str, Error> {
+    response_json
+        .as_object()
+        .and_then(|response| response.get("file_id"))
+        .and_then(Value::as_str)
+        .filter(|file_id| !file_id.is_empty())
+        .ok_or_else(|| {
+            Error::InvalidResponse(format!(
+                "Reducto /upload returned 200 without a file_id; got payload={response_json}"
+            ))
+        })
+}
+
+pub fn build_parse_v3_request(
+    file_id: &str,
+    optional_params: Map<String, Value>,
+) -> OcrRequestData {
+    let data = std::iter::once(("input".to_string(), Value::String(file_id.to_string())))
+        .chain(optional_params)
+        .collect();
+    OcrRequestData {
+        data: Value::Object(data),
+        files: None,
+    }
+}
+
+pub fn build_parse_legacy_request(
+    file_id: &str,
+    optional_params: &Map<String, Value>,
+) -> OcrRequestData {
+    let options = optional_params
+        .get("enhance")
+        .filter(|enhance| !enhance.is_null())
+        .map(|enhance| json!({"options": {"enhance": enhance}}));
+    let data = match options {
+        Some(Value::Object(options)) => std::iter::once((
+            "document_url".to_string(),
+            Value::String(file_id.to_string()),
+        ))
+        .chain(options)
+        .collect(),
+        _ => Map::from_iter([(
+            "document_url".to_string(),
+            Value::String(file_id.to_string()),
+        )]),
+    };
+    OcrRequestData {
+        data: Value::Object(data),
+        files: None,
+    }
+}
+
+fn source_file_id(document: &Value) -> Result<String, Error> {
+    match extract_document_source(document)? {
+        ReductoDocumentSource::FileId(file_id) => Ok(file_id),
+        ReductoDocumentSource::Upload { .. } => Err(Error::Unsupported(DATA_URI_UPLOAD_REQUIRED)),
+    }
+}
+
+fn page_number(block: &Map<String, Value>) -> Option<i64> {
+    let page = block.get("bbox")?.as_object()?.get("page")?;
+    page.as_i64()
+        .or_else(|| page.as_u64().and_then(|page| i64::try_from(page).ok()))
+        .or_else(|| page.as_str().and_then(|page| page.parse().ok()))
+}
+
+fn chunks(result: &Map<String, Value>) -> &[Value] {
+    result
+        .get("chunks")
+        .and_then(Value::as_array)
+        .map(Vec::as_slice)
+        .unwrap_or_default()
+}
+
+fn build_pages(result: &Map<String, Value>) -> Vec<Value> {
+    let blocks_by_page = chunks(result)
+        .iter()
+        .filter_map(Value::as_object)
+        .filter_map(|chunk| chunk.get("blocks").and_then(Value::as_array))
+        .flatten()
+        .filter_map(|block| block.as_object().map(|object| (block, object)))
+        .filter_map(|(block, object)| page_number(object).map(|page| (page, block.clone())))
+        .fold(
+            BTreeMap::<i64, Vec<Value>>::new(),
+            |mut pages, (page, block)| {
+                pages.entry(page).or_default().push(block);
+                pages
+            },
+        );
+
+    if blocks_by_page.is_empty() {
+        let markdown = chunks(result)
+            .iter()
+            .filter_map(Value::as_object)
+            .filter_map(|chunk| chunk.get("content").and_then(Value::as_str))
+            .filter(|content| !content.is_empty())
+            .collect::<Vec<_>>()
+            .join("\n\n");
+        return if markdown.is_empty() {
+            Vec::new()
+        } else {
+            vec![json!({"index": 0, "markdown": markdown})]
+        };
+    }
+
+    blocks_by_page
+        .into_iter()
+        .map(|(page, blocks)| {
+            let markdown = blocks
+                .iter()
+                .filter_map(Value::as_object)
+                .filter_map(|block| block.get("content").and_then(Value::as_str))
+                .filter(|content| !content.is_empty())
+                .collect::<Vec<_>>()
+                .join("\n\n");
+            json!({
+                "index": page.saturating_sub(1).max(0),
+                "markdown": markdown,
+                "blocks": blocks,
+            })
+        })
+        .collect()
+}
+
+pub fn transform_reducto_response(
+    model: &str,
+    response_json: Value,
+) -> Result<OcrResponseData, Error> {
+    let response = response_json
+        .as_object()
+        .ok_or_else(|| Error::InvalidType {
+            expected: "object",
+            actual: json_type_name(&response_json),
+        })?;
+    let empty_result = Map::new();
+    let result = match response.get("result") {
+        Some(Value::Object(result)) => result,
+        Some(Value::Null) => &empty_result,
+        Some(_) => {
+            return Err(Error::InvalidResponse(
+                "Reducto result must be an object".to_string(),
+            ));
+        }
+        None => response,
+    };
+    let usage = response
+        .get("usage")
+        .and_then(Value::as_object)
+        .cloned()
+        .unwrap_or_default();
+    let usage_info = Some(json!({
+        "pages_processed": usage.get("num_pages").cloned().unwrap_or(Value::Null),
+        "credits": usage.get("credits").cloned().unwrap_or(Value::Null),
+    }));
+
+    Ok(OcrResponseData {
+        pages: build_pages(result),
+        model: model.to_string(),
+        document_annotation: None,
+        usage_info,
+        object: "ocr".to_string(),
+        extra_fields: Map::new(),
+        provider_native_response: Some(response_json),
+    })
+}
+
+impl OcrProviderConfig for ReductoParseV3Config {
+    fn supported_ocr_params(&self) -> &'static [&'static str] {
+        PARSE_V3_SUPPORTED_OCR_PARAMS
+    }
+
+    fn transform_ocr_request(
+        &self,
+        _model: &str,
+        document: Value,
+        optional_params: Map<String, Value>,
+    ) -> Result<OcrRequestData, Error> {
+        let file_id = source_file_id(&document)?;
+        Ok(build_parse_v3_request(&file_id, optional_params))
+    }
+
+    fn transform_ocr_response(
+        &self,
+        model: &str,
+        response_json: Value,
+    ) -> Result<OcrResponseData, Error> {
+        transform_reducto_response(model, response_json)
+    }
+
+    fn complete_url(
+        &self,
+        api_base: Option<&str>,
+        _model: &str,
+        _optional_params: &Map<String, Value>,
+        _env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        Ok(parse_url(api_base))
+    }
+
+    fn resolve_api_key(
+        &self,
+        api_key: Option<&str>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        resolve_api_key(api_key, env_lookup)
+    }
+}
+
+impl OcrProviderConfig for ReductoParseLegacyConfig {
+    fn supported_ocr_params(&self) -> &'static [&'static str] {
+        PARSE_LEGACY_SUPPORTED_OCR_PARAMS
+    }
+
+    fn transform_ocr_request(
+        &self,
+        _model: &str,
+        document: Value,
+        optional_params: Map<String, Value>,
+    ) -> Result<OcrRequestData, Error> {
+        let file_id = source_file_id(&document)?;
+        Ok(build_parse_legacy_request(&file_id, &optional_params))
+    }
+
+    fn transform_ocr_response(
+        &self,
+        model: &str,
+        response_json: Value,
+    ) -> Result<OcrResponseData, Error> {
+        transform_reducto_response(model, response_json)
+    }
+
+    fn complete_url(
+        &self,
+        api_base: Option<&str>,
+        _model: &str,
+        _optional_params: &Map<String, Value>,
+        _env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        Ok(parse_url(api_base))
+    }
+
+    fn resolve_api_key(
+        &self,
+        api_key: Option<&str>,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        resolve_api_key(api_key, env_lookup)
+    }
+}

litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs

@@ -212,6 +212,7 @@ impl OcrProviderConfig for VertexAiOcrConfig {
         MISTRAL_OCR_CONFIG.supported_ocr_params()
     }
 
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
     fn transform_ocr_request(
         &self,
         model: &str,
@@ -229,6 +230,7 @@ impl OcrProviderConfig for VertexAiOcrConfig {
         MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json)
     }
 
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
     fn complete_url(
         &self,
         api_base: Option<&str>,
@@ -253,10 +255,21 @@ impl OcrProviderConfig for VertexAiOcrConfig {
 }
 
 impl OcrProviderConfig for VertexAiDeepSeekOcrConfig {
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
     fn supported_ocr_params(&self) -> &'static [&'static str] {
         DEEPSEEK_SUPPORTED_OCR_PARAMS
     }
 
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
+    fn map_ocr_params(&self, non_default_params: &Map<String, Value>) -> Map<String, Value> {
+        non_default_params
+            .iter()
+            .filter(|(name, _)| DEEPSEEK_SUPPORTED_OCR_PARAMS.contains(&name.as_str()))
+            .map(|(name, value)| (name.clone(), value.clone()))
+            .collect()
+    }
+
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
     fn transform_ocr_request(
         &self,
         model: &str,
@@ -283,6 +296,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig {
         })
     }
 
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
     fn transform_ocr_response(
         &self,
         model: &str,
@@ -335,9 +349,12 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig {
             document_annotation: object.get("document_annotation").cloned(),
             usage_info,
             object: "ocr".to_string(),
+            extra_fields: Map::new(),
+            provider_native_response: None,
         })
     }
 
+    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
     fn complete_url(
         &self,
         api_base: Option<&str>,
@@ -360,6 +377,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use rstest::rstest;
 
     #[test]
     fn vertex_mistral_url_uses_project_location_and_model() {
@@ -411,6 +429,22 @@ mod tests {
         );
     }
 
+    #[rstest]
+    #[case::bare_model("deepseek-ocr-maas")]
+    #[case::namespaced_model("deepseek-ai/deepseek-ocr-maas")]
+    fn vertex_deepseek_request_uses_single_provider_namespace(#[case] model: &str) {
+        let body = VERTEX_AI_DEEPSEEK_OCR_CONFIG
+            .transform_ocr_request(
+                model,
+                json!({"type": "image_url", "image_url": "data:image/png;base64,AA=="}),
+                Map::new(),
+            )
+            .expect("request transforms")
+            .data;
+
+        assert_eq!(body["model"], "deepseek-ai/deepseek-ocr-maas");
+    }
+
     #[test]
     fn vertex_deepseek_response_wraps_markdown_content() {
         let response = VERTEX_AI_DEEPSEEK_OCR_CONFIG

litellm-rust/crates/core/tests/workspace_crate_allowlist.rs

@@ -1,6 +1,7 @@
-//! Enforcement: the litellm-rust workspace has exactly four crates.
+//! Enforcement: the litellm-rust workspace has exactly five crates.
 //!
-//! `core` (the Rust SDK), `ai-gateway` (the HTTP/WebSocket host),
+//! `core` (the Rust SDK), `config` (the config-loading boundary),
+//! `ai-gateway` (the HTTP/WebSocket host),
 //! `python-interop` (domain-neutral PyO3 primitives), and `python-bridge` (the
 //! PyO3 cdylib). Adding or removing a crate must be a
 //! deliberate act: this test fails until the allowlist here is updated, forcing
@@ -19,13 +20,20 @@ use std::path::{Path, PathBuf};
 /// workspace legitimately gains or loses a crate.
 const EXPECTED_MEMBERS: &[&str] = &[
     "crates/core",
+    "crates/config",
     "crates/ai-gateway",
     "crates/python-interop",
     "crates/python-bridge",
 ];
 
 /// The crate subdirectory names that must exist under `crates/`.
-const EXPECTED_CRATE_DIRS: &[&str] = &["core", "ai-gateway", "python-interop", "python-bridge"];
+const EXPECTED_CRATE_DIRS: &[&str] = &[
+    "core",
+    "config",
+    "ai-gateway",
+    "python-interop",
+    "python-bridge",
+];
 
 const MISMATCH: &str = "litellm-rust crate set changed — update this allowlist AND litellm-rust/AGENTS.md, and justify the crate per the rule (crate = layer needing independent compilation / its own deps / a separate artifact).";
 

litellm-rust/crates/python-bridge/Cargo.toml

@@ -14,11 +14,15 @@ default = ["abi3"]
 abi3 = ["pyo3/abi3-py310"]
 extension-module = ["pyo3/extension-module"]
 panic-test = []
+trace-parity = [
+    "dep:tracing",
+    "litellm-core/observability",
+    "litellm-ai-gateway/trace-parity",
+]
 
 [dependencies]
 futures-util.workspace = true
-tracing.workspace = true
-tracing-subscriber.workspace = true
+tracing = { workspace = true, optional = true }
 litellm-core = { workspace = true, features = ["bedrock-auth"] }
 litellm-ai-gateway = { workspace = true, default-features = false }
 litellm-python-interop.workspace = true
@@ -31,6 +35,7 @@ tokio.workspace = true
 [dev-dependencies]
 criterion = "0.8.2"
 tokio-tungstenite.workspace = true
+tracing.workspace = true
 
 [[bench]]
 name = "serialization"

litellm-rust/crates/python-bridge/src/constants.rs

@@ -1 +0,0 @@
-pub(crate) const FUNCTION_TRACE_TARGET: &str = "litellm::function_trace";

litellm-rust/crates/python-bridge/src/errors.rs

@@ -59,3 +59,41 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add("RustBridgeDeclined", py.get_type::<RustBridgeDeclined>())?;
     module.add("RustUpstreamError", py.get_type::<RustUpstreamError>())
 }
+
+pub(crate) fn ocr_error_to_pyerr(err: Error) -> PyErr {
+    match err {
+        Error::MissingField("document_url" | "image_url") => {
+            PyValueError::new_err("Document URL is required")
+        }
+        Error::Http { status, body } => RustUpstreamError::new_err((status, body)),
+        other => core_error_to_pyerr(other),
+    }
+}
+
+#[cfg(test)]
+mod ocr_error_tests {
+    use super::*;
+
+    #[test]
+    fn ocr_errors_preserve_python_validation_and_provider_details() {
+        Python::initialize();
+        Python::attach(|py| {
+            for field in ["document_url", "image_url"] {
+                let mapped = ocr_error_to_pyerr(Error::MissingField(field));
+                assert!(mapped.is_instance_of::<PyValueError>(py));
+                assert_eq!(mapped.value(py).to_string(), "Document URL is required");
+            }
+            let mapped = ocr_error_to_pyerr(Error::Http {
+                status: 429,
+                body: r#"{"message":"rate limited"}"#.to_string(),
+            });
+            assert!(mapped.is_instance_of::<RustUpstreamError>(py));
+            let args: (u16, String) = mapped
+                .value(py)
+                .getattr("args")
+                .and_then(|args| args.extract())
+                .expect("OCR failures retain status and unprefixed provider message");
+            assert_eq!(args, (429, r#"{"message":"rate limited"}"#.to_string()));
+        });
+    }
+}

litellm-rust/crates/python-bridge/src/function_trace.rs

@@ -1,216 +1,22 @@
 use std::future::Future;
-use std::sync::{Arc, Mutex};
 
+use litellm_core::observability::{FunctionTrace, FunctionTraceEvent};
 use serde::Serialize;
 use tracing::instrument::WithSubscriber;
-use tracing::span::{Attributes, Id};
-use tracing::{Dispatch, Level, Subscriber};
-use tracing_subscriber::filter::{LevelFilter, filter_fn};
-use tracing_subscriber::layer::Context;
-use tracing_subscriber::prelude::*;
-use tracing_subscriber::registry::LookupSpan;
-use tracing_subscriber::{Layer, Registry};
-
-use crate::constants::FUNCTION_TRACE_TARGET;
 
 #[derive(Serialize)]
-#[serde(untagged)]
-pub(crate) enum TraceResponse<T> {
-    Plain(T),
-    Traced {
-        response: T,
-        trace: Vec<FunctionTraceEvent>,
-    },
+pub(crate) struct TracedResponse<T> {
+    response: T,
+    trace: Vec<FunctionTraceEvent>,
 }
 
-pub(crate) async fn trace_call<T, E>(
+pub(crate) async fn capture<T, E>(
     future: impl Future<Output = Result<T, E>>,
-    enabled: bool,
-) -> Result<TraceResponse<T>, E> {
-    if !enabled {
-        return future.await.map(TraceResponse::Plain);
-    }
+) -> Result<TracedResponse<T>, E> {
     let trace = FunctionTrace::default();
     let response = future.with_subscriber(trace.dispatcher()).await?;
-    Ok(TraceResponse::Traced {
+    Ok(TracedResponse {
         response,
         trace: trace.events(),
     })
 }
-
-#[derive(Clone, Debug, PartialEq, Serialize)]
-pub struct FunctionTraceEvent {
-    pub function: &'static str,
-    pub depth: usize,
-}
-
-#[derive(Clone, Default)]
-pub struct FunctionTrace {
-    events: Arc<Mutex<Vec<FunctionTraceEvent>>>,
-}
-
-impl FunctionTrace {
-    pub fn dispatcher(&self) -> Dispatch {
-        let filter = filter_fn(|metadata| {
-            metadata.is_span()
-                && metadata.target() == FUNCTION_TRACE_TARGET
-                && *metadata.level() == Level::TRACE
-        })
-        .with_max_level_hint(LevelFilter::TRACE);
-        Dispatch::new(
-            Registry::default().with(
-                FunctionTraceLayer {
-                    trace: self.clone(),
-                }
-                .with_filter(filter),
-            ),
-        )
-    }
-
-    pub fn events(&self) -> Vec<FunctionTraceEvent> {
-        self.events
-            .lock()
-            .unwrap_or_else(|error| error.into_inner())
-            .clone()
-    }
-}
-
-struct FunctionTraceLayer {
-    trace: FunctionTrace,
-}
-
-impl<S> Layer<S> for FunctionTraceLayer
-where
-    S: Subscriber + for<'lookup> LookupSpan<'lookup>,
-{
-    fn on_new_span(&self, attributes: &Attributes<'_>, id: &Id, context: Context<'_, S>) {
-        let depth = context
-            .span(id)
-            .map(|span| span.scope().skip(1).count())
-            .unwrap_or_default();
-        self.trace
-            .events
-            .lock()
-            .unwrap_or_else(|error| error.into_inner())
-            .push(FunctionTraceEvent {
-                function: attributes.metadata().name(),
-                depth,
-            });
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
-    async fn outer() {
-        tokio::task::yield_now().await;
-        inner().await;
-    }
-
-    #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
-    async fn inner() {
-        tokio::task::yield_now().await;
-    }
-
-    #[tokio::test]
-    async fn concurrent_futures_keep_separate_traces_across_yields() {
-        use tracing::instrument::WithSubscriber;
-
-        let first = FunctionTrace::default();
-        let second = FunctionTrace::default();
-        let outside = FunctionTrace::default();
-
-        async {
-            tokio::join!(
-                outer().with_subscriber(first.dispatcher()),
-                inner().with_subscriber(second.dispatcher()),
-            );
-            inner().await;
-        }
-        .with_subscriber(outside.dispatcher())
-        .await;
-
-        assert_eq!(
-            first.events(),
-            vec![
-                FunctionTraceEvent {
-                    function: "outer",
-                    depth: 0
-                },
-                FunctionTraceEvent {
-                    function: "inner",
-                    depth: 1
-                },
-            ],
-        );
-        assert_eq!(
-            second.events(),
-            vec![FunctionTraceEvent {
-                function: "inner",
-                depth: 0
-            }],
-        );
-        assert_eq!(
-            outside.events(),
-            vec![FunctionTraceEvent {
-                function: "inner",
-                depth: 0
-            }],
-        );
-    }
-
-    #[test]
-    fn records_matching_spans_in_creation_order() {
-        let trace = FunctionTrace::default();
-        let dispatch = trace.dispatcher();
-
-        tracing::dispatcher::with_default(&dispatch, || {
-            let _ignored = tracing::trace_span!(target: "other", "ignored");
-            let _first = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name");
-            let _wrong_level = tracing::debug_span!(target: FUNCTION_TRACE_TARGET, "wrong_level");
-            let _second = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name");
-        });
-
-        assert_eq!(
-            trace.events(),
-            vec![
-                FunctionTraceEvent {
-                    function: "same_name",
-                    depth: 0,
-                },
-                FunctionTraceEvent {
-                    function: "same_name",
-                    depth: 0,
-                },
-            ]
-        );
-    }
-
-    #[test]
-    fn records_matching_span_nesting_depth() {
-        let trace = FunctionTrace::default();
-        let dispatch = trace.dispatcher();
-
-        tracing::dispatcher::with_default(&dispatch, || {
-            let outer = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "outer");
-            let _outer_guard = outer.enter();
-            let _inner = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "inner");
-        });
-
-        assert_eq!(
-            trace.events(),
-            vec![
-                FunctionTraceEvent {
-                    function: "outer",
-                    depth: 0,
-                },
-                FunctionTraceEvent {
-                    function: "inner",
-                    depth: 1,
-                },
-            ]
-        );
-    }
-}

litellm-rust/crates/python-bridge/src/lib.rs

@@ -1,8 +1,8 @@
-mod constants;
 mod diagnostics;
 mod errors;
 mod execution;
-pub mod function_trace;
+#[cfg(feature = "trace-parity")]
+mod function_trace;
 mod marshal;
 mod routes;
 
@@ -115,9 +115,43 @@ mod tests {
                 .extract::<Vec<String>>()
                 .expect("module names should be strings")
                 .into_iter()
-                .filter(|name| !name.starts_with("__"))
+                .filter(|name| !name.starts_with('_'))
                 .collect();
             assert_eq!(public_names, expected);
+
+            #[cfg(not(feature = "trace-parity"))]
+            assert!(!module.hasattr("_trace").expect("module lookup should work"));
+
+            #[cfg(feature = "trace-parity")]
+            {
+                let trace = module
+                    .getattr("_trace")
+                    .expect("trace build should expose its diagnostic namespace");
+                let trace_names: Vec<String> = trace
+                    .cast::<PyModule>()
+                    .expect("trace namespace should be a module")
+                    .dict()
+                    .keys()
+                    .extract::<Vec<String>>()
+                    .expect("trace names should be strings")
+                    .into_iter()
+                    .filter(|name| !name.starts_with("__"))
+                    .collect();
+                assert_eq!(
+                    trace_names,
+                    [
+                        "ocr",
+                        "aocr",
+                        "transcription",
+                        "atranscription",
+                        "messages",
+                        "amessages",
+                        "chat_completions",
+                        "achat_completions",
+                        "gateway_messages",
+                    ]
+                );
+            }
         });
     }
 

litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs

@@ -54,16 +54,16 @@ bridge_route! {
     required = {
         model: String,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        audio: Value,
+        audio: serde_json::Value,
     },
     optional = {
         api_key: Option<String>,
         api_base: Option<String>,
         custom_llm_provider: Option<String>,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        extra_headers: Option<Value>,
+        extra_headers: Option<serde_json::Value>,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        optional_params: Option<Value>,
+        optional_params: Option<serde_json::Value>,
         timeout_seconds: Option<f64>,
     },
     prepare = prepare_transcription,

litellm-rust/crates/python-bridge/src/routes/chat_completions.rs

@@ -73,16 +73,16 @@ bridge_route! {
     required = {
         model: String,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        messages: Value,
+        messages: serde_json::Value,
     },
     optional = {
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        optional_params: Option<Value>,
+        optional_params: Option<serde_json::Value>,
         api_key: Option<String>,
         api_base: Option<String>,
         custom_llm_provider: Option<String>,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        extra_headers: Option<Value>,
+        extra_headers: Option<serde_json::Value>,
         timeout_seconds: Option<f64>,
     },
     prepare = prepare_chat_completions,

litellm-rust/crates/python-bridge/src/routes/definition.rs

@@ -20,43 +20,33 @@ macro_rules! bridge_route {
         }
 
         #[pyfunction]
-        #[pyo3(signature = ($($required_name),*, $($optional_name=None,)* trace=false))]
+        #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))]
         #[allow(clippy::too_many_arguments)]
         fn $sync_name(
             py: pyo3::Python<'_>,
             $($(#[$required_attr])* $required_name: $required_type,)*
             $($(#[$optional_attr])* $optional_name: $optional_type,)*
-            trace: bool,
         ) -> pyo3::PyResult<pyo3::Py<pyo3::PyAny>> {
             let future = $prepare($inputs {
                 $($required_name,)*
                 $($optional_name),*
             })?;
-            $crate::execution::run_sync(
-                py,
-                $crate::function_trace::trace_call(future, trace),
-                $map_error,
-            )
+            $crate::execution::run_sync(py, future, $map_error)
         }
 
         #[pyfunction]
-        #[pyo3(signature = ($($required_name),*, $($optional_name=None,)* trace=false))]
+        #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))]
         #[allow(clippy::too_many_arguments)]
         fn $async_name(
             py: pyo3::Python<'_>,
             $($(#[$required_attr])* $required_name: $required_type,)*
             $($(#[$optional_attr])* $optional_name: $optional_type,)*
-            trace: bool,
         ) -> pyo3::PyResult<pyo3::Bound<'_, pyo3::PyAny>> {
             let future = $prepare($inputs {
                 $($required_name,)*
                 $($optional_name),*
             })?;
-            $crate::execution::run_async(
-                py,
-                $crate::function_trace::trace_call(future, trace),
-                $map_error,
-            )
+            $crate::execution::run_async(py, future, $map_error)
         }
 
         pub(super) fn register(
@@ -67,6 +57,71 @@ macro_rules! bridge_route {
             $crate::routes::definition::add_function(module, pyo3::wrap_pyfunction!($async_name, module)?)?;
             Ok(())
         }
+
+        #[cfg(feature = "trace-parity")]
+        mod trace {
+            use pyo3::prelude::*;
+            use super::{$inputs, $map_error, $prepare};
+
+            #[pyfunction]
+            #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))]
+            #[allow(clippy::too_many_arguments)]
+            fn $sync_name(
+                py: pyo3::Python<'_>,
+                $($(#[$required_attr])* $required_name: $required_type,)*
+                $($(#[$optional_attr])* $optional_name: $optional_type,)*
+            ) -> pyo3::PyResult<pyo3::Py<pyo3::PyAny>> {
+                let future = $prepare($inputs {
+                    $($required_name,)*
+                    $($optional_name),*
+                })?;
+                $crate::execution::run_sync(
+                    py,
+                    $crate::function_trace::capture(future),
+                    $map_error,
+                )
+            }
+
+            #[pyfunction]
+            #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))]
+            #[allow(clippy::too_many_arguments)]
+            fn $async_name(
+                py: pyo3::Python<'_>,
+                $($(#[$required_attr])* $required_name: $required_type,)*
+                $($(#[$optional_attr])* $optional_name: $optional_type,)*
+            ) -> pyo3::PyResult<pyo3::Bound<'_, pyo3::PyAny>> {
+                let future = $prepare($inputs {
+                    $($required_name,)*
+                    $($optional_name),*
+                })?;
+                $crate::execution::run_async(
+                    py,
+                    $crate::function_trace::capture(future),
+                    $map_error,
+                )
+            }
+
+            pub(super) fn register(
+                module: &pyo3::Bound<'_, pyo3::types::PyModule>,
+            ) -> pyo3::PyResult<()> {
+                $crate::routes::definition::add_function(
+                    module,
+                    pyo3::wrap_pyfunction!($sync_name, module)?,
+                )?;
+                $crate::routes::definition::add_function(
+                    module,
+                    pyo3::wrap_pyfunction!($async_name, module)?,
+                )?;
+                Ok(())
+            }
+        }
+
+        #[cfg(feature = "trace-parity")]
+        pub(super) fn register_trace(
+            module: &pyo3::Bound<'_, pyo3::types::PyModule>,
+        ) -> pyo3::PyResult<()> {
+            trace::register(module)
+        }
     };
 }
 
@@ -130,20 +185,26 @@ mod tests {
         ) -> PyResult<impl Future<Output = Result<String, Error>> + Send + 'static> {
             FUTURE_DROPPED.store(false, Ordering::SeqCst);
             let drop_guard = (inputs.value == "pending").then_some(DropGuard);
-            Ok(async move {
-                let _drop_guard = drop_guard;
-                tokio::task::yield_now().await;
-                match inputs.value.as_str() {
-                    "error" => Err(Error::InvalidRequest("synthetic error".to_string())),
-                    "map_panic" => Err(Error::InvalidRequest("panic in mapper".to_string())),
-                    "panic" => panic!("synthetic panic"),
-                    "pending" => {
-                        pending::<()>().await;
-                        unreachable!()
-                    }
-                    _ => Ok(inputs.value),
+            Ok(execute_echo(inputs, drop_guard))
+        }
+
+        #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
+        async fn execute_echo(
+            inputs: EchoInputs,
+            drop_guard: Option<DropGuard>,
+        ) -> Result<String, Error> {
+            let _drop_guard = drop_guard;
+            tokio::task::yield_now().await;
+            match inputs.value.as_str() {
+                "error" => Err(Error::InvalidRequest("synthetic error".to_string())),
+                "map_panic" => Err(Error::InvalidRequest("panic in mapper".to_string())),
+                "panic" => panic!("synthetic panic"),
+                "pending" => {
+                    pending::<()>().await;
+                    unreachable!()
                 }
-            })
+                _ => Ok(inputs.value),
+            }
         }
 
         fn map_error(error: Error) -> PyErr {
@@ -164,22 +225,22 @@ mod tests {
                 (
                     "ocr",
                     "aocr",
-                    "(model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None, trace=False)",
+                    "(model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None)",
                 ),
                 (
                     "transcription",
                     "atranscription",
-                    "(model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None, trace=False)",
+                    "(model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None)",
                 ),
                 (
                     "messages",
                     "amessages",
-                    "(model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None, trace=False)",
+                    "(model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None)",
                 ),
                 (
                     "chat_completions",
                     "achat_completions",
-                    "(model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None, trace=False)",
+                    "(model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None)",
                 ),
             ];
 
@@ -411,6 +472,32 @@ asyncio.run(exercise())
         });
     }
 
+    #[cfg(feature = "trace-parity")]
+    #[test]
+    fn diagnostic_route_returns_the_response_and_filtered_trace() {
+        Python::initialize();
+        Python::attach(|py| {
+            let module = PyModule::new(py, "synthetic").expect("module should be created");
+            synthetic::register_trace(&module).expect("trace routes should register");
+            let locals = PyDict::new(py);
+            locals
+                .set_item("routes", &module)
+                .expect("module should enter Python locals");
+            let code = CString::new(
+                r#"
+result = routes.echo("traced")
+assert result == {
+    "response": "traced",
+    "trace": [{"function": "execute_echo", "depth": 0}],
+}
+"#,
+            )
+            .expect("Python source should not contain null bytes");
+            py.run(&code, Some(&locals), Some(&locals))
+                .expect("diagnostic route should return its response and trace");
+        });
+    }
+
     #[test]
     fn route_registration_rejects_duplicate_python_names() {
         Python::initialize();

litellm-rust/crates/python-bridge/src/routes/gateway_messages.rs

@@ -0,0 +1,29 @@
+use pyo3::prelude::*;
+use serde_json::Value;
+
+use crate::errors::core_error_to_pyerr;
+
+#[pyfunction]
+fn gateway_messages<'py>(
+    py: Python<'py>,
+    model_alias: String,
+    provider_model: String,
+    api_base: String,
+    #[pyo3(from_py_with = litellm_python_interop::from_py)] body: Value,
+) -> PyResult<Bound<'py, PyAny>> {
+    let future = litellm_ai_gateway::trace_parity::messages_request(
+        model_alias,
+        provider_model,
+        api_base,
+        body,
+    );
+    crate::execution::run_async(
+        py,
+        crate::function_trace::capture(future),
+        core_error_to_pyerr,
+    )
+}
+
+pub(super) fn register_trace(module: &Bound<'_, PyModule>) -> PyResult<()> {
+    super::definition::add_function(module, wrap_pyfunction!(gateway_messages, module)?)
+}

litellm-rust/crates/python-bridge/src/routes/messages.rs

@@ -50,14 +50,14 @@ bridge_route! {
     required = {
         model: String,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        body: Value,
+        body: serde_json::Value,
     },
     optional = {
         api_key: Option<String>,
         api_base: Option<String>,
         custom_llm_provider: Option<String>,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        extra_headers: Option<Value>,
+        extra_headers: Option<serde_json::Value>,
         timeout_seconds: Option<f64>,
     },
     prepare = prepare_messages,

litellm-rust/crates/python-bridge/src/routes/mod.rs

@@ -3,6 +3,9 @@ use pyo3::prelude::*;
 #[macro_use]
 mod definition;
 
+#[cfg(feature = "trace-parity")]
+mod gateway_messages;
+
 mod audio_transcription;
 mod chat_completions;
 mod messages;
@@ -12,5 +15,16 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
     ocr::register(module)?;
     audio_transcription::register(module)?;
     messages::register(module)?;
-    chat_completions::register(module)
+    chat_completions::register(module)?;
+    #[cfg(feature = "trace-parity")]
+    {
+        let trace = PyModule::new(module.py(), "_trace")?;
+        ocr::register_trace(&trace)?;
+        audio_transcription::register_trace(&trace)?;
+        messages::register_trace(&trace)?;
+        chat_completions::register_trace(&trace)?;
+        gateway_messages::register_trace(&trace)?;
+        module.add_submodule(&trace)?;
+    }
+    Ok(())
 }

litellm-rust/crates/python-bridge/src/routes/ocr.rs

@@ -5,7 +5,7 @@ use litellm_ai_gateway::io::ocr::{OcrRequest, ocr as run_ocr};
 use pyo3::prelude::*;
 use serde_json::Value;
 
-use crate::errors::core_error_to_pyerr;
+use crate::errors::ocr_error_to_pyerr;
 use crate::marshal::{RouteOptions, RouteOptionsInputs, object_or_empty};
 
 fn prepare_ocr(
@@ -56,18 +56,18 @@ bridge_route! {
     required = {
         model: String,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        document: Value,
+        document: serde_json::Value,
     },
     optional = {
         api_key: Option<String>,
         api_base: Option<String>,
         custom_llm_provider: Option<String>,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        extra_headers: Option<Value>,
+        extra_headers: Option<serde_json::Value>,
         #[pyo3(from_py_with = litellm_python_interop::from_py)]
-        optional_params: Option<Value>,
+        optional_params: Option<serde_json::Value>,
         timeout_seconds: Option<f64>,
     },
     prepare = prepare_ocr,
-    errors = core_error_to_pyerr,
+    errors = ocr_error_to_pyerr,
 }

litellm-rust/crates/python-bridge/src/routes/runtime.rs

@@ -1,423 +0,0 @@
-use std::future::Future;
-use std::panic::AssertUnwindSafe;
-use std::time::Duration;
-
-use futures_util::FutureExt;
-use litellm_core::error::Error;
-use litellm_python_interop::{Pythonized, panic_to_pyerr, release_gil};
-use pyo3::exceptions::PyRuntimeError;
-use pyo3::prelude::*;
-use serde::Serialize;
-use tokio::runtime::{Handle, Runtime};
-use tokio::time::{self, MissedTickBehavior};
-
-pub(super) fn run_sync<T, F>(
-    py: Python<'_>,
-    future: F,
-    map_error: fn(Error) -> PyErr,
-) -> PyResult<Py<PyAny>>
-where
-    T: Serialize + Send + 'static,
-    F: Future<Output = Result<T, Error>> + Send + 'static,
-{
-    run_sync_on(
-        py,
-        pyo3_async_runtimes::tokio::get_runtime(),
-        future,
-        map_error,
-    )
-}
-
-fn run_sync_on<T, F>(
-    py: Python<'_>,
-    runtime: &Runtime,
-    future: F,
-    map_error: fn(Error) -> PyErr,
-) -> PyResult<Py<PyAny>>
-where
-    T: Serialize + Send + 'static,
-    F: Future<Output = Result<T, Error>> + Send + 'static,
-{
-    if Handle::try_current().is_ok() {
-        return Err(PyRuntimeError::new_err(
-            "synchronous native routes cannot run from a Tokio context; use the async route",
-        ));
-    }
-
-    let result = release_gil(py, move || runtime.block_on(wait_for_sync_result(future)))?;
-    let result = map_core_result(result, map_error)?;
-    Pythonized(result).into_pyobject(py).map(Bound::unbind)
-}
-
-pub(super) fn run_async<T, F>(
-    py: Python<'_>,
-    future: F,
-    map_error: fn(Error) -> PyErr,
-) -> PyResult<Bound<'_, PyAny>>
-where
-    T: Serialize + Send + 'static,
-    F: Future<Output = Result<T, Error>> + Send + 'static,
-{
-    pyo3_async_runtimes::tokio::future_into_py(py, async move {
-        let result = catch_route_panic(future).await?;
-        let result = map_core_result(result, map_error)?;
-        Ok(Pythonized(result))
-    })
-}
-
-fn map_core_result<T>(result: Result<T, Error>, map_error: fn(Error) -> PyErr) -> PyResult<T> {
-    match result {
-        Ok(value) => Ok(value),
-        Err(error) => Err(
-            std::panic::catch_unwind(AssertUnwindSafe(|| map_error(error)))
-                .map_err(panic_to_pyerr)?,
-        ),
-    }
-}
-
-async fn catch_route_panic<T, F>(future: F) -> PyResult<Result<T, Error>>
-where
-    F: Future<Output = Result<T, Error>>,
-{
-    AssertUnwindSafe(future)
-        .catch_unwind()
-        .await
-        .map_err(panic_to_pyerr)
-}
-
-async fn wait_for_sync_result<T, F>(future: F) -> PyResult<Result<T, Error>>
-where
-    F: Future<Output = Result<T, Error>>,
-{
-    let future = catch_route_panic(future);
-    tokio::pin!(future);
-
-    let signal_interval = Duration::from_millis(50);
-    let mut signal_checks =
-        time::interval_at(time::Instant::now() + signal_interval, signal_interval);
-    signal_checks.set_missed_tick_behavior(MissedTickBehavior::Delay);
-    loop {
-        tokio::select! {
-            result = &mut future => return result,
-            _ = signal_checks.tick() => Python::attach(|py| py.check_signals())?,
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::ffi::CString;
-    use std::future::poll_fn;
-    use std::sync::atomic::{AtomicUsize, Ordering};
-    use std::sync::{Arc, mpsc};
-    use std::task::Poll;
-    use std::thread;
-    use std::time::Instant;
-
-    use pyo3::panic::PanicException;
-    use pyo3::types::{PyDict, PyModule};
-    use serde::Serializer;
-    use tokio::runtime::Builder;
-
-    use super::*;
-
-    fn runtime_error(error: Error) -> PyErr {
-        PyRuntimeError::new_err(error.to_string())
-    }
-
-    fn panicking_error_mapper(_error: Error) -> PyErr {
-        panic!("error mapper panicked")
-    }
-
-    struct PanickingOutput;
-
-    static ASYNC_PROBE_COMPLETED: AtomicUsize = AtomicUsize::new(0);
-
-    impl Serialize for PanickingOutput {
-        fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
-        where
-            S: Serializer,
-        {
-            panic!("serializer panicked")
-        }
-    }
-
-    #[pyfunction]
-    fn async_serialization_panic(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
-        run_async(py, async { Ok(PanickingOutput) }, runtime_error)
-    }
-
-    #[pyfunction]
-    fn async_runtime_probe(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
-        run_async(
-            py,
-            async {
-                ASYNC_PROBE_COMPLETED.fetch_add(1, Ordering::SeqCst);
-                Ok(true)
-            },
-            runtime_error,
-        )
-    }
-
-    #[pyfunction]
-    fn runtime_worker_count() -> usize {
-        pyo3_async_runtimes::tokio::get_runtime()
-            .metrics()
-            .num_workers()
-    }
-
-    #[pyfunction]
-    fn runtime_is_responsive(_py: Python<'_>, expected_completions: usize) -> bool {
-        let completion_deadline = Instant::now() + Duration::from_secs(2);
-        while ASYNC_PROBE_COMPLETED.load(Ordering::SeqCst) < expected_completions {
-            if Instant::now() >= completion_deadline {
-                return false;
-            }
-            thread::sleep(Duration::from_millis(1));
-        }
-
-        let (heartbeat_tx, heartbeat_rx) = mpsc::sync_channel(1);
-        pyo3_async_runtimes::tokio::get_runtime().spawn(async move {
-            let _ = heartbeat_tx.send(());
-        });
-        heartbeat_rx.recv_timeout(Duration::from_secs(2)).is_ok()
-    }
-
-    fn extract_bool(py: Python<'_>, result: PyResult<Py<PyAny>>) -> bool {
-        result
-            .expect("route should complete")
-            .bind(py)
-            .extract()
-            .expect("result should convert")
-    }
-
-    #[test]
-    fn sync_runner_polls_future_on_the_caller_thread() {
-        Python::initialize();
-        Python::attach(|py| {
-            let caller_thread = std::thread::current().id();
-            let result = run_sync(
-                py,
-                async move { Ok(std::thread::current().id() == caller_thread) },
-                runtime_error,
-            );
-
-            assert!(extract_bool(py, result));
-        });
-    }
-
-    #[test]
-    fn sync_runner_releases_gil_while_waiting() {
-        Python::initialize();
-        Python::attach(|py| {
-            let result = run_sync(
-                py,
-                async {
-                    let gil_acquired = tokio::time::timeout(
-                        Duration::from_secs(2),
-                        tokio::task::spawn_blocking(|| Python::attach(|_| true)),
-                    )
-                    .await;
-                    Ok(matches!(gil_acquired, Ok(Ok(true))))
-                },
-                runtime_error,
-            );
-
-            assert!(extract_bool(py, result));
-        });
-    }
-
-    #[test]
-    fn sync_runner_rejects_calls_from_a_tokio_context() {
-        Python::initialize();
-        let runtime = Builder::new_current_thread()
-            .enable_all()
-            .build()
-            .expect("runtime should build");
-
-        let error = runtime.block_on(async {
-            Python::attach(|py| {
-                run_sync::<bool, _>(py, async { Ok(true) }, runtime_error)
-                    .expect_err("sync route should reject a nested Tokio runtime")
-            })
-        });
-
-        assert_eq!(
-            error.to_string(),
-            "RuntimeError: synchronous native routes cannot run from a Tokio context; use the async route"
-        );
-    }
-
-    #[test]
-    fn sync_runner_can_drive_a_current_thread_runtime() {
-        Python::initialize();
-        let runtime = Builder::new_current_thread()
-            .enable_all()
-            .build()
-            .expect("runtime should build");
-        Python::attach(|py| {
-            let result = run_sync_on(
-                py,
-                &runtime,
-                async {
-                    tokio::task::yield_now().await;
-                    Ok(true)
-                },
-                runtime_error,
-            );
-            assert!(extract_bool(py, result));
-        });
-    }
-
-    #[test]
-    fn sync_runner_maps_a_panicked_future() {
-        Python::initialize();
-        Python::attach(|py| {
-            let error = run_sync::<bool, _>(
-                py,
-                poll_fn(|_| -> Poll<Result<bool, Error>> { panic!("route future panicked") }),
-                runtime_error,
-            )
-            .expect_err("panicked route should become a Python exception");
-
-            assert!(error.is_instance_of::<PanicException>(py));
-            assert_eq!(error.to_string(), "PanicException: route future panicked");
-        });
-    }
-
-    #[test]
-    fn sync_runner_maps_a_panicked_error_mapper() {
-        Python::initialize();
-        Python::attach(|py| {
-            let error = run_sync::<bool, _>(
-                py,
-                async { Err(Error::InvalidRequest("invalid".to_string())) },
-                panicking_error_mapper,
-            )
-            .expect_err("panicked mapper should become a Python exception");
-
-            assert!(error.is_instance_of::<PanicException>(py));
-            assert_eq!(error.to_string(), "PanicException: error mapper panicked");
-        });
-    }
-
-    #[test]
-    fn sync_runner_surfaces_serializer_panics() {
-        Python::initialize();
-        Python::attach(|py| {
-            let error = run_sync(py, async { Ok(PanickingOutput) }, runtime_error)
-                .expect_err("serializer panic should become a Python exception");
-
-            assert!(error.is_instance_of::<PanicException>(py));
-            assert_eq!(error.to_string(), "PanicException: serializer panicked");
-        });
-    }
-
-    #[test]
-    fn sync_runner_supports_concurrent_callers_on_the_shared_runtime() {
-        Python::initialize();
-        let barrier = Arc::new(tokio::sync::Barrier::new(2));
-        let callers: Vec<_> = (0..2)
-            .map(|_| {
-                let barrier = Arc::clone(&barrier);
-                thread::spawn(move || {
-                    Python::attach(|py| {
-                        extract_bool(
-                            py,
-                            run_sync(
-                                py,
-                                async move {
-                                    Ok(tokio::time::timeout(Duration::from_secs(2), barrier.wait())
-                                        .await
-                                        .is_ok())
-                                },
-                                runtime_error,
-                            ),
-                        )
-                    })
-                })
-            })
-            .collect();
-        let results: Vec<_> = callers
-            .into_iter()
-            .map(|caller| caller.join().expect("caller should not panic"))
-            .collect();
-
-        assert_eq!(results, vec![true, true]);
-    }
-
-    #[test]
-    fn async_runner_surfaces_serializer_panics() {
-        Python::initialize();
-        Python::attach(|py| {
-            let module = PyModule::new(py, "runtime").expect("module should be created");
-            module
-                .add_function(
-                    wrap_pyfunction!(async_serialization_panic, &module)
-                        .expect("function should wrap"),
-                )
-                .expect("function should register");
-            let locals = PyDict::new(py);
-            locals
-                .set_item("runtime", &module)
-                .expect("module should enter Python locals");
-            let code = CString::new(
-                r#"
-import asyncio
-
-async def exercise():
-    try:
-        await runtime.async_serialization_panic()
-    except BaseException as error:
-        assert type(error).__name__ == "PanicException"
-        assert str(error) == "serializer panicked"
-    else:
-        raise AssertionError("serializer panic was not raised")
-
-asyncio.run(exercise())
-"#,
-            )
-            .expect("Python source should not contain null bytes");
-            py.run(&code, Some(&locals), Some(&locals))
-                .expect("serializer panic should reach the Python awaiter");
-        });
-    }
-
-    #[test]
-    fn async_result_delivery_does_not_stall_tokio_workers() {
-        Python::initialize();
-        ASYNC_PROBE_COMPLETED.store(0, Ordering::SeqCst);
-        Python::attach(|py| {
-            let module = PyModule::new(py, "runtime").expect("module should be created");
-            for function in [
-                wrap_pyfunction!(async_runtime_probe, &module).expect("function should wrap"),
-                wrap_pyfunction!(runtime_worker_count, &module).expect("function should wrap"),
-                wrap_pyfunction!(runtime_is_responsive, &module).expect("function should wrap"),
-            ] {
-                module
-                    .add_function(function)
-                    .expect("function should register");
-            }
-            let locals = PyDict::new(py);
-            locals
-                .set_item("runtime", &module)
-                .expect("module should enter Python locals");
-            let code = CString::new(
-                r#"
-import asyncio
-
-async def exercise():
-    worker_count = runtime.runtime_worker_count()
-    awaitables = [runtime.async_runtime_probe() for _ in range(worker_count)]
-    assert runtime.runtime_is_responsive(worker_count)
-    assert await asyncio.gather(*awaitables) == [True] * worker_count
-
-asyncio.run(exercise())
-"#,
-            )
-            .expect("Python source should not contain null bytes");
-            py.run(&code, Some(&locals), Some(&locals))
-                .expect("result delivery should leave Tokio workers responsive");
-        });
-    }
-}

litellm/__init__.py

@@ -1421,7 +1421,7 @@ from .skills.main import (
 )
 from .containers.main import *
 from .ocr.main import *
-from .rust_bridge import use_litellm_rust
+from .rust_bridge import rust
 from .rag.main import *
 from .sandbox.main import *
 from .search.main import *

litellm/_service_logger.py

@@ -1,4 +1,5 @@
 import asyncio
+from collections.abc import Callable, Coroutine
 from datetime import datetime, timedelta
 from typing import TYPE_CHECKING, Any, Final
 
@@ -83,6 +84,47 @@ class ServiceLogging(CustomLogger):
                 return open_telemetry_logger
         return None
 
+    @staticmethod
+    def _sync_dispatch_loop() -> asyncio.AbstractEventLoop | None:
+        """The event loop a blocking caller can dispatch on, or ``None`` if it has none."""
+        try:
+            loop: Final = asyncio.get_event_loop()
+        except RuntimeError:
+            return None
+        return None if loop.is_closed() else loop
+
+    @staticmethod
+    async def _emit_guarded(hook: Callable[[], Coroutine[object, object, None]]) -> None:
+        """Emit one service event, absorbing anything the callbacks raise.
+
+        Monitoring must not break the call it monitors. Sync callers are the ones that
+        swallow their own service failures (a Redis batch read returns an empty dict),
+        so an exception from a misconfigured callback would replace a Redis outage with
+        a callback error and skip the caller's fallback handling.
+        """
+        try:
+            await hook()
+        except Exception as e:
+            verbose_logger.exception("Error emitting service event - %s", e)
+
+    @staticmethod
+    def _dispatch_from_sync(hook: Callable[[], Coroutine[object, object, None]]) -> None:
+        """Run an async service hook from a blocking caller, whatever event loop it holds.
+
+        Takes a factory rather than a coroutine so the hook is built on the path that
+        runs it, and only ever once.
+        """
+        loop: Final = ServiceLogging._sync_dispatch_loop()
+        try:
+            if loop is None:
+                asyncio.run(ServiceLogging._emit_guarded(hook))
+            elif loop.is_running():
+                loop.create_task(ServiceLogging._emit_guarded(hook))
+            else:
+                loop.run_until_complete(ServiceLogging._emit_guarded(hook))
+        except Exception as e:
+            verbose_logger.exception("Error dispatching service event - %s", e)
+
     def service_success_hook(
         self,
         service: ServiceTypes,
@@ -99,54 +141,45 @@ class ServiceLogging(CustomLogger):
         if self.mock_testing:
             self.mock_testing_sync_success_hook += 1
 
-        try:
-            # Try to get the current event loop
-            loop: Final = asyncio.get_event_loop()
-            # Check if the loop is running
-            if loop.is_running():
-                # If we're in a running loop, create a task
-                loop.create_task(
-                    self.async_service_success_hook(
-                        service=service,
-                        duration=duration,
-                        call_type=call_type,
-                        parent_otel_span=parent_otel_span,
-                        start_time=start_time,
-                        end_time=end_time,
-                    )
-                )
-            else:
-                # Loop exists but not running, we can use run_until_complete
-                loop.run_until_complete(
-                    self.async_service_success_hook(
-                        service=service,
-                        duration=duration,
-                        call_type=call_type,
-                        parent_otel_span=parent_otel_span,
-                        start_time=start_time,
-                        end_time=end_time,
-                    )
-                )
-        except RuntimeError:
-            # No event loop exists, create a new one and run
-            asyncio.run(
-                self.async_service_success_hook(
-                    service=service,
-                    duration=duration,
-                    call_type=call_type,
-                    parent_otel_span=parent_otel_span,
-                    start_time=start_time,
-                    end_time=end_time,
-                )
+        self._dispatch_from_sync(
+            lambda: self.async_service_success_hook(
+                service=service,
+                duration=duration,
+                call_type=call_type,
+                parent_otel_span=parent_otel_span,
+                start_time=start_time,
+                end_time=end_time,
             )
+        )
 
-    def service_failure_hook(self, service: ServiceTypes, duration: float, error: Exception, call_type: str):
+    def service_failure_hook(
+        self,
+        service: ServiceTypes,
+        duration: float,
+        error: Exception,
+        call_type: str,
+        parent_otel_span: Span | None = None,
+        start_time: datetime | float | None = None,
+        end_time: float | datetime | None = None,
+    ):
         """
-        [TODO] Not implemented for sync calls yet. V0 is focused on async monitoring (used by proxy).
+        Handles both sync and async monitoring by checking for existing event loop.
         """
         if self.mock_testing:
             self.mock_testing_sync_failure_hook += 1
 
+        self._dispatch_from_sync(
+            lambda: self.async_service_failure_hook(
+                service=service,
+                duration=duration,
+                error=error,
+                call_type=call_type,
+                parent_otel_span=parent_otel_span,
+                start_time=start_time,
+                end_time=end_time,
+            )
+        )
+
     async def async_service_success_hook(
         self,
         service: ServiceTypes,

litellm/anthropic_interface/exceptions/exceptions.py

@@ -1,8 +1,9 @@
 """Anthropic error format type definitions."""
 
+from collections.abc import Mapping
 from typing import Literal
 
-from typing_extensions import Required, TypedDict
+from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
 
 # Known Anthropic error types
 # Source: https://docs.anthropic.com/en/api/errors
@@ -23,6 +24,7 @@ class AnthropicErrorDetail(TypedDict):
 
     type: AnthropicErrorType
     message: str
+    provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]]
 
 
 class AnthropicErrorResponse(TypedDict, total=False):

litellm/caching/caching.py

@@ -100,6 +100,7 @@ class Cache:
         qdrant_semantic_cache_vector_size: int | None = None,
         semantic_cache_embedding_max_input_tokens: int | None = None,
         semantic_cache_embedding_timeout: float | None = None,
+        semantic_cache_scope: str = SemanticCacheScope.KEY.value,
         # GCP IAM authentication parameters
         gcp_service_account: str | None = None,
         gcp_ssl_ca_certs: str | None = None,
@@ -127,6 +128,7 @@ class Cache:
             similarity_threshold (float, optional): The similarity threshold for semantic-caching, Required if type is "redis-semantic" or "qdrant-semantic".
             semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens.
             semantic_cache_embedding_timeout (float, optional): Seconds a semantic-cache lookup may spend embedding the prompt before it gives up and lets the request continue to the LLM. Defaults to SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS.
+            semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key".
 
             # Disk Cache Args
             disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
@@ -274,6 +276,7 @@ class Cache:
         self.redis_flush_size = redis_flush_size
         self.ttl = ttl
         self.mode: CacheMode = mode or CacheMode.default_on
+        self.semantic_cache_scope: str = SemanticCacheScope(semantic_cache_scope).value
 
         if self.type == LiteLLMCacheType.LOCAL and default_in_memory_ttl is not None:
             self.ttl = default_in_memory_ttl
@@ -301,6 +304,7 @@ class Cache:
         "user_api_key_team_id",
         "user_api_key_org_id",
     )
+    _SEMANTIC_CACHE_END_USER_SCOPE_FIELD: Final = "user_api_key_end_user_id"
 
     def _is_semantic_cache(self) -> bool:
         return self.type in (
@@ -309,19 +313,21 @@ class Cache:
             LiteLLMCacheType.VALKEY_SEMANTIC,
         )
 
-    def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str:
-        metadata: Final[dict] = kwargs.get("metadata") or {}
-        litellm_params: Final[dict] = kwargs.get("litellm_params") or {}
-        metadata_in_litellm_params: Final[dict] = litellm_params.get("metadata") or {}
+    def _semantic_cache_scope_fields(self) -> tuple[str, ...]:
+        if self.semantic_cache_scope == SemanticCacheScope.END_USER:
+            return (*self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS, self._SEMANTIC_CACHE_END_USER_SCOPE_FIELD)
+        return self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS
 
-        scope = ""
-        for field in self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS:
-            value = metadata.get(field)
-            if value is None:
-                value = metadata_in_litellm_params.get(field)
-            if value is not None:
-                scope += f"{field}: {value}"
-        return scope
+    def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str:
+        litellm_params: Final[dict] = kwargs.get("litellm_params") or {}
+        metadata_sources: Final[tuple[dict, ...]] = tuple(
+            source.get(key) or {} for source in (kwargs, litellm_params) for key in ("metadata", "litellm_metadata")
+        )
+        scope_values: Final = (
+            (field, next((source[field] for source in metadata_sources if source.get(field) is not None), None))
+            for field in self._semantic_cache_scope_fields()
+        )
+        return "".join(f"{field}: {value}" for field, value in scope_values if value is not None)
 
     def get_cache_key(self, **kwargs) -> str:
         """

litellm/caching/dual_cache.py

@@ -8,10 +8,9 @@ Has 4 primary methods:
     - async_get_cache
 """
 
-import asyncio
 import time
 import traceback
-from concurrent.futures import ThreadPoolExecutor
+from collections.abc import Sequence
 from threading import Lock
 from typing import TYPE_CHECKING, Any, Final
 
@@ -188,31 +187,38 @@ class DualCache(BaseCache):
         local_only: bool = False,
         **kwargs,
     ):
-        received_args: Final = locals()
-        received_args.pop("self")
-
-        def run_in_new_loop():
-            """Run the coroutine in a new event loop within this thread."""
-            new_loop: Final = asyncio.new_event_loop()
-            try:
-                asyncio.set_event_loop(new_loop)
-                return new_loop.run_until_complete(self.async_batch_get_cache(**received_args))
-            finally:
-                new_loop.close()
-                asyncio.set_event_loop(None)
-
         try:
-            # First, try to get the current event loop
-            _ = asyncio.get_running_loop()
-            # If we're already in an event loop, run in a separate thread
-            # to avoid nested event loop issues
-            with ThreadPoolExecutor(max_workers=1) as executor:
-                future: Final = executor.submit(run_in_new_loop)
-                return future.result()
+            in_memory_result: Final = (
+                self.in_memory_cache.batch_get_cache(keys, **kwargs) if self.in_memory_cache is not None else None
+            )
+            result: Final = in_memory_result if in_memory_result is not None else tuple(None for _ in keys)
 
-        except RuntimeError:
-            # No running event loop, we can safely run in this thread
-            return run_in_new_loop()
+            if None not in result or self.redis_cache is None or local_only:
+                return result
+
+            sublist_keys, previous_access_times = self._reserve_redis_batch_keys(time.time(), keys, result)
+            if len(sublist_keys) == 0:
+                return result
+
+            try:
+                redis_result: Final = self.redis_cache.batch_get_cache(
+                    key_list=sublist_keys, parent_otel_span=parent_otel_span
+                )
+            except Exception:
+                # Do not throttle subsequent callers if the Redis read fails.
+                self._rollback_redis_batch_key_reservations(previous_access_times)
+                raise
+
+            if self.in_memory_cache is not None:
+                for key, value in redis_result.items():
+                    if value is not None:
+                        self.in_memory_cache.set_cache(key, value, **self._backfill_kwargs(kwargs))
+
+            return list(  # mutable-ok: public list contract
+                redis_result.get(key) if value is None else value for key, value in zip(keys, result)
+            )
+        except Exception:
+            verbose_logger.error(traceback.format_exc())
 
     async def async_get_cache(
         self,
@@ -251,7 +257,7 @@ class DualCache(BaseCache):
         self,
         current_time: float,
         keys: list[str],
-        result: list[Any],
+        result: Sequence[Any],
     ) -> tuple[list[str], dict[str, float | None]]:
         """
         Atomically choose keys to fetch from Redis and reserve their access time.

litellm/caching/redis_cache.py

@@ -27,6 +27,7 @@ from litellm.constants import (
     REDIS_CIRCUIT_BREAKER_ENABLED,
     REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD,
     REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT,
+    REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION,
 )
 from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
 from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
@@ -41,6 +42,8 @@ from .base_cache import BaseCache
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
+    from prometheus_client import Counter as _PromCounter
+    from prometheus_client import Gauge as _PromGauge
     from redis.asyncio import Redis, RedisCluster
     from redis.asyncio.client import Pipeline
     from redis.asyncio.cluster import ClusterPipeline
@@ -78,10 +81,18 @@ class _AsyncRedisCommands(Protocol):
     def pipeline(self, transaction: bool = True) -> "Pipeline[bytes]": ...
 
 
+_BREAKER_GUARD_FRAME_NAMES: Final = frozenset(
+    {"<lambda>", "wrapper", "_run_under_circuit_breaker", "_run_under_circuit_breaker_sync"}
+)
+
+
 def _get_call_stack_info(num_frames: int = 2) -> str:
     """
     Get the function names from the previous 1-2 functions in the call stack.
 
+    Frames belonging to this module's circuit-breaker guards are skipped so the
+    reported callers stay the real ones even on guarded methods.
+
     Args:
         num_frames: Number of previous frames to include (default: 2)
 
@@ -102,11 +113,11 @@ def _get_call_stack_info(num_frames: int = 2) -> str:
             return "unknown"
         function_names: Final = []
 
-        for _ in range(num_frames):
-            if frame is None:
-                break
-            func_name = frame.f_code.co_name
-            function_names.append(func_name)
+        while frame is not None and len(function_names) < num_frames:
+            if frame.f_code.co_name in _BREAKER_GUARD_FRAME_NAMES and frame.f_globals.get("__name__") == __name__:
+                frame = frame.f_back
+                continue
+            function_names.append(frame.f_code.co_name)
             frame = frame.f_back
 
         if not function_names:
@@ -127,10 +138,20 @@ class RedisCircuitBreaker:
       HALF_OPEN - recovery probe: allow one request through
 
     Transitions:
-      CLOSED    -> OPEN      after failure_threshold consecutive failures
+      CLOSED    -> OPEN      after failure_threshold consecutive hard connectivity
+                             failures, or after an unbroken run of timeout failures
+                             (no success or hard failure in between) that reaches
+                             failure_threshold and spans timeout_min_duration seconds
       OPEN      -> HALF_OPEN after recovery_timeout seconds
       HALF_OPEN -> CLOSED    on success
       HALF_OPEN -> OPEN      on failure (resets timer)
+
+    Timeouts are accounted separately from hard connectivity failures because the async
+    Redis timeout includes time waiting for the worker event loop to resume: one loop
+    stall makes every in-flight operation time out together, which satisfies a purely
+    consecutive threshold instantly even though Redis is healthy. Requiring a
+    timeout-only streak to also span timeout_min_duration filters such bursts while a
+    real outage that surfaces as timeouts still opens the breaker after that duration.
     """
 
     CLOSED = "closed"
@@ -142,13 +163,19 @@ class RedisCircuitBreaker:
         failure_threshold: int,
         recovery_timeout: int,
         enabled: bool = True,
+        timeout_min_duration: float = REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION,
     ) -> None:
         self.failure_threshold = failure_threshold
         self.recovery_timeout = recovery_timeout
         self.enabled = enabled
+        self.timeout_min_duration = timeout_min_duration
         self._failure_count = 0
+        self._hard_failure_count = 0
+        self._timeout_count = 0
+        self._timeout_streak_started_at: float | None = None
         self._opened_at: float | None = None
         self._state = self.CLOSED
+        _breaker_metrics().record_state_change(None, self._state)
 
     def is_open(self) -> bool:
         """Returns True if Redis calls should be skipped."""
@@ -161,24 +188,45 @@ class RedisCircuitBreaker:
             return True
         if self._state == self.OPEN:
             if time.time() - (self._opened_at or 0) > self.recovery_timeout:
-                self._state = self.HALF_OPEN
+                self._set_state(self.HALF_OPEN)
                 return False  # this caller is the designated probe
             return True
         return False
 
-    def record_failure(self) -> None:
+    def _should_open(self, now: float) -> bool:
+        if self._state == self.HALF_OPEN:
+            return True
+        if self._hard_failure_count >= self.failure_threshold:
+            return True
+        if self._timeout_count < self.failure_threshold:
+            return False
+        return now - (self._timeout_streak_started_at or now) >= self.timeout_min_duration
+
+    def record_failure(self, is_timeout: bool = False) -> None:
         if not self.enabled:
             return
+        now: Final = time.time()
         self._failure_count += 1
-        self._opened_at = time.time()
-        if self._failure_count >= self.failure_threshold:
+        if is_timeout:
+            self._timeout_count += 1
+            if self._timeout_streak_started_at is None:
+                self._timeout_streak_started_at = now
+        else:
+            self._hard_failure_count += 1
+            self._timeout_count = 0
+            self._timeout_streak_started_at = None
+        self._opened_at = now
+        _breaker_metrics().record_failure("timeout" if is_timeout else "connectivity")
+        if self._should_open(now):
             if self._state != self.OPEN:
                 verbose_logger.warning(
-                    "Redis circuit breaker OPENED after %d consecutive failures — fast-failing Redis calls for %ds",
+                    "Redis circuit breaker OPENED after %d consecutive failures"
+                    " (%d hard connectivity) — fast-failing Redis calls for %ds",
                     self._failure_count,
+                    self._hard_failure_count,
                     self.recovery_timeout,
                 )
-            self._state = self.OPEN
+            self._set_state(self.OPEN)
 
     def record_success(self) -> None:
         if not self.enabled:
@@ -186,7 +234,17 @@ class RedisCircuitBreaker:
         if self._state == self.HALF_OPEN:
             verbose_logger.info("Redis circuit breaker CLOSED — Redis recovered")
         self._failure_count = 0
-        self._state = self.CLOSED
+        self._hard_failure_count = 0
+        self._timeout_count = 0
+        self._timeout_streak_started_at = None
+        self._set_state(self.CLOSED)
+
+    def _set_state(self, state: str) -> None:
+        if state == self._state:
+            return
+        _breaker_metrics().record_transition(state)
+        _breaker_metrics().record_state_change(self._state, state)
+        self._state = state
 
 
 _RedisCallResult = TypeVar("_RedisCallResult")
@@ -226,6 +284,78 @@ def _is_redis_health_failure(exc: BaseException) -> bool:
         return True
 
 
+@functools.lru_cache(maxsize=1)
+def _redis_timeout_error_types() -> tuple[type, ...]:
+    """Health failures that are timeouts rather than unambiguous connectivity errors.
+
+    ``builtins.TimeoutError`` covers ``asyncio.TimeoutError`` and ``socket.timeout``
+    (aliases since py3.11 / py3.10). ``redis.exceptions.TimeoutError`` does not subclass
+    either, so it is listed explicitly.
+    """
+    try:
+        from redis.exceptions import TimeoutError as RedisTimeoutError
+    except ImportError:
+        return (TimeoutError,)
+    return (RedisTimeoutError, TimeoutError)
+
+
+def _is_redis_timeout_failure(exc: BaseException) -> bool:
+    return isinstance(exc, _redis_timeout_error_types())
+
+
+class _BreakerMetrics:
+    """Prometheus metrics for the Redis circuit breaker; no-ops when the client is absent.
+
+    Registered lazily on the default registry (which /metrics serves) via the module-level
+    ``_breaker_metrics`` singleton so repeated RedisCache construction never re-registers.
+    """
+
+    def __init__(self) -> None:
+        self._state_gauge: _PromGauge | None = None
+        self._transitions: _PromCounter | None = None
+        self._failures: _PromCounter | None = None
+        try:
+            from prometheus_client import Counter as PromCounter
+            from prometheus_client import Gauge
+        except ImportError:
+            return
+        self._state_gauge = Gauge(
+            "litellm_redis_circuit_breaker_state",
+            "Number of Redis circuit breakers currently in each state",
+            labelnames=("state",),
+        )
+        self._transitions = PromCounter(
+            "litellm_redis_circuit_breaker_transitions",
+            "Redis circuit breaker state transitions",
+            labelnames=("state",),
+        )
+        self._failures = PromCounter(
+            "litellm_redis_circuit_breaker_failures",
+            "Redis health failures counted by the circuit breaker",
+            labelnames=("failure_class",),
+        )
+
+    def record_state_change(self, old_state: str | None, new_state: str) -> None:
+        if self._state_gauge is None:
+            return
+        if old_state is not None:
+            self._state_gauge.labels(old_state).dec()
+        self._state_gauge.labels(new_state).inc()
+
+    def record_transition(self, state: str) -> None:
+        if self._transitions is not None:
+            self._transitions.labels(state).inc()
+
+    def record_failure(self, failure_class: str) -> None:
+        if self._failures is not None:
+            self._failures.labels(failure_class).inc()
+
+
+@functools.lru_cache(maxsize=1)
+def _breaker_metrics() -> _BreakerMetrics:
+    return _BreakerMetrics()
+
+
 def _record_swallowed_redis_failure(breaker: RedisCircuitBreaker, exc: BaseException) -> None:
     """Record a Redis failure that the calling method is about to swallow.
 
@@ -237,10 +367,27 @@ def _record_swallowed_redis_failure(breaker: RedisCircuitBreaker, exc: BaseExcep
     """
     if not _is_redis_health_failure(exc):
         return
-    breaker.record_failure()
+    breaker.record_failure(is_timeout=_is_redis_timeout_failure(exc))
     _swallowed_redis_failures.set(_swallowed_redis_failures.get() + 1)
 
 
+def _enter_circuit_breaker(breaker: RedisCircuitBreaker, name: str) -> int:
+    """Reject the call if the breaker is open, else return the swallowed-failure count to compare against."""
+    if breaker.is_open():
+        raise Exception(f"Redis circuit breaker is open — skipping {name}")
+    return _swallowed_redis_failures.get()
+
+
+def _exit_circuit_breaker(breaker: RedisCircuitBreaker, swallowed_before: int) -> None:
+    """Record success only when nothing failed while the call ran.
+
+    Several Redis methods catch their own connection errors and return a default, so a
+    method that returned is not on its own proof of a healthy Redis.
+    """
+    if _swallowed_redis_failures.get() == swallowed_before:
+        breaker.record_success()
+
+
 async def _run_under_circuit_breaker(
     breaker: RedisCircuitBreaker,
     name: str,
@@ -249,20 +396,33 @@ async def _run_under_circuit_breaker(
     """Run one Redis coroutine under a circuit breaker.
 
     Shared by the method decorator and the Lua script executor so both feed the same
-    health signal. Success is recorded only when nothing failed while ``call`` ran,
-    because several Redis methods catch their own connection errors and return a default.
+    health signal.
     """
-    if breaker.is_open():
-        raise Exception(f"Redis circuit breaker is open — skipping {name}")
-    swallowed_before: Final = _swallowed_redis_failures.get()
+    swallowed_before: Final = _enter_circuit_breaker(breaker, name)
     try:
         result: Final = await call()
+    except Exception as e:
+        if _is_redis_health_failure(e):
+            breaker.record_failure(is_timeout=_is_redis_timeout_failure(e))
+        raise
+    _exit_circuit_breaker(breaker, swallowed_before)
+    return result
+
+
+def _run_under_circuit_breaker_sync(
+    breaker: RedisCircuitBreaker,
+    name: str,
+    call: Callable[[], _RedisCallResult],
+) -> _RedisCallResult:
+    """Run one blocking Redis call under a circuit breaker, feeding the same health signal as the async path."""
+    swallowed_before: Final = _enter_circuit_breaker(breaker, name)
+    try:
+        result: Final = call()
     except Exception as e:
         if _is_redis_health_failure(e):
             breaker.record_failure()
         raise
-    if _swallowed_redis_failures.get() == swallowed_before:
-        breaker.record_success()
+    _exit_circuit_breaker(breaker, swallowed_before)
     return result
 
 
@@ -288,6 +448,14 @@ def _redis_circuit_breaker_guard(method):
     return wrapper
 
 
+def _redis_circuit_breaker_guard_sync(method: Callable[..., _RedisCallResult]) -> Callable[..., _RedisCallResult]:
+    return functools.wraps(method)(
+        lambda self, *args, **kwargs: _run_under_circuit_breaker_sync(
+            self._circuit_breaker, method.__name__, lambda: method(self, *args, **kwargs)
+        )
+    )
+
+
 class RedisCache(BaseCache):
     # if users don't provider one, use the default litellm cache
 
@@ -1146,14 +1314,13 @@ class RedisCache(BaseCache):
         """
         key_value_dict = {}
         _key_list: Final = [key for key in key_list if key is not None]
+        start_time: Final = time.time()
 
         try:
-            _keys: Final = []
-            for cache_key in _key_list:
-                cache_key = self.check_and_fix_namespace(key=cache_key or "")
-                _keys.append(cache_key)
-            start_time: Final = time.time()
+            swallowed_before: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache")
+            _keys: Final = [self.check_and_fix_namespace(key=cache_key or "") for cache_key in _key_list]
             results: Final = self._run_redis_mget_operation(keys=_keys)
+            _exit_circuit_breaker(self._circuit_breaker, swallowed_before)
             end_time: Final = time.time()
             _duration: Final = end_time - start_time
             self.service_logger_obj.service_success_hook(
@@ -1178,7 +1345,18 @@ class RedisCache(BaseCache):
 
             return decoded_results
         except Exception as e:
+            failed_at: Final = time.time()
+            self.service_logger_obj.service_failure_hook(
+                service=ServiceTypes.REDIS,
+                duration=failed_at - start_time,
+                error=e,
+                call_type=f"batch_get_cache <- {_get_call_stack_info()}",
+                start_time=start_time,
+                end_time=failed_at,
+                parent_otel_span=parent_otel_span,
+            )
             verbose_logger.error("Error occurred in batch get cache - %s", e)
+            _record_swallowed_redis_failure(self._circuit_breaker, e)
             return key_value_dict
 
     @_redis_circuit_breaker_guard

litellm/caching/redis_cluster_node_isolation.py

@@ -19,13 +19,13 @@ connections untouched. Every other branch (MOVED, ASK, CLUSTERDOWN, slot-not-cov
 retry-exhaustion) is unchanged from upstream, since those already carry real evidence the
 topology changed.
 
-redis-py 8.x fixed this upstream with gentler machinery than this override's
-``node.disconnect()`` (which also kills connections other coroutines are mid-operation
-on, so one timeout cascades into a reconnect storm and, with TLS, a fresh handshake per
-killed connection): it marks in-use connections for reconnect only after their current
-operation completes, disconnects only the idle pooled ones, and defers reinitialization
-to the outer retry loop. When the installed ``ClusterNode`` has that per-connection
-recovery API, the factory returns the base ``RedisCluster`` unmodified.
+redis-py 8.x recovers connections per-connection, so the copied override is not used. Upstream
+still flips the shared ``_initialize`` flag on any node's timeout, funneling every concurrent
+caller through the reinit lock and, if ``CLUSTER SLOTS`` lands on the slow node, into a full
+teardown. For those versions the factory returns a thin wrapper around upstream's
+``_execute_command`` that clears the flag again after an isolated timeout (a ConnectionError,
+a third consecutive timeout on the same node, or a concurrent request from any other command
+or ``aclose()`` still reinits).
 """
 
 import asyncio
@@ -44,6 +44,8 @@ class _ClusterNodeAttrs(Protocol):
     mode; typing ``target_node`` as this Protocol at the one boundary keeps the override's
     own logic fully typed without a banned ``typing.cast``."""
 
+    name: str
+
     async def execute_command(
         self,
         *args: object,
@@ -78,18 +80,20 @@ class _ClusterAttrs(Protocol):
 #: this override can't see (Python won't error -- it'll just run our now-stale copy), so
 #: construction logs a loud warning rather than silently trusting an unverified copy.
 _VERIFIED_REDIS_VERSIONS: Final = frozenset({"5.3.1"})
+_CONSECUTIVE_TIMEOUTS_BEFORE_REINIT: Final = 3
 
 
-def get_litellm_async_redis_cluster_class(
+def get_litellm_async_redis_cluster_class(  # noqa: C901  # supports redis-py version-specific cluster implementations
     cluster_node_class: type | None = None,
+    base_cluster_class: type | None = None,
 ) -> type["_AsyncRedisClusterType"]:
-    """Returns the base ``RedisCluster`` when the installed redis-py already recovers a
-    node-level connection error per-connection (8.x+), else builds the ``RedisCluster``
-    subclass with the per-node isolation fix for older versions whose upstream branch
-    tears down the whole cluster client.
+    """Returns a timeout-tolerant ``RedisCluster`` subclass when installed redis-py already
+    recovers node-level connections per-connection (8.x+), else builds the ``RedisCluster``
+    subclass with the per-node isolation fix for older versions whose upstream branch tears
+    down the whole cluster client.
 
-    ``cluster_node_class`` exists for dependency injection in tests; production callers
-    leave it unset and the installed ``ClusterNode`` is used.
+    ``cluster_node_class`` and ``base_cluster_class`` exist for dependency injection in tests;
+    production callers leave them unset and the installed redis-py classes are used.
 
     Imported lazily because this module is reachable from a base ``import litellm`` while
     redis is not a base dependency. Cheap to call repeatedly: the underlying redis
@@ -118,13 +122,68 @@ def get_litellm_async_redis_cluster_class(
     from redis.exceptions import TimeoutError as _RedisTimeoutError
 
     node_class: Final = cluster_node_class if cluster_node_class is not None else _AsyncClusterNode
+    base_class: Final = base_cluster_class if base_cluster_class is not None else _BaseAsyncRedisCluster
     if hasattr(node_class, "update_active_connections_for_reconnect"):
         verbose_logger.debug(
-            "redis-py %s recovers a node-level connection error per-connection upstream; "
-            "using the base RedisCluster without litellm's node-isolation override.",
+            "redis-py %s recovers node connections per-connection upstream; using "
+            "LiteLLM's timeout-tolerant RedisCluster wrapper.",
             redis.__version__,
         )
-        return _BaseAsyncRedisCluster
+
+        class LiteLLMAsyncRedisClusterTimeoutTolerant(
+            base_class  # pyright: ignore[reportGeneralTypeIssues, reportUntypedBaseClass]  # the injected base class is selected at runtime
+        ):
+            def __init__(
+                self,
+                *args: object,
+                **kwargs: object,  # kwargs-ok: passes redis-py's constructor kwargs through untouched
+            ) -> None:
+                self._litellm_initialize = False
+                self._litellm_reinit_requests = 0
+                self._litellm_tolerated_timeouts = 0
+                super().__init__(*args, **kwargs)
+                self._litellm_consecutive_timeouts: dict[  # mutable-ok: per-node counter updated on the command hot path
+                    str, int
+                ] = {}
+
+            @property
+            def _initialize(self) -> bool:
+                return self._litellm_initialize
+
+            @_initialize.setter
+            def _initialize(self, value: bool) -> None:
+                if value:
+                    self._litellm_reinit_requests += 1
+                self._litellm_initialize = value
+
+            async def _execute_command(
+                self,
+                target_node: _ClusterNodeAttrs,
+                *args: object,
+                **kwargs: object,  # kwargs-ok: matches redis-py's own command dispatch signature
+            ) -> object:
+                outstanding_before: Final = self._litellm_reinit_requests - self._litellm_tolerated_timeouts
+                pending_before: Final = self._litellm_initialize
+                try:
+                    result: Final = await super()._execute_command(target_node, *args, **kwargs)
+                except _RedisTimeoutError:
+                    timeouts: Final = self._litellm_consecutive_timeouts.get(target_node.name, 0) + 1
+                    if timeouts >= _CONSECUTIVE_TIMEOUTS_BEFORE_REINIT:
+                        self._litellm_consecutive_timeouts.pop(target_node.name, None)
+                        raise
+                    self._litellm_consecutive_timeouts[target_node.name] = timeouts
+                    self._litellm_tolerated_timeouts += 1
+                    if (
+                        not pending_before
+                        and self._litellm_reinit_requests - self._litellm_tolerated_timeouts == outstanding_before
+                    ):
+                        self._initialize = False
+                    raise
+                if self._litellm_consecutive_timeouts:
+                    self._litellm_consecutive_timeouts.pop(target_node.name, None)
+                return result
+
+        return LiteLLMAsyncRedisClusterTimeoutTolerant
 
     if redis.__version__ not in _VERIFIED_REDIS_VERSIONS:
         verbose_logger.warning(

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -932,7 +932,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         return OpenAiResponsesToChatCompletionStreamIterator(streaming_response, sync_stream, json_mode)
 
     def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]:
-        if role == "user" or role == "system" or role == "tool":
+        if role in ("user", "system", "developer", "tool"):
             return {"type": "input_text", "text": content}
         else:
             return {"type": "output_text", "text": content}

litellm/constants.py

@@ -7,6 +7,7 @@ from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_in_ran
 
 DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"))
 AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview"))
+AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"})
 ROUTER_MAX_FALLBACKS: Final = int(os.getenv("ROUTER_MAX_FALLBACKS", 5))
 ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000
 RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
@@ -39,6 +40,7 @@ ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset(
         "router_general_settings",
         "ignore_invalid_deployments",
         "fallback_access_check",
+        "heuristic_v2_router_limit",
     }
 )
 DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512))
@@ -149,6 +151,7 @@ DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD = float(os.getenv("DEFAULT_SEMANTIC_
 MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS: Final = int(os.getenv("MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS", "60"))
 MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE: Final = int(os.getenv("MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE", "200"))
 MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL: Final = int(os.getenv("MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL", "3600"))
+MCP_SSO_ASSERTION_CACHE_TTL_SECONDS: Final = int(os.getenv("MCP_SSO_ASSERTION_CACHE_TTL_SECONDS", "60"))
 
 # Default npm cache directory for STDIO MCP servers.
 # npm/npx needs a writable cache dir; in containers the default (~/.npm)
@@ -430,6 +433,9 @@ REDIS_CONNECTION_POOL_TIMEOUT: Final = int(os.getenv("REDIS_CONNECTION_POOL_TIME
 REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD: Final = int(os.getenv("REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", 5))
 REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT: Final = int(os.getenv("REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", 60))
 REDIS_CIRCUIT_BREAKER_ENABLED: Final = os.getenv("REDIS_CIRCUIT_BREAKER_ENABLED", "true").lower() == "true"
+# minimum seconds a timeout-only failure streak must span before it can open the breaker,
+# so one event-loop stall timing out many queued calls at once does not trip it
+REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION: Final = float(os.getenv("REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION", 5.0))
 # Seconds of idle before a Redis cluster connection is validated with a PING and
 # reconnected if dead, so a connection silently dropped by a cluster restart
 # (e.g. ElastiCache Serverless maintenance) is not reused while broken
@@ -1450,6 +1456,7 @@ SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affin
 CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
 SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
+SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
 LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated"
 LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = (
     "Truncation is a DB storage safeguard. "

litellm/experimental_mcp_client/client.py

@@ -70,7 +70,7 @@ def to_basic_auth(auth_value: str) -> str:
 
 
 def strip_auth_scheme(auth_value: str, scheme: str) -> str:
-    """Return ``auth_value`` with a leading ``<scheme> `` removed, or unchanged when absent.
+    """Return ``auth_value`` with a leading ``<scheme>`` and separator removed, or unchanged when absent.
 
     Callers supply both a bare credential and a complete header value, so prefixing
     unconditionally yields ``Bearer Bearer <jwt>``. Scheme names are case-insensitive per
@@ -78,10 +78,9 @@ def strip_auth_scheme(auth_value: str, scheme: str) -> str:
     with the scheme text and a scheme with nothing behind it are returned untouched.
     Surrounding whitespace is left to ``_strip_header_whitespace`` at header-build time.
     """
-    scheme_name, _, remainder = auth_value.lstrip().partition(" ")
-    credential: Final = remainder.lstrip()
-    if credential and scheme_name.lower() == scheme.lower():
-        return credential
+    parts: Final = auth_value.split(None, 1)
+    if len(parts) == 2 and parts[0].lower() == scheme.lower():
+        return parts[1]
     return auth_value